merge cloud/integration; regenerate VM scripts

This commit is contained in:
Neil
2026-11-15 04:06:25 -08:00
43 changed files with 1167 additions and 72 deletions
@@ -7,6 +7,7 @@ import type { SandboxProviderApi } from '../src/sandbox-provider'
export const VM_SCRIPTS = [
'orca-sandbox-start',
'orca-sandbox-drain',
'orca-memguard',
'orca-sandbox-env',
'orca-session-env.sh',
'orca-env',
@@ -220,7 +220,7 @@ if [ -n "${NOVNC_VERSION:-}" ]; then
fi
log 'Orca entry points'
for name in orca-sandbox-start orca-sandbox-drain orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
for name in orca-sandbox-start orca-sandbox-drain orca-memguard orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
var="ORCA_VM_SCRIPT_${name//[-.]/_}"
printf '%s' "${!var}" | base64 -d >"$ORCA_HOME/bin/$name"
chmod 0755 "$ORCA_HOME/bin/$name"
File diff suppressed because one or more lines are too long
+308
View File
@@ -0,0 +1,308 @@
#!/usr/bin/env bash
# Memory guard for the workspace VM (runs as root: `sudo -n orca-memguard start`). A build or test
# that eats the VM's memory must cost that process, never the workspace: without this the kernel
# thrashes for minutes before its OOM killer picks something, and the VM looks dead meanwhile.
# start start the guard for this boot unless it already runs (returns at once)
# run the guard loop itself (start runs it detached)
# release-swap turn off and delete on-demand swap (before a stop, so the snapshot stays small)
# sweep apply OOM-killer priorities once and exit (tests, diagnostics)
# victim print "<pid> <rss KiB> <name>" of the process the guard would stop now (no kill)
# Every few seconds it ranks processes for the kernel's OOM killer (oom_score_adj):
# -900 Orca's own runtime: orcad, its terminal daemon, the share proxy (Orca's bundled Node)
# -500 terminal shells (children of the terminal daemon) and agent CLIs
# 500 everything else (builds, tests, dev servers, browsers); never lowered
# When MemAvailable falls under KILL_PCT (8%) of RAM it kills the largest process ranked 0 or more,
# records it in $RUN/kills and $RUN/state, and waits for memory to return. The kernel's OOM killer
# stays the backstop for allocations faster than the guard's 200 ms look, with the same ranks.
# Swap is added on demand (2 GiB files up to 1.5x RAM) when memory runs low, if the kernel takes it,
# and dropped after 10 quiet minutes. It also sizes /dev/shm to half of RAM (the provider mounts
# 64 MB, which crashes Chrome and Electron renderers).
# $RUN/state (key=value, world-readable) is what the control plane's status probe reads.
set -uo pipefail
RUN=/run/orca-memguard
# Tests point it at a fake process table.
PROC="${ORCA_MEMGUARD_PROC:-/proc}"
SWAP_DIR=/var/lib/orca-memguard
ORCA_HOME=/opt/orca
KILL_PCT="${ORCA_MEMGUARD_KILL_PCT:-8}"
HIGH_PCT=15
SWAP_ADD_PCT=20
QUIET_PCT=50
QUIET_SECONDS=600
TICK=1
FAST_TICK=0.2
SWEEP_EVERY=3
STATE_EVERY=15
SWAP_CHUNK_KIB=$((2 * 1024 * 1024))
boot_id() { local id; read -r id </proc/sys/kernel/random/boot_id && printf '%s' "$id"; }
now_ms() { local t="${EPOCHREALTIME//[.,]/}"; printf '%s' "${t:0:13}"; }
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$RUN/memguard.log"; }
# Name of a process for people: the script a node/python/... runs, else its command name.
proc_name() {
local pid="$1" comm arg name='' skip=0 module=0 args=()
read -r comm 2>/dev/null <"$PROC/$pid/comm" || comm=''
mapfile -d '' -t args 2>/dev/null <"$PROC/$pid/cmdline" || true
case "${args[0]##*/}" in
node | nodejs | bun | deno | python* | ruby | perl | java | sh | bash | dash | zsh)
for arg in "${args[@]:1}"; do
if [ "$skip" -eq 1 ]; then skip=0; continue; fi
if [ "$module" -eq 1 ]; then name="${arg##*/}"; break; fi
case "$arg" in
# Inline code is no name (and may hold anything): the interpreter's name it is.
-e | -p | -c | --eval | --print) break ;;
-m | -jar) module=1 ;; # python -m pytest, java -jar gradle.jar: the module or jar
-r | --require | --import | --loader | -X | -W | -cp | -classpath) skip=1 ;;
-*) ;;
*) name="${arg##*/}"; break ;;
esac
done ;;
esac
[ -n "$name" ] || name="${args[0]##*/}"
[ -n "$name" ] || name="$comm"
name="${name%.js}"
name="${name%.mjs}"
name="${name%.cjs}"
name="${name%.jar}"
name="${name//[^A-Za-z0-9._+-]/}"
printf '%s' "${name:0:40}"
}
# The terminal daemon's pid(s): Orca-runtime node processes running the daemon entry.
daemon_pids=' '
# Sets RANK to the oom_score_adj a process should have; returns 1 for a kernel thread or a gone pid.
RANK=500
rank() {
local pid="$1" args=() stat ppid
mapfile -d '' -t -n 3 args 2>/dev/null <"$PROC/$pid/cmdline" || return 1
[ "${#args[@]}" -gt 0 ] || return 1
RANK=-900
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) return 0 ;; esac
RANK=-500
read -r stat 2>/dev/null <"$PROC/$pid/stat" || return 1
stat="${stat##*) }"
ppid="${stat#* }"
ppid="${ppid%% *}"
case "$daemon_pids" in *" $ppid "*) return 0 ;; esac
case "${args[0]}/${args[1]:-}/${args[2]:-}" in "$ORCA_HOME"/tools/* | */"$ORCA_HOME"/tools/*) return 0 ;; esac
case "${args[0]##*/}" in claude | codex | opencode | gemini | cursor-agent | copilot | amp | droid) return 0 ;; esac
RANK=500
}
sweep() {
local p pid want adj args=()
daemon_pids=' '
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
mapfile -d '' -t -n 3 args 2>/dev/null <"$p/cmdline" || continue
[ "${#args[@]}" -ge 2 ] || continue
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) ;; *) continue ;; esac
case "${args[*]:1}" in *"$ORCA_HOME"/runtime/orcad/daemon*) daemon_pids+="$pid " ;; esac
done
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
# -1000 is a process that asked for it as root (sshd): leave it alone.
[ "$adj" -gt -1000 ] || continue
rank "$pid" || continue
want="$RANK"
# Never lower what a process raised for itself (Chrome ranks its renderers above its browser).
if [ "$want" -eq 500 ]; then
[ "$adj" -lt 500 ] || continue
fi
[ "$adj" -ne "$want" ] || continue
printf '%s\n' "$want" 2>/dev/null >"$p/oom_score_adj" || true
done
}
mem_total=0 mem_avail=0 swap_total=0 swap_free=0
read_mem() {
local key value _
while read -r key value _; do
case "$key" in
MemTotal:) mem_total="$value" ;;
MemAvailable:) mem_avail="$value" ;;
SwapTotal:) swap_total="$value" ;;
SwapFree:) swap_free="$value" ;;
esac
done <"$PROC/meminfo"
}
# PSI percentages (avg10, integer part); 0 when the kernel doesn't report them.
psi_some=0 psi_full=0
read_psi() {
local kind avg _
psi_some=0 psi_full=0
[ -r "$PROC/pressure/memory" ] || return 0
while read -r kind avg _; do
avg="${avg#avg10=}"
avg="${avg%%.*}"
case "$kind" in some) psi_some="$avg" ;; full) psi_full="$avg" ;; esac
done 2>/dev/null <"$PROC/pressure/memory" || true
}
kills=0 last_kill_name='' last_kill_at=0 swap_mode=on-demand level=ok
write_state() {
local pct="$1"
{
printf 'bootId=%s\nat=%s\npid=%s\nlevel=%s\navailPct=%s\nmemTotalKiB=%s\nmemAvailKiB=%s\n' \
"$BOOT" "$(now_ms)" "$$" "$level" "$pct" "$mem_total" "$mem_avail"
printf 'swapTotalKiB=%s\nswapFreeKiB=%s\nswap=%s\npsiSome10=%s\npsiFull10=%s\n' \
"$swap_total" "$swap_free" "$swap_mode" "$psi_some" "$psi_full"
printf 'kills=%s\nlastKillName=%s\nlastKillAt=%s\n' "$kills" "$last_kill_name" "$last_kill_at"
} >"$RUN/state.tmp" && mv -f "$RUN/state.tmp" "$RUN/state"
}
# Sets VICTIM/VICTIM_RSS (pages) to the largest process ranked 0 or more; returns 1 when there is none.
VICTIM=0 VICTIM_RSS=0
choose_victim() {
local p pid adj rss _ best=0 best_rss=0 args=()
sweep
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
[ "$adj" -ge 0 ] || continue
read -r _ rss _ 2>/dev/null <"$p/statm" || continue
# Kernel threads have no command line (and /proc sizes read 0, so `-s` can't tell).
mapfile -d '' -t -n 1 args 2>/dev/null <"$p/cmdline" || continue
[ "${#args[@]}" -gt 0 ] || continue
if [ "$rss" -gt "$best_rss" ]; then
best="$pid" best_rss="$rss"
fi
done
# Nothing big enough to matter (64 MiB): what fills memory is protected, so leave it to the kernel.
[ "$best" -gt 1 ] && [ "$best_rss" -ge 16384 ] || return 1
VICTIM="$best" VICTIM_RSS="$best_rss"
}
# Kills the largest process ranked 0 or more ($1: MemAvailable %). Returns 1 when there is none.
kill_largest() {
choose_victim || return 1
local best="$VICTIM" best_rss="$VICTIM_RSS" name at
name="$(proc_name "$best")"
kill -KILL "$best" 2>/dev/null || return 1
at="$(now_ms)"
kills=$((kills + 1)) last_kill_name="${name:-process}" last_kill_at="$at"
printf '%s %s %s %s %s\n' "$at" "$best" "$((best_rss * 4))" "$1" "$last_kill_name" >>"$RUN/kills"
tail -n 50 "$RUN/kills" >"$RUN/kills.tmp" && mv -f "$RUN/kills.tmp" "$RUN/kills"
log "killed pid=$best name=$last_kill_name rssKiB=$((best_rss * 4)) availPct=$1"
# Shells hear it like a kernel OOM kill (exit 137); this says why where someone may look.
logger -t orca-memguard "stopped $last_kill_name (pid $best, $((best_rss * 4)) KiB) at $1% memory available" 2>/dev/null || true
return 0
}
swap_files() { compgen -G "$SWAP_DIR/swap-*" || true; }
add_swap() {
[ "$swap_mode" = on-demand ] || return 1
[ $((swap_total + SWAP_CHUNK_KIB)) -le $((mem_total * 3 / 2)) ] || return 1
local free_kib file
free_kib="$(df -k --output=avail "$SWAP_DIR" 2>/dev/null | tail -n 1 | tr -d ' ')"
# Keep 4 GiB of the disk for the workspace itself.
[ "${free_kib:-0}" -gt $((SWAP_CHUNK_KIB + 4 * 1024 * 1024)) ] || return 1
file="$SWAP_DIR/swap-$(now_ms)"
if fallocate -l "$((SWAP_CHUNK_KIB))K" "$file" 2>/dev/null && chmod 600 "$file" \
&& mkswap -q "$file" >/dev/null 2>&1 && swapon "$file" 2>/dev/null; then
log "swap on $file"
return 0
fi
rm -f "$file"
# One refusal means this kernel or disk won't swap: don't retry every tick.
swap_mode=unsupported
log 'swap unavailable; guard kills only'
return 1
}
release_swap() {
local file
for file in $(swap_files | sort -r); do
swapoff "$file" 2>/dev/null && rm -f "$file" && log "swap off $file"
done
}
run() {
BOOT="$(boot_id)"
mkdir -p "$RUN" "$SWAP_DIR"
chmod 0755 "$RUN"
chmod 0700 "$SWAP_DIR"
# A resumed disk carries the last session's files: its kills and swap belong to that boot.
rm -f "$RUN/kills" "$RUN/state" "$RUN"/*.tmp
for file in $(swap_files); do
grep -q "^$file " /proc/swaps 2>/dev/null || rm -f "$file"
done
printf '%s %s\n' "$BOOT" "$$" >"$RUN/pid"
echo -1000 >/proc/self/oom_score_adj
read_mem
# Half of RAM for /dev/shm (tmpfs only takes pages that are written).
mount -o remount,size=50% /dev/shm 2>/dev/null && log "/dev/shm $(df -h --output=size /dev/shm | tail -n 1 | tr -d ' ')"
log "start mem=${mem_total}KiB"
local tick=0 pct quiet_since=0 prev='' now
while :; do
read_mem
read_psi
pct=$((mem_total > 0 ? mem_avail * 100 / mem_total : 100))
if [ $((tick % SWEEP_EVERY)) -eq 0 ]; then
sweep
fi
if [ "$pct" -lt "$SWAP_ADD_PCT" ] && { [ "$swap_total" -eq 0 ] || [ $((swap_free * 4)) -lt "$swap_total" ]; }; then
add_swap && read_mem
fi
# Swap only raises MemAvailable by paging idle memory out: a fast allocator outruns it (the
# kernel OOM-killed a 300 MB/s hog with 1.7 GB of swap free), so the threshold ignores swap.
if [ "$pct" -lt "$KILL_PCT" ]; then
level=critical
if kill_largest "$pct"; then
write_state "$pct"
prev="$level:$kills"
sleep 1 # let the kernel reclaim the victim's memory before judging again
tick=$((tick + 1))
continue
fi
elif [ "$psi_full" -ge 10 ]; then
level=critical
elif [ "$pct" -lt "$HIGH_PCT" ] || [ "$psi_some" -ge 20 ]; then
level=high
else
level=ok
fi
now="${EPOCHSECONDS}"
if [ "$pct" -ge "$QUIET_PCT" ] && [ "$swap_total" -gt 0 ]; then
[ "$quiet_since" -gt 0 ] || quiet_since="$now"
if [ $((now - quiet_since)) -ge "$QUIET_SECONDS" ] && [ $(((swap_total - swap_free) * 2)) -lt "$mem_avail" ]; then
release_swap
quiet_since=0
fi
else
quiet_since=0
fi
if [ "$level:$kills" != "$prev" ] || [ $((tick % STATE_EVERY)) -eq 0 ]; then
write_state "$pct"
prev="$level:$kills"
fi
tick=$((tick + 1))
# Memory can go from 15% to nothing in two seconds: look five times as often when it is low.
if [ "$level" = ok ]; then sleep "$TICK"; else sleep "$FAST_TICK"; fi
done
}
running() {
local boot pid
read -r boot pid <"$RUN/pid" 2>/dev/null || return 1
[ "$boot" = "$(boot_id)" ] && [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null \
&& grep -qa orca-memguard "/proc/$pid/cmdline" 2>/dev/null
}
case "${1:-}" in
start)
[ "$(id -u)" -eq 0 ] || { echo 'orca-memguard: needs root' >&2; exit 1; }
running && exit 0
mkdir -p "$RUN"
setsid nohup "$0" run </dev/null >/dev/null 2>>"$RUN/memguard.log" &
exit 0 ;;
run) run ;;
sweep) sweep ;;
victim) choose_victim && printf '%s %s %s\n' "$VICTIM" "$((VICTIM_RSS * 4))" "$(proc_name "$VICTIM")" ;;
release-swap) release_swap ;;
*) echo 'usage: orca-memguard start|run|sweep|victim|release-swap' >&2; exit 2 ;;
esac
@@ -12,6 +12,8 @@ kill -TERM "$pid"
for _ in $(seq 1 $((WAIT_SECONDS * 10))); do
if ! kill -0 "$pid" 2>/dev/null; then
rm -f "$STATE/orcad.pid" "$STATE/ready.json"
# On-demand swap files would ride the stop snapshot (2 GiB each). Best effort, bounded.
[ -x /opt/orca/bin/orca-memguard ] && sudo -n timeout 20 /opt/orca/bin/orca-memguard release-swap >/dev/null 2>&1 || true
exit 0
fi
sleep 0.1
@@ -36,9 +36,15 @@ anthropic_key_scrub=''
gateway_key=''
gemini_key=''
codex_placeholder=''
sign_in_inside=''
while IFS= read -r line; do
name="${line%%=*}" b64="${line#*=}"
# Not for shells: the Codex account placeholder becomes ~/.codex/auth.json below.
# Not for shells: the agents the person signs in inside this workspace (e.g. `claude,codex`).
if [ "$name" = ORCA_SIGN_IN_INSIDE ]; then
sign_in_inside="$(printf '%s' "$b64" | base64 -d)"
case "$sign_in_inside" in *[!a-z,]*) sign_in_inside='' ;; esac
fi
if [ "$name" = ORCA_CODEX_AUTH_PLACEHOLDER ]; then
codex_placeholder="$(printf '%s' "$b64" | base64 -d)"
case "$codex_placeholder" in *[!A-Za-z0-9._-]*) codex_placeholder='' ;; esac
@@ -164,7 +170,9 @@ fi
# through them, and the agent would sit there doing nothing. Merges; never touches credentials.
node_bin="$(command -v node || find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"
if [ -n "$node_bin" ]; then
ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
claude_inside=''
case ",$sign_in_inside," in *,claude,*) claude_inside=1 ;; esac
ORCA_CLAUDE_INSIDE="$claude_inside" ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
ORCA_GATEWAY_MODEL="${gateway_key:+${anthropic_model:-$openai_model}}" "$node_bin" -e '
const fs = require("fs"), path = require("path"), os = require("os")
const home = os.homedir()
@@ -178,7 +186,11 @@ const write = (file, value) => {
const claudeFile = path.join(process.env.CLAUDE_CONFIG_DIR || home, ".claude.json")
const c = readJson(claudeFile)
const before = JSON.stringify(c)
c.hasCompletedOnboarding = true
// Signed in inside this workspace and no login here yet: leave first-run onboarding to Claude, so it opens
// on its own sign-in screens instead of "Not logged in". Once a login exists, onboarding is done as usual.
const claudeCredentials = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(home, ".claude"), ".credentials.json")
if (process.env.ORCA_CLAUDE_INSIDE === "1" && !fs.existsSync(claudeCredentials)) delete c.hasCompletedOnboarding
else c.hasCompletedOnboarding = true
c.theme = c.theme || "dark"
c.bypassPermissionsModeAccepted = true
c.projects = c.projects && typeof c.projects === "object" ? c.projects : {}
@@ -110,6 +110,10 @@ case "${ORCA_DOCKER:-}" in
esac
start_share_proxy
unset ORCA_SHARE_PROXY_KEY
# MEMGUARD: OOM-killer ranks that spare orcad and terminals, a guard that stops the largest build or
# test before the VM stalls, on-demand swap and a /dev/shm sized for browsers. Before the
# already-started exit so a repair start brings a dead guard back; it returns at once.
sudo -n "$ORCA_HOME/bin/orca-memguard" start </dev/null >/dev/null 2>&1 9>&- || true
if [ "$(cat "$STATE/boot-id" 2>/dev/null || true)" = "$boot_id" ] && [ -s "$STATE/ready.json" ] \
&& kill -0 "$(cat "$STATE/orcad.pid" 2>/dev/null || echo 0)" 2>/dev/null; then
print_ready
@@ -213,12 +213,12 @@ export async function deleteAgentSourcesForSandbox(db: D1Database, sandboxId: st
/**
* How each agent authenticates in this session (D10):
* - 'account': the owner's Orca-account credential through an egress placeholder (keys for that agent dropped);
* - 'omit': the agent's own login in the sandbox, keys dropped: the owner signs in inside each workspace
* ('in-workspace'), or the kill-switch fallback (Source sign-in, provider off);
* - 'inside': the owner signs in inside each workspace ('in-workspace'): keys dropped, the agent's own login;
* - 'omit': kill-switch fallback (Source sign-in, provider off): keys dropped, the agent's own login in the sandbox;
* - 'keep': API keys as resolved (precedence), the default.
* Source: explicit (workspace, then default), else implicit: a covering key wins, else a usable account.
*/
export type AgentMode = 'account' | 'omit' | 'keep'
export type AgentMode = 'account' | 'inside' | 'omit' | 'keep'
export async function agentModesFor(
db: D1Database,
row: Pick<SandboxRow, 'id' | 'user_id' | 'visibility' | 'ownership_transferred_at'>,
@@ -240,7 +240,7 @@ export async function agentModesFor(
if (!workspace[agent] && inWorkspace[agent]) {
// Nothing of Orca's for this agent: no placeholder, no key, no egress route, so the login the person
// makes inside the sandbox (stored in its home, kept across sleep) reaches the vendor untouched.
modes[agent] = 'omit'
modes[agent] = 'inside'
continue
}
const explicit = workspace[agent] ?? defaults[agent]
+36 -5
View File
@@ -891,6 +891,34 @@ function fail(error: string, status: number, reason?: string): Response {
return json(reason ? { error, reason } : { error }, status)
}
/** CREATE-STEP-WATCH: logs a step still pending after 10 s and, once the create starts, every step's time. */
export function createStepWatch(enabled: boolean, now: () => number = Date.now) {
const started = now()
const steps: Record<string, number> = {}
let reported = false
return {
async step<T>(name: string, work: Promise<T>): Promise<T> {
if (!enabled) {
return work
}
const at = now()
const slow = setTimeout(() => console.log(JSON.stringify({ event: 'create_step_slow', step: name, ms: now() - at })), 10_000)
try {
return await work
} finally {
clearTimeout(slow)
steps[name] = now() - at
}
},
done() {
if (enabled && !reported) {
reported = true
console.log(JSON.stringify({ event: 'create_steps', ms: now() - started, steps }))
}
}
}
}
async function readJson(request: Request, maxBytes = MAX_JSON_BYTES): Promise<unknown> {
const text = await request.text()
if (text.length > maxBytes) {
@@ -1809,11 +1837,13 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (!match) {
return fail('not_found', 404)
}
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73). Name the step a create waits in.
const watch = createStepWatch(request.method === 'POST' && !match[1])
const caller = await watch.step('auth', authenticateDesktop(env, request, (deps.now ?? Date.now)()))
if (!caller) {
return fail('unauthorized', 401)
}
const life = await lifecycleDeps(env, deps)
const life = await watch.step('deps', lifecycleDeps(env, deps))
if (!life) {
return fail('not_configured', 503)
}
@@ -1836,12 +1866,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (request.method !== 'POST') {
return fail('not_found', 404)
}
if (await tokenMintedBeforeMemberChange(life.db, caller)) {
if (await watch.step('stale_token', tokenMintedBeforeMemberChange(life.db, caller))) {
// ORG-TOKEN-STALE: never create (and bill) with a token minted before a role change.
return fail('unauthorized', 401, 'stale_token')
}
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
const raw = await readJson(request)
await watch.step('create_token', consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now()))
const raw = await watch.step('body', readJson(request))
const body = CreateRequestSchema.safeParse(raw)
if (!body.success) {
return fail('invalid_request', 400)
@@ -1859,6 +1889,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
life.db,
{ accountId: caller.accountId, scope: `sandbox.create:${caller.userId}`, key: readIdempotencyKey(request), body: raw, now: life.now() },
async () => {
watch.done()
// Desktops that show the clone as its own step ask for it in the background (GET .../clone).
// Member identities install after the response; the VM's git holds commits and pushes until then.
let ready: Ready
@@ -6,6 +6,8 @@ import { z } from 'zod'
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
const PROJECT = '/vercel/sandbox/project'
/** Written by the VM's memory guard (image/vm/orca-memguard). */
const GUARD_STATE = '/run/orca-memguard/state'
/**
* CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project
@@ -28,9 +30,22 @@ victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64
[ "$victim" = MainThread ] && victim=node
orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); [ "$pid" -gt 1 ] 2>/dev/null && kill -0 "$pid" 2>/dev/null && orcad=true
load=$(awk '{print $1}' /proc/loadavg)
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \
# MEMGUARD: the memory guard's state (orca-memguard), when it runs this boot and wrote in the last minute.
guard=null
if [ -r ${GUARD_STATE} ]; then
g() { sed -n "s/^$1=//p" ${GUARD_STATE} | head -n 1; }
g_at=$(g at); g_level=$(g level); g_kills=$(g kills); g_name=$(g lastKillName | tr -cd 'A-Za-z0-9._+-' | head -c 40); g_kill_at=$(g lastKillAt)
g_swap=$(g swapTotalKiB); [[ "$g_swap" =~ ^[0-9]+$ ]] && g_swap=$((g_swap * 1024)) || g_swap=0
now_ms=$(( $(date +%s%3N) ))
if [ "$(g bootId)" = "$(cat /proc/sys/kernel/random/boot_id)" ] && [[ "$g_at" =~ ^[0-9]+$ ]] && [ $((now_ms - g_at)) -lt 60000 ] \
&& [[ "$g_level" =~ ^(ok|high|critical)$ ]] && [[ "$g_kills" =~ ^[0-9]+$ ]] && [[ "$g_kill_at" =~ ^[0-9]+$ ]]; then
guard=$(printf '{"level":"%s","kills":%d,"lastKillName":%s,"lastKillAt":%s,"swapBytes":%d}' "$g_level" "$g_kills" \
"$( [ -n "$g_name" ] && printf '"%s"' "$g_name" || printf null )" "$( [ "$g_kill_at" -gt 0 ] && printf '%s' "$g_kill_at" || printf null )" "$g_swap")
fi
fi
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s,"guard":%s}\n' \
"$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad"
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad" "$guard"
`.replaceAll('\\${', '${')
export const VmMetricsSchema = z
@@ -51,6 +66,41 @@ export const VmMetricsSchema = z
export type VmMetrics = z.infer<typeof VmMetricsSchema>
/** The memory guard's last word: its pressure level, the processes it stopped this boot, its swap. */
export const VmGuardSchema = z
.object({
level: z.enum(['ok', 'high', 'critical']),
kills: z.number().int().min(0),
lastKillName: z.string().max(40).nullable(),
lastKillAt: z.number().int().positive().nullable(),
swapBytes: z.number().int().min(0)
})
.strict()
export type VmGuard = z.infer<typeof VmGuardSchema>
// The probe's line: the metrics plus the guard (null when no guard ran this boot: an older image).
const VmProbeSchema = VmMetricsSchema.extend({ guard: VmGuardSchema.nullable().optional() }).strict()
/** Memory pressure for the status read: the guard's level, else one from MemAvailable alone. */
export type WorkspaceMemory = {
pressure: 'ok' | 'high' | 'critical'
/** Processes the guard stopped this boot (the kernel's own OOM kills are `metrics.oomKills`). */
guardKills: number
lastGuardVictim: string | null
lastGuardKillAt: number | null
/** False when the VM runs no guard: pressure is then MemAvailable's, and nothing is stopped early. */
guarded: boolean
}
export function workspaceMemory(metrics: VmMetrics, guard: VmGuard | null): WorkspaceMemory {
if (guard) {
return { pressure: guard.level, guardKills: guard.kills, lastGuardVictim: guard.lastKillName, lastGuardKillAt: guard.lastKillAt, guarded: true }
}
const availPct = metrics.memTotalBytes > 0 ? ((metrics.memTotalBytes - metrics.memUsedBytes) * 100) / metrics.memTotalBytes : 100
const pressure = availPct < 8 ? 'critical' : availPct < 15 ? 'high' : 'ok'
return { pressure, guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false }
}
const MAX_TREE = 2000
const MAX_CHANGED = 300
const MAX_COMMITS = 20
@@ -164,8 +214,17 @@ async function runJson<T>(
}
/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */
export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise<VmMetrics | null> {
return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000)
export async function readVmMetrics(
provider: SandboxProviderApi,
sessionId: string
): Promise<{ metrics: VmMetrics; memory: WorkspaceMemory } | null> {
const probe = await runJson(provider, sessionId, METRICS_SCRIPT, VmProbeSchema, 8_000)
if (!probe) {
return null
}
// `metrics` keeps its exact shape: desktops parse it strictly.
const { guard, ...metrics } = probe
return { metrics, memory: workspaceMemory(metrics, guard ?? null) }
}
export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise<ArchiveMetadata | null> {
+10 -3
View File
@@ -14,6 +14,9 @@ import { SecretsKmsError } from './secrets-kms'
import type { SandboxRow } from './store'
import { secretLog } from './secret-log'
/** Session env name the VM env script reads for the agents signed in inside the workspace ('claude,codex'). */
export const SIGN_IN_INSIDE_ENV = 'ORCA_SIGN_IN_INSIDE'
export type SessionSecrets = {
env: Record<string, string>
/** Row value for `egress_secret_hosts`. */
@@ -134,6 +137,10 @@ export async function sessionSecretEnv(
const env = withoutSignInCredentials(derived, dropped)
const scopes = deriveProviderScopes(stored, env, storedScopes)
const egressAllowHosts = providerHosts(env).join(',') || null
// Not a secret and never in shells (ORCA_*): tells the VM env script which agents sign in there, so a
// first run with no login of its own shows the agent's sign-in screens instead of skipping them.
const inside = (Object.keys(modes) as CredentialAgent[]).filter((agent) => modes[agent] === 'inside')
const insideEnv: Record<string, string> = inside.length > 0 ? { [SIGN_IN_INSIDE_ENV]: inside.join(',') } : {}
// Counts only.
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
if (withhold) {
@@ -153,7 +160,7 @@ export async function sessionSecretEnv(
}
}
const hosts = [encodeEgressSecretHosts(withheld.targets), ...accountRoutes].filter(Boolean).join(',') || null
return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts }
return { env: { ...withheld.env, ...accountEnv, ...insideEnv }, egressSecretHosts: hosts, egressAllowHosts }
}
if (!swap) {
// An operator turned placeholders off for this account (SECRETS_EGRESS_PLACEHOLDERS=off or accounts:<ids>
@@ -162,7 +169,7 @@ export async function sessionSecretEnv(
if (plain > 0 && !egressSwapWanted(deps, row) && firstReadThisSession('plain', row)) {
secretLog('egress_swap_disabled', { sandboxId: row.id, plainKeys: plain })
}
return { env, egressSecretHosts: null, egressAllowHosts }
return { env: { ...env, ...insideEnv }, egressSecretHosts: null, egressAllowHosts }
}
const swapped = applySecretPlaceholders(row, env, scopes)
const accountEnv: Record<string, string> = {}
@@ -175,7 +182,7 @@ export async function sessionSecretEnv(
}
}
const secretHosts = [encodeEgressSecretHosts(swapped.targets), ...accountRoutes].filter(Boolean).join(',') || null
return { env: { ...swapped.env, ...accountEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
return { env: { ...swapped.env, ...accountEnv, ...insideEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
}
/** Provider hosts from the session's secrets plus the workspace's declared tool hosts (MCP). */
@@ -22,6 +22,7 @@ import {
ArchiveMetadataSchema,
captureArchiveMetadata,
readVmMetrics,
type WorkspaceMemory,
type ArchiveMetadata,
type VmMetrics
} from './sandbox-vm-inspect'
@@ -359,6 +360,8 @@ export async function expiredArchives(db: D1Database, now: number): Promise<Sand
export type WorkspaceStatus = {
state: WorkspaceState
metrics: VmMetrics | null
/** MEMGUARD: memory pressure and the guard's kills; null when the VM was not probed. */
memory: WorkspaceMemory | null
/** Since orcad last became ready (this session). */
uptimeMs: number | null
usage: { totalMs: number; centsPerHour: number; costCents: number }
@@ -393,13 +396,15 @@ export async function workspaceStatus(deps: LifecycleDeps, input: SandboxRow, co
// A lapsed lease: the provider may have stopped it, which is "sleeping", not "unresponsive".
const row = input.status === 'running' ? await confirmRunning(deps, input) : input
const probed = row.status === 'running' && row.session_id !== null
const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
const probe = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
const metrics = probe?.metrics ?? null
const now = deps.now()
const totalMs = await sandboxUsageMs(deps.db, row, now)
const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size]
return {
state: stateOf(row, metrics, probed),
metrics,
memory: probe?.memory ?? null,
uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null,
usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) },
archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null,
@@ -76,6 +76,8 @@ describe('Source "in-workspace" (sign in inside this workspace)', () => {
expect(env[name]).toBeUndefined()
}
expect(Object.values(env).some((value) => value.startsWith('orca-ph-'))).toBe(false)
// The VM env script's cue (never a shell variable): these agents sign in inside the workspace.
expect(env.ORCA_SIGN_IN_INSIDE).toBe('claude,codex')
// Open egress: "*" plus only hosts that carry rules. No vendor host carries one (no claim, no 403).
const allow = policy()
expect(allow['*']).toEqual([])
@@ -116,8 +118,10 @@ describe('Source "in-workspace" (sign in inside this workspace)', () => {
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined()
expect([...h.vercel.boxes.values()][0]!.policy.allow['api.anthropic.com']).toBeUndefined()
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBe('claude')
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: null } })
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBeUndefined()
})
it('is a per-person default only: a workspace override is refused, and an explicit workspace Source still wins', async () => {
@@ -0,0 +1,38 @@
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73); the steps before the create log.
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createStepWatch } from '../src/index'
describe('CREATE-STEP-WATCH', () => {
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
it('names a step still pending after 10 s, then reports every step once the create starts', async () => {
vi.useFakeTimers()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const watch = createStepWatch(true, () => Date.now())
let finish: (value: string) => void = () => undefined
const body = watch.step('body', new Promise<string>((resolve) => (finish = resolve)))
await vi.advanceTimersByTimeAsync(10_000)
expect(log).toHaveBeenCalledWith(expect.stringContaining('"step":"body"'))
finish('{}')
await expect(body).resolves.toBe('{}')
watch.done()
watch.done()
const reports = log.mock.calls.filter(([line]) => String(line).includes('"create_steps"'))
expect(reports).toHaveLength(1)
expect(String(reports[0]![0])).toContain('"body":10000')
})
it('stays quiet for other routes', async () => {
vi.useFakeTimers()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const watch = createStepWatch(false)
const pending = watch.step('auth', new Promise(() => undefined))
await vi.advanceTimersByTimeAsync(20_000)
watch.done()
expect(log).not.toHaveBeenCalled()
void pending
})
})
@@ -0,0 +1,85 @@
// MEMGUARD: the VM's memory guard (image/vm/orca-memguard) ranks processes for the kernel's OOM killer
// so Orca's runtime and terminals outlive a build that eats the VM's memory, and names the process it
// would stop. Run against a fake process table (ORCA_MEMGUARD_PROC); the kill itself is verified in a VM.
import { spawnSync } from 'node:child_process'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
const GUARD = fileURLToPath(new URL('../image/vm/orca-memguard', import.meta.url).href)
const START = fileURLToPath(new URL('../image/vm/orca-sandbox-start', import.meta.url).href)
// mapfile -d needs bash 4.4+ (the VM has 5.x; macOS's /bin/bash is 3.2).
const bash = ['/opt/homebrew/bin/bash', '/usr/local/bin/bash', '/usr/bin/bash', '/bin/bash'].find(
(path) => spawnSync(path, ['-c', '[ "${BASH_VERSINFO[0]}${BASH_VERSINFO[1]}" -ge 44 ]']).status === 0
)
const NODE = '/opt/orca/runtime/runtimes/node-abc/bin/node'
let proc = ''
function fake(pid: number, opts: { ppid: number; adj: number; rssPages: number; argv: string[]; comm?: string }): void {
const dir = join(proc, String(pid))
mkdirSync(dir, { recursive: true })
writeFileSync(join(dir, 'cmdline'), opts.argv.length ? `${opts.argv.join('\0')}\0` : '')
writeFileSync(join(dir, 'comm'), `${opts.comm ?? 'x'}\n`)
writeFileSync(join(dir, 'stat'), `${pid} (${opts.comm ?? 'x'}) S ${opts.ppid} ${pid} ${pid} 0 -1\n`)
writeFileSync(join(dir, 'statm'), `${opts.rssPages * 2} ${opts.rssPages} 100 1 0 1 0\n`)
writeFileSync(join(dir, 'oom_score_adj'), `${opts.adj}\n`)
}
const adj = (pid: number): number => Number(readFileSync(join(proc, String(pid), 'oom_score_adj'), 'utf8').trim())
const guard = (command: string) => spawnSync(bash!, [GUARD, command], { env: { PATH: process.env.PATH, ORCA_MEMGUARD_PROC: proc }, encoding: 'utf8' })
beforeEach(() => {
proc = mkdtempSync(join(tmpdir(), 'orca-memguard-proc-'))
fake(1, { ppid: 0, adj: 500, rssPages: 3000, argv: ['/run/vercel/share/sandbox-init'] })
fake(100, { ppid: 1, adj: 500, rssPages: 2_000_000, argv: [NODE, '/opt/orca/bin/orcad-boot', '--code-cache'], comm: 'MainThread' })
fake(101, { ppid: 100, adj: 500, rssPages: 20_000, argv: [NODE, '--max-semi-space-size=4', '/opt/orca/runtime/orcad/daemon-entry.js'] })
// A terminal's shell, and a typecheck it started (inherited the shell's rank).
fake(102, { ppid: 101, adj: 500, rssPages: 1_000, argv: ['/bin/bash', '-l'], comm: 'bash' })
fake(103, { ppid: 102, adj: -500, rssPages: 900_000, argv: ['node', '--max-old-space-size=8192', '/w/node_modules/typescript/bin/tsc', '-p', '.'], comm: 'MainThread' })
fake(104, { ppid: 102, adj: 300, rssPages: 40_000, argv: ['/opt/google/chrome/chrome', '--type=renderer'], comm: 'chrome' })
fake(105, { ppid: 102, adj: 800, rssPages: 40_000, argv: ['/usr/bin/some-tool'] })
fake(106, { ppid: 1, adj: -1000, rssPages: 600, argv: ['sshd: /opt/orca/openssh/sshd'] })
fake(107, { ppid: 102, adj: 500, rssPages: 400_000, argv: ['/opt/orca/tools/bin/claude'], comm: 'claude' })
fake(108, { ppid: 2, adj: 0, rssPages: 0, argv: [] }) // a kernel thread
})
afterEach(() => rmSync(proc, { recursive: true, force: true }))
describe.skipIf(!bash)('orca-memguard', () => {
it('protects the runtime, terminals and agents, and makes everything else the first victim', () => {
expect(guard('sweep').status).toBe(0)
expect([100, 101].map(adj)).toEqual([-900, -900])
expect(adj(102)).toBe(-500) // the terminal's shell
expect(adj(107)).toBe(-500) // an agent CLI
expect(adj(103)).toBe(500) // the shell's typecheck no longer inherits its rank
expect(adj(104)).toBe(500)
expect(adj(105)).toBe(800) // a process that raised its own rank keeps it
expect(adj(106)).toBe(-1000) // what root pinned stays pinned
expect(adj(108)).toBe(0) // kernel threads are not touched
expect(adj(1)).toBe(500)
})
it('would stop the largest unprotected process and names it by its script', () => {
const result = guard('victim')
expect(result.status).toBe(0)
expect(result.stdout.trim()).toBe(`103 ${900_000 * 4} tsc`)
})
it('names inline code by its interpreter, and leaves small processes to the kernel', () => {
fake(103, { ppid: 102, adj: 500, rssPages: 900_000, argv: ['node', '-e', 'const secret = 1'], comm: 'MainThread' })
expect(guard('victim').stdout.trim()).toBe(`103 ${900_000 * 4} node`)
rmSync(join(proc, '103'), { recursive: true })
rmSync(join(proc, '104'), { recursive: true })
rmSync(join(proc, '105'), { recursive: true })
// Only protected memory is large: nothing worth stopping.
expect(guard('victim').status).toBe(1)
})
it('starts from orca-sandbox-start before its already-running exit', () => {
const start = readFileSync(START, 'utf8')
const launch = start.indexOf('orca-memguard" start')
expect(launch).toBeGreaterThan(0)
expect(launch).toBeLessThan(start.indexOf('print_ready\n exit 0'))
})
})
@@ -135,3 +135,37 @@ describe('orca-sandbox-env: Claude Code API-key approval (G9)', () => {
expect(claude(file).hasCompletedOnboarding).toBe(true)
})
})
describe('orca-sandbox-env: Claude signed in inside the workspace', () => {
const claudeJson = (file: (path: string) => string | null) => JSON.parse(file('.claude.json') ?? '{}')
it('with no login yet, leaves onboarding to Claude so it opens on its own sign-in screens; never a shell variable', () => {
const { run, file, sessionEnv } = scratch()
run({ ORCA_SIGN_IN_INSIDE: 'claude,codex', FOO: 'bar' })
const c = claudeJson(file)
expect(c.hasCompletedOnboarding).toBeUndefined()
// The other first-run answers still apply.
expect(c.bypassPermissionsModeAccepted).toBe(true)
expect(sessionEnv()).not.toContain('ORCA_SIGN_IN_INSIDE')
})
it('clears a mark an earlier session made, and marks onboarding done once a login exists here', () => {
const { home, run, file } = scratch()
run({})
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
expect(claudeJson(file).hasCompletedOnboarding).toBeUndefined()
mkdirSync(join(home, '.claude'), { recursive: true })
writeFileSync(join(home, '.claude', '.credentials.json'), '{}')
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
})
it('only Codex inside, or a malformed value, keeps Claude onboarding done', () => {
const { run, file } = scratch()
run({ ORCA_SIGN_IN_INSIDE: 'codex' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
run({ ORCA_SIGN_IN_INSIDE: 'claude;rm -rf /' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
})
})
@@ -154,6 +154,28 @@ describe('status panel', () => {
expect(status.metrics).toMatchObject({ oomKills: 1, lastOomVictim: 'node' })
})
it('reports memory pressure and guard kills beside the metrics, which keep their strict shape', async () => {
const { h, id } = await running()
const guard = { level: 'high', kills: 2, lastKillName: 'tsc', lastKillAt: 1_800_000_000_000, swapBytes: 2 ** 31 }
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ guard })}\n`, stderr: '' } : undefined)
const body = (await h.call('GET', `/v1/sandboxes/${id}/status`)).body
expect(body.status.metrics).not.toHaveProperty('guard')
const { status } = HostedWorkspaceStatusResponseSchema.parse(body)
expect(status.state).toBe('ready')
expect(status.memory).toEqual({ pressure: 'high', guardKills: 2, lastGuardVictim: 'tsc', lastGuardKillAt: 1_800_000_000_000, guarded: true })
})
it('derives pressure from MemAvailable when the VM runs no guard (an older image)', async () => {
const { h, id } = await running()
h.vercel.onScript = (script) =>
script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ memUsedBytes: 7.5e9, guard: null })}\n`, stderr: '' } : undefined
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(status.memory).toEqual({ pressure: 'critical', guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false })
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine()}\n`, stderr: '' } : undefined)
const again = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(again.status.memory?.pressure).toBe('ok')
})
it('counts settled sessions in the cost and does not probe a sleeping VM', async () => {
const { h, id } = await running()
h.advance(60 * 60_000)
@@ -57,6 +57,30 @@ describe('OOM tracker', () => {
expect(events).toHaveLength(3)
})
it('announces guard kills as the guard stopping a named process, counted with kernel kills', () => {
const events: CloudOomEvent[] = []
const tracker = createOomTracker((event) => events.push(event))
const guarded = (checkedAt: number, uptime: number, kernel: number, guardKills: number) => ({
...status(checkedAt, uptime, kernel),
memory: {
pressure: 'high' as const,
guardKills,
lastGuardVictim: guardKills ? 'tsc' : null,
lastGuardKillAt: guardKills ? checkedAt : null,
guarded: true
}
})
const t0 = 1_000_000_000
tracker.observe('ws', 'sbx', guarded(t0, 100, 0, 0))
tracker.observe('ws', 'sbx', guarded(t0 + 60_000, 160, 0, 1))
tracker.observe('ws', 'sbx', guarded(t0 + 120_000, 220, 0, 1))
tracker.observe('ws', 'sbx', guarded(t0 + 180_000, 280, 1, 1))
expect(events).toEqual([
{ workspaceId: 'ws', oomKills: 1, victim: 'tsc', stoppedBy: 'guard', at: expect.any(Number) },
{ workspaceId: 'ws', oomKills: 2, victim: 'node', at: expect.any(Number) }
])
})
it('ignores a status without metrics', () => {
const events: CloudOomEvent[] = []
createOomTracker((event) => events.push(event)).observe('ws', 'sbx', {
@@ -115,12 +115,13 @@ export function trimCheckouts(input: HostedArchiveCheckout[]): HostedArchiveChec
}
/**
* Announces each new kernel OOM kill once. The counter resets per VM boot; two readings belong to
* one boot when their implied boot times agree within two minutes (uptime and checkedAt come from
* Announces each new OOM kill once: the kernel's, and the memory guard's (MEMGUARD: processes the
* VM stopped before the kernel had to). Both counters reset per VM boot; two readings belong to one
* boot when their implied boot times agree within two minutes (uptime and checkedAt come from
* different clocks).
*/
export function createOomTracker(announce: (event: CloudOomEvent) => void) {
const seen = new Map<string, { bootAt: number; kills: number }>()
const seen = new Map<string, { bootAt: number; kernel: number; guard: number }>()
return {
observe(
workspaceId: string,
@@ -132,19 +133,26 @@ export function createOomTracker(announce: (event: CloudOomEvent) => void) {
if (!metrics) {
return
}
const guardKills = status.memory?.guardKills ?? 0
const bootAt = status.checkedAt - metrics.uptimeSeconds * 1000
const previous = seen.get(sandboxId)
const sameBoot = previous !== undefined && Math.abs(previous.bootAt - bootAt) < 120_000
const announced = sameBoot ? previous.kills : 0
const kernel = sameBoot ? previous.kernel : 0
const guard = sameBoot ? previous.guard : 0
seen.set(sandboxId, {
bootAt: sameBoot ? previous.bootAt : bootAt,
kills: Math.max(announced, metrics.oomKills)
kernel: Math.max(kernel, metrics.oomKills),
guard: Math.max(guard, guardKills)
})
if (metrics.oomKills > announced) {
const byGuard = guardKills > guard
if (byGuard || metrics.oomKills > kernel) {
announce({
workspaceId,
oomKills: metrics.oomKills,
victim: metrics.lastOomVictim,
// One count for dismissal: either kind of new kill shows the notice again.
oomKills: Math.max(kernel, metrics.oomKills) + Math.max(guard, guardKills),
// The guard names the process it chose; the kernel's name comes from its log.
victim: byGuard ? (status.memory?.lastGuardVictim ?? null) : metrics.lastOomVictim,
...(byGuard ? { stoppedBy: 'guard' as const } : {}),
at: now
})
}
@@ -123,7 +123,8 @@ describe('cloudAgentSetupStripModel (PC-3b)', () => {
expect(idle.primary?.action).toEqual({ kind: 'setUpOnce' })
expect(idle.alternative).toEqual({
label: 'Sign in inside this workspace instead',
detail: 'Orca won’t store your sign-in. You’ll sign in again in each new workspace.',
detail:
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.',
action: { kind: 'signInInsideWorkspace' }
})
const failed = cloudAgentSetupStripModel({
@@ -38,7 +38,7 @@ export function signInInsideWorkspaceAlternative(
label: t('agentSignInStrip.signInInsideWorkspace', 'Sign in inside this workspace instead'),
detail: t(
'agentSignInStrip.signInInsideWorkspaceDetail',
'Orca won’t store your sign-in. You’ll sign in again in each new workspace.'
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.'
),
action: { kind: 'signInInsideWorkspace' }
}
@@ -64,3 +64,38 @@ describe('cloudChatSignInContent', () => {
expect(cloudChatSignInContent(true, 'notSignedIn', 'Antigravity')).toBeUndefined()
})
})
describe('the not-signed-in card when the agent signs in inside the workspace', () => {
it('says so and offers a terminal running the agent’s login', async () => {
cloud.on = true
const { setOrcaCloudAgentAccountsStateForTest } =
await import('@/lib/orca-cloud/orca-cloud-agent-accounts-store')
setOrcaCloudAgentAccountsStateForTest({
load: {
kind: 'ready',
status: {
credentials: {
claude: { state: 'none' },
codex: { state: 'none' },
providerEnabled: { claude: true, codex: true }
},
sources: {
defaults: {},
coveredByKey: { claude: false, codex: false },
inWorkspace: { claude: true, codex: false }
}
}
},
setups: { claude: { phase: 'idle' }, codex: { phase: 'idle' } },
keyKinds: { claude: null, codex: null },
queue: { agents: [], index: 0 }
})
card('Claude')
expect(
screen.getByText('Claude needs you to sign in inside this workspace')
).toBeInTheDocument()
expect(screen.queryByText(/Renew it above/)).toBeNull()
screen.getByRole('button', { name: 'Open terminal to sign in' }).click()
expect(signIn.signIn).toHaveBeenCalled()
})
})
@@ -9,6 +9,9 @@ import type { RuntimeClientTarget } from '../../../../shared/runtime-client-targ
import { isOrcaCloudEnvironmentId } from '@/lib/orca-cloud/orca-cloud-environment-id'
import { agentSessionSetupFor } from '../../../../shared/agent-session/lifecycle/agent-session-setup-commands'
import { HOST_AGENT_SIGN_IN_AGENT_NAME } from '../../../../shared/host-agent-sign-in-status'
import { useTranslate } from '@/i18n/use-translate'
import { orcaCloudAgentCoverageFromAccounts } from '@/lib/orca-cloud/orca-cloud-agent-accounts-model'
import { useOrcaCloudAgentAccounts } from '@/lib/orca-cloud/orca-cloud-agent-accounts-store'
const CloudChatHostContext = createContext(false)
@@ -28,15 +31,25 @@ export function useCloudChatHost(): boolean {
return useContext(CloudChatHostContext)
}
/** The sign-in card's words on an Orca Cloud chat, or undefined to keep the card as it is. */
/**
* The sign-in card's words on an Orca Cloud chat, or undefined to keep the card as it is. An agent
* signed in inside the workspace keeps the generic card (its Sign in opens a terminal on the host);
* the caller words that one with useCloudChatSignInInside.
*/
export function cloudChatSignInContent(
cloudHost: boolean,
failureKind: string | undefined,
agentName: string | undefined,
t: Translate = translate
t: Translate = translate,
inside: CloudChatSignInInside | null = null
): string | undefined {
const agent = agentSessionSetupFor(agentName)?.agent
if (!cloudHost || failureKind !== 'notSignedIn' || (agent !== 'claude' && agent !== 'codex')) {
if (
inside ||
!cloudHost ||
failureKind !== 'notSignedIn' ||
(agent !== 'claude' && agent !== 'codex')
) {
return undefined
}
return t(
@@ -45,3 +58,38 @@ export function cloudChatSignInContent(
{ agent: HOST_AGENT_SIGN_IN_AGENT_NAME[agent] }
)
}
/** What a "not signed in" chat says when its agent signs in inside the workspace, and its button. */
export type CloudChatSignInInside = { text: string; actionLabel: string }
export function cloudChatSignInInsideWords(
agent: 'claude' | 'codex',
t: Translate = translate
): CloudChatSignInInside {
return {
text: t('orcaCloud.chatSignIn.inside', '{{agent}} needs you to sign in inside this workspace', {
agent: HOST_AGENT_SIGN_IN_AGENT_NAME[agent]
}),
actionLabel: t('orcaCloud.chatSignIn.openTerminal', 'Open terminal to sign in')
}
}
/**
* A structured chat on Orca Cloud whose Claude or Codex the person signs in inside each workspace:
* nothing of Orca's signs it in, so "not signed in" means the agent's own login, run in a terminal on
* the workspace (the chat's Sign in button already opens one with the agent's login command).
*/
export function useCloudChatSignInInside(
agentName: string | undefined
): CloudChatSignInInside | null {
const t = useTranslate()
const cloudHost = useCloudChatHost()
const accounts = useOrcaCloudAgentAccounts(cloudHost)
const agent = agentSessionSetupFor(agentName)?.agent
if (!cloudHost || (agent !== 'claude' && agent !== 'codex')) {
return null
}
return orcaCloudAgentCoverageFromAccounts(agent, accounts)?.kind === 'inWorkspace'
? cloudChatSignInInsideWords(agent, t)
: null
}
@@ -7,25 +7,30 @@ import {
import { useTranslate } from '../../../i18n/use-translate'
export function NativeChatSetupButton({
failureKind
failureKind,
label
}: {
failureKind: string | undefined
/** Words for the button in place of the default (e.g. "Open terminal to sign in"). */
label?: string
}): React.JSX.Element | null {
return isNativeChatSetupFailureKind(failureKind) ? (
<EligibleNativeChatSetupButton failureKind={failureKind} />
<EligibleNativeChatSetupButton failureKind={failureKind} label={label} />
) : null
}
function EligibleNativeChatSetupButton({
failureKind
failureKind,
label
}: {
failureKind: string | undefined
label?: string
}): React.JSX.Element | null {
const t = useTranslate()
const action = nativeChatSetupAction(failureKind, useNativeChatHostSetupView(), t)
return action ? (
<NativeChatActionButton primary disabled={action.disabled} onClick={action.onClick}>
{action.label}
{label && !action.disabled ? label : action.label}
</NativeChatActionButton>
) : null
}
@@ -1,5 +1,6 @@
import {
cloudChatSignInContent,
useCloudChatSignInInside,
useCloudChatHost
} from '../../../agent-sign-in-strip/cloud-chat-sign-in-notice'
import {
@@ -92,6 +93,7 @@ export function NativeChatNoticeRow({
useTranslation()
const orcaStopView = useNativeChatOrcaStopView()
const cloudHost = useCloudChatHost()
const signInInside = useCloudChatSignInInside(agentName)
const openForkSource = useContext(NativeChatForkedFromContext)
if (block.presentation === PULL_REQUEST_EVENT_PRESENTATION) {
@@ -239,7 +241,8 @@ export function NativeChatNoticeRow({
failure?.account,
t
)
const cloudSignIn = cloudChatSignInContent(cloudHost, failure?.kind, agentName, t)
const insideSignIn = failure?.kind === 'notSignedIn' ? signInInside : null
const cloudSignIn = cloudChatSignInContent(cloudHost, failure?.kind, agentName, t, insideSignIn)
const usageLimitContinue =
block.usageLimitTurnItemId !== undefined &&
block.usageLimitTurnItemId === orcaStopView.usageLimitTurnItemId
@@ -320,7 +323,7 @@ export function NativeChatNoticeRow({
<div className="flex items-start gap-2">
{Icon ? <Icon aria-hidden="true" className="mt-0.5 size-4 shrink-0" /> : null}
<p className="min-w-0 flex-1 whitespace-pre-wrap break-words">
{cloudSignIn ?? antigravitySignInContent ?? text}
{cloudSignIn ?? insideSignIn?.text ?? antigravitySignInContent ?? text}
</p>
{continueOffered ? (
<NativeChatInterruptedContinueButton onContinue={orcaStopView.continueNow} />
@@ -336,7 +339,12 @@ export function NativeChatNoticeRow({
{usageLimitContinue && orcaStopView.usageLimitAutoContinue ? (
<NativeChatUsageLimitAutoContinue state={orcaStopView.usageLimitAutoContinue} />
) : null}
{cloudSignIn ? null : <NativeChatSetupButton failureKind={block.failure?.kind} />}
{cloudSignIn ? null : (
<NativeChatSetupButton
failureKind={block.failure?.kind}
label={insideSignIn?.actionLabel}
/>
)}
{block.providerFrame || detail ? (
<ProviderFrameRow
block={block}
@@ -0,0 +1,38 @@
// @vitest-environment happy-dom
import { renderHook } from '@testing-library/react'
import { describe, expect, it, vi } from 'vitest'
import { useNativeChatAvailabilityNotice } from './use-native-chat-availability-notice'
const signIn = { signIn: vi.fn(), signingIn: false }
describe('useNativeChatAvailabilityNotice: an Orca Cloud agent signed in inside the workspace', () => {
const base = {
unavailable: { reason: 'notSignedIn' as const },
setup: { signIn, install: null },
agent: 'claude' as const,
launchFailure: null,
journalItems: []
}
it('says the agent needs a sign-in inside the workspace, with a button that opens the login terminal', () => {
const { result } = renderHook(() =>
useNativeChatAvailabilityNotice({
...base,
signInInside: {
text: 'Claude needs you to sign in inside this workspace',
actionLabel: 'Open terminal to sign in'
}
})
)
expect(result.current?.text).toBe('Claude needs you to sign in inside this workspace')
expect(result.current?.action?.label).toBe('Open terminal to sign in')
result.current?.action?.onClick()
expect(signIn.signIn).toHaveBeenCalledOnce()
})
it('keeps the host’s own words without it', () => {
const { result } = renderHook(() => useNativeChatAvailabilityNotice(base))
expect(result.current?.text).not.toContain('inside this workspace')
expect(result.current?.action?.label).toBe('Sign in')
})
})
@@ -74,6 +74,8 @@ export function useNativeChatAvailabilityNotice(input: {
submissions?: readonly NoticeSubmission[]
/** The lines under the chat's messages, by message key: a refused send's says why. */
deliveryNotices?: ReadonlyMap<string, unknown>
/** An Orca Cloud agent signed in inside the workspace: "not signed in" says so, with its button. */
signInInside?: { text: string; actionLabel: string } | null
}): NativeChatComposerNotice | null {
const t = useTranslate()
const { unavailable, journalItems } = input
@@ -109,9 +111,20 @@ export function useNativeChatAvailabilityNotice(input: {
return null
}
const agentName = structuredAgentLabel(input.agent)
const inside = unavailable.reason === 'notSignedIn' ? (input.signInInside ?? null) : null
const action = nativeChatSetupAction(unavailable.reason, input.setup, t)
if (inside) {
return {
key: 'availability',
action: action && !action.disabled ? { ...action, label: inside.actionLabel } : action,
kind: 'error',
text: inside.text,
onDismiss: () => setDismissal({ key, dismissed: true })
}
}
return {
key: 'availability',
action: nativeChatSetupAction(unavailable.reason, input.setup, t),
action,
kind: 'error',
text:
unavailable.reason === 'cliMissing'
@@ -1,4 +1,5 @@
import { useCallback } from 'react'
import { useCloudChatSignInInside } from '../../../agent-sign-in-strip/cloud-chat-sign-in-notice'
import {
useCloudHeldChatUnavailable,
useIsCloudHeldChat
@@ -94,7 +95,8 @@ export function useNativeChatStructuredPaneNotices({
launchFailure: launch.lifecycle === 'failed' ? provisionalLaunch.failure : null,
journalItems: controller.journalItems,
submissions: controller.submissions,
deliveryNotices
deliveryNotices,
signInInside: useCloudChatSignInInside(props.agent)
})
// While Install shows, it says what to do; a start failure's "send again" or Retry can't help.
const notices = structuredSessionNotices(
@@ -0,0 +1,88 @@
import { Cloud } from 'lucide-react'
import { defineFixture } from '../../component-fixtures/component-fixture'
import { seedCloudWorkspaceSnapshot } from '@/lib/orca-cloud/orca-cloud-workspace-store'
import type { HostedWorkspaceStatus } from '../../../../shared/hosted-sandbox/hosted-sandbox-wire'
import { CloudMemoryPressureMark } from './CloudMemoryPressureMark'
import { CloudWorkspaceNotice } from './CloudWorkspaceNotice'
const noop = (): void => {}
type Pressure = 'high' | 'critical'
// MEMGUARD: a running cloud workspace's sidebar row while its VM is short of memory.
function seed(pressure: Pressure): void {
const status: HostedWorkspaceStatus = {
state: 'ready',
metrics: null,
memory: {
pressure,
guardKills: 0,
lastGuardVictim: null,
lastGuardKillAt: null,
guarded: true
},
uptimeMs: 60_000,
usage: { totalMs: 60_000, centsPerHour: 75, costCents: 1 },
archive: null,
checkedAt: Date.now()
}
seedCloudWorkspaceSnapshot({
ids: new Set(['gudgeon']),
idsLoaded: true,
status: { gudgeon: { status, loading: false, error: null, fetchedAt: Date.now() } }
})
}
function Row(): React.JSX.Element {
return (
<div className="flex w-[300px] items-center gap-1.5 rounded-md bg-accent/40 px-2 py-1.5 text-[13px]">
<Cloud className="size-3 text-muted-foreground" aria-hidden />
<span>gudgeon</span>
<CloudMemoryPressureMark worktreeId="gudgeon" />
</div>
)
}
const markFixture = (pressure: Pressure) =>
defineFixture({
width: 380,
render: () => {
seed(pressure)
return (
<div className="h-[110px] p-3">
<Row />
</div>
)
},
interact: async (frame: HTMLElement) => {
await new Promise((resolve) => setTimeout(resolve, 50))
const mark = frame.querySelector<HTMLElement>('[data-cloud-memory-pressure]')
mark?.dispatchEvent(new PointerEvent('pointermove', { bubbles: true }))
mark?.dispatchEvent(new PointerEvent('pointerenter', { bubbles: true }))
await new Promise((resolve) => setTimeout(resolve, 900))
}
})
/** Memory is low (under 15% available, or the kernel stalls on it): the quiet mark. */
export const sidebarRowMemoryHigh = markFixture('high')
/** Under 8%: Orca's guard may stop the largest process. Amber. */
export const sidebarRowMemoryCritical = markFixture('critical')
/** MEMGUARD: Orca's memory guard stopped the process; the workspace stayed up, so no Restart. */
export const noticeOomGuard = defineFixture({
width: 420,
render: () => (
<div className="bg-background p-4">
<CloudWorkspaceNotice
notice={{
kind: 'oom',
oom: { workspaceId: 'ws', oomKills: 1, victim: 'tsc', stoppedBy: 'guard', at: Date.now() }
}}
restarting={false}
onRestart={noop}
onMachineSize={noop}
onDismiss={noop}
/>
</div>
)
})
@@ -0,0 +1,53 @@
import React from 'react'
import { MemoryStick } from 'lucide-react'
import { cn } from '@/lib/class-names'
import { useTranslate } from '@/i18n/use-translate'
import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip'
import { useCloudWorkspace } from '@/lib/orca-cloud/orca-cloud-workspace-store'
import { cloudMemoryPressure } from '@/lib/orca-cloud/cloud-memory-pressure'
/**
* MEMGUARD: the sidebar row's mark for a cloud workspace short of memory, from the status read
* (refreshed every minute while it runs). Quiet while memory is low; amber once Orca may stop the
* largest process to keep the workspace responsive. Renders nothing otherwise.
*/
export function CloudMemoryPressureMark({
worktreeId
}: {
worktreeId: string
}): React.JSX.Element | null {
const t = useTranslate()
const pressure = useCloudWorkspace((state) =>
cloudMemoryPressure(state.status[worktreeId]?.status)
)
if (!pressure) {
return null
}
const label =
pressure === 'critical'
? t(
'orcaCloud.memoryPressure.critical',
'Out of memory. Orca may stop the largest process to keep the workspace responsive.'
)
: t('orcaCloud.memoryPressure.high', 'Running low on memory')
return (
<Tooltip>
<TooltipTrigger asChild>
<span
role="img"
aria-label={label}
data-cloud-memory-pressure={pressure}
className={cn(
'inline-flex size-3.5 shrink-0 items-center justify-center',
pressure === 'critical' ? 'text-status-warning-text' : 'text-muted-foreground'
)}
>
<MemoryStick className="size-3" aria-hidden />
</span>
</TooltipTrigger>
<TooltipContent side="right" sideOffset={6}>
{label}
</TooltipContent>
</Tooltip>
)
}
@@ -2,6 +2,7 @@ import React, { useId } from 'react'
import type { LucideIcon } from 'lucide-react'
import {
CircleArrowUp,
Gauge,
Loader2,
MemoryStick,
Plus,
@@ -85,21 +86,59 @@ function CloudNoticeCard({
/**
* The workspace stopped answering, or a process ran out of memory. Both offer Restart (same disk).
* MEMGUARD: when Orca's memory guard stopped the process, the workspace is fine and needs no
* restart: the notice says what was stopped and offers machine sizes instead.
*/
export function CloudWorkspaceNotice({
notice,
restarting,
onRestart,
onMachineSize,
onDismiss
}: {
notice: CloudNoticeKind
restarting: boolean
onRestart: () => void
/** Opens where machine sizes are chosen (cloud environments); without it the guard notice has no action. */
onMachineSize?: () => void
onDismiss?: () => void
}): React.JSX.Element {
const t = useTranslate()
const isOom = notice.kind === 'oom'
const victim = isOom ? notice.oom.victim : null
if (isOom && notice.oom.stoppedBy === 'guard') {
return (
<CloudNoticeCard
noticeKind="oom-guard"
icon={MemoryStick}
title={
victim
? t(
'orcaCloudWorkspace.oom.guardTitle',
'Orca stopped {{name}} to keep the workspace responsive',
{ name: victim }
)
: t(
'orcaCloudWorkspace.oom.guardTitleProcess',
'Orca stopped a process to keep the workspace responsive'
)
}
detail={t(
'orcaCloudWorkspace.oom.guardDetail',
'The workspace was out of memory. Consider a larger machine.'
)}
onDismiss={onDismiss}
action={
onMachineSize ? (
<Button size="xs" variant="outline" onClick={onMachineSize}>
<Gauge />
{t('orcaCloudWorkspace.oom.machineSize', 'Machine size')}
</Button>
) : null
}
/>
)
}
const title = isOom
? victim
? t('orcaCloudWorkspace.oom.titleRan', '{{name}} ran out of memory', { name: victim })
@@ -260,6 +260,30 @@ describe('Orca Cloud surfaces', () => {
expect(onDismiss).toHaveBeenCalled()
})
it('a guard kill says Orca stopped the process and offers machine sizes, not Restart', () => {
const onMachineSize = vi.fn()
const onRestart = vi.fn()
const { getByRole, queryByRole } = render(
<CloudWorkspaceNotice
notice={{
kind: 'oom',
oom: { workspaceId: 'ws', oomKills: 1, victim: 'tsc', stoppedBy: 'guard', at: 1 }
}}
restarting={false}
onRestart={onRestart}
onMachineSize={onMachineSize}
onDismiss={noop}
/>
)
const text = getByRole('alert').textContent
expect(text).toContain('Orca stopped tsc to keep the workspace responsive')
expect(text).toContain('The workspace was out of memory. Consider a larger machine.')
expect(queryByRole('button', { name: 'Restart' })).toBeNull()
fireEvent.click(getByRole('button', { name: 'Machine size' }))
expect(onMachineSize).toHaveBeenCalled()
expect(onRestart).not.toHaveBeenCalled()
})
it('version skew: an older sandbox offers a new workspace, a newer one an update', () => {
const onCheckForUpdates = vi.fn()
const onCreateWorkspace = vi.fn()
@@ -44,6 +44,13 @@ import { clearWakeInput } from '@/components/terminal-pane/input/wake-input-buff
import { runUnarchiveCloudWorkspace, wakeCloudWorkspace } from './cloud-workspace-actions'
import { keepWorkspaceAwakeDuringSleep } from '@/components/sidebar/workspace-actions/sleep/workspace-sleep-follower'
/** Machine sizes live on the team's cloud environments (MEMGUARD's "Consider a larger machine"). */
function openMachineSizes(): void {
const store = useAppStore.getState()
store.openSettingsTarget({ pane: 'cloud-environments', repoId: null })
store.openSettingsPage()
}
/**
* The Orca Cloud layer over the active workspace, one mount for all of it: the archived view,
* the "Waking…" overlay, the asleep (cached, read-only) view, and the not-responding / out-of-
@@ -265,6 +272,7 @@ function CloudSurface({ workspaceId }: { workspaceId: string }): React.JSX.Eleme
notice={{ kind: 'oom', oom: view.oom }}
restarting={view.pending === 'restart'}
onRestart={restart}
onMachineSize={openMachineSizes}
onDismiss={() => dismissCloudOom(workspaceId)}
/>
)
@@ -3,7 +3,7 @@
import { getIntlLocale } from '@/i18n/i18n'
import type { Translate } from '@/i18n/use-translate'
import type { CloudDotState } from '@/lib/orca-cloud/orca-cloud-workspace-state'
import type { HostedVmMetrics } from '../../../../shared/hosted-sandbox/hosted-sandbox-wire'
import type { HostedVmMetrics } from '../../../../shared/hosted-sandbox/hosted-workspace-vm-wire'
import {
ORCA_CLOUD_IDLE_SLEEP_MINUTES,
ORCA_CLOUD_SANDBOX_SPEC
@@ -83,6 +83,10 @@ export function AccountRow({
variant="ghost"
size="sm"
className="text-muted-foreground"
title={t(
'agentSignInStrip.signInInsideWorkspaceDetail',
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.'
)}
onClick={() =>
void actions.setSource(agent, IN_WORKSPACE_SOURCE).then((error) => {
if (error) {
@@ -108,7 +112,7 @@ export function AccountRow({
title={t('orcaCloudAccounts.inWorkspace', 'Signs in inside each workspace')}
detail={t(
'orcaCloudAccounts.inWorkspaceDetail',
'Orca doesn’t store your {{product}} sign-in. You sign in again in each new cloud workspace.',
'Orca doesn’t store your {{product}} sign-in. You sign in again in each new cloud workspace. Team API keys aren’t used for this agent.',
{ product }
)}
actions={
@@ -194,7 +194,7 @@ describe('OrcaCloudAccountsView', () => {
expect(screen.getByText('Signs in inside each workspace')).toBeTruthy()
expect(
screen.getByText(
'Orca doesn’t store your ChatGPT sign-in. You sign in again in each new cloud workspace.'
'Orca doesn’t store your ChatGPT sign-in. You sign in again in each new cloud workspace. Team API keys aren’t used for this agent.'
)
).toBeTruthy()
fireEvent.click(screen.getByRole('button', { name: 'Use my ChatGPT account' }))
@@ -26,6 +26,7 @@ import { WorktreeHostContextBadge } from '../host/WorktreeHostContextBadge'
import { WorktreeCardHostMark, WorktreeCardHostStateWord } from '../host/WorktreeCardHostMark'
import { WorktreeTitleInlineRename } from '../identity/WorktreeTitleInlineRename'
import { CloudAgentHeldMark } from '@/components/orca-cloud/CloudAgentHeldMark'
import { CloudMemoryPressureMark } from '@/components/orca-cloud/CloudMemoryPressureMark'
import { cardElement } from './card-template-compiler'
type Card = WorktreeCardController
@@ -58,6 +59,8 @@ function title(props: Readonly<Record<string, unknown>>, card: Card): React.Reac
/>
{/* N-03: the agent waits on its pane's strip (Orca Cloud): this mark is the signal away. */}
{card.titleRenaming ? null : <CloudAgentHeldMark worktreeId={worktree.id} />}
{/* MEMGUARD: the cloud VM is short of memory. */}
{card.titleRenaming ? null : <CloudMemoryPressureMark worktreeId={worktree.id} />}
</>
)
}
+14 -4
View File
@@ -27542,7 +27542,11 @@
"oom": {
"titleRan": "{{name}} ran out of memory",
"titleProcess": "A process ran out of memory",
"stopped": "It was stopped so the sandbox stays up. Restart to recover."
"stopped": "It was stopped so the sandbox stays up. Restart to recover.",
"guardTitle": "Orca stopped {{name}} to keep the workspace responsive",
"guardTitleProcess": "Orca stopped a process to keep the workspace responsive",
"guardDetail": "The workspace was out of memory. Consider a larger machine.",
"machineSize": "Machine size"
},
"unresponsive": {
"sandboxTitle": "This sandbox isn’t responding"
@@ -28103,7 +28107,9 @@
"renewAbove": "Renew {{agent}} sign-in above to send"
},
"chatSignIn": {
"renewAbove": "Your {{agent}} sign-in for Orca Cloud didn’t work. Renew it above, then send your message again."
"renewAbove": "Your {{agent}} sign-in for Orca Cloud didn’t work. Renew it above, then send your message again.",
"inside": "{{agent}} needs you to sign in inside this workspace",
"openTerminal": "Open terminal to sign in"
},
"chatHold": {
"notSetUp": "Set up {{agent}} above to start",
@@ -28203,6 +28209,10 @@
"waiting": "Waiting for GitHub… This continues on its own once access is granted.",
"holding": "Your sandbox is ready. It opens as soon as you grant access or skip.",
"openNow": "Open now, set up later"
},
"memoryPressure": {
"critical": "Out of memory. Orca may stop the largest process to keep the workspace responsive.",
"high": "Running low on memory"
}
},
"runActions": {
@@ -28686,7 +28696,7 @@
"deviceCodeOffHint": "If OpenAI says device code sign-in is off, turn it on in ChatGPT → Settings → Security, then try again.",
"signInInsideWorkspace": "Sign in inside each workspace instead",
"inWorkspace": "Signs in inside each workspace",
"inWorkspaceDetail": "Orca doesn’t store your {{product}} sign-in. You sign in again in each new cloud workspace."
"inWorkspaceDetail": "Orca doesn’t store your {{product}} sign-in. You sign in again in each new cloud workspace. Team API keys aren’t used for this agent."
},
"vmBuildAgents": {
"menu": {
@@ -28862,7 +28872,7 @@
"pasteAgentTokenDetail": "Run `claude setup-token` in any terminal where {{agent}} is installed, then paste the token here. It’s saved to your Orca account, never to a sandbox.",
"moreChoices": "More choices",
"signInInsideWorkspace": "Sign in inside this workspace instead",
"signInInsideWorkspaceDetail": "Orca won’t store your sign-in. You’ll sign in again in each new workspace."
"signInInsideWorkspaceDetail": "Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent."
},
"cloudSetupPage": {
"back": "Back",
@@ -0,0 +1,33 @@
import { describe, expect, it } from 'vitest'
import type { HostedWorkspaceStatus } from '../../../../shared/hosted-sandbox/hosted-sandbox-wire'
import { cloudMemoryPressure } from './cloud-memory-pressure'
const status = (state: HostedWorkspaceStatus['state']): HostedWorkspaceStatus => ({
state,
metrics: null,
uptimeMs: null,
usage: { totalMs: 0, centsPerHour: 75, costCents: 0 },
archive: null,
checkedAt: 1
})
describe('cloudMemoryPressure', () => {
it('marks memory pressure only on a ready VM that is short of memory', () => {
const memory = (pressure: 'ok' | 'high' | 'critical') => ({
pressure,
guardKills: 0,
lastGuardVictim: null,
lastGuardKillAt: null,
guarded: true
})
expect(cloudMemoryPressure({ ...status('ready'), memory: memory('high') })).toBe('high')
expect(cloudMemoryPressure({ ...status('ready'), memory: memory('critical') })).toBe('critical')
expect(cloudMemoryPressure({ ...status('ready'), memory: memory('ok') })).toBeNull()
// An older control plane says nothing; a sleeping or unresponsive VM shows its own state.
expect(cloudMemoryPressure(status('ready'))).toBeNull()
expect(
cloudMemoryPressure({ ...status('unresponsive'), memory: memory('critical') })
).toBeNull()
expect(cloudMemoryPressure(null)).toBeNull()
})
})
@@ -0,0 +1,15 @@
import type { HostedWorkspaceStatus } from '../../../../shared/hosted-sandbox/hosted-sandbox-wire'
/**
* MEMGUARD: the sidebar row's memory mark. Only for a VM read as ready (a sleeping or unresponsive
* one has its own state), and only when memory is short.
*/
export function cloudMemoryPressure(
status: HostedWorkspaceStatus | null | undefined
): 'high' | 'critical' | null {
if (status?.state !== 'ready') {
return null
}
const pressure = status.memory?.pressure
return pressure === 'high' || pressure === 'critical' ? pressure : null
}
@@ -55,9 +55,11 @@ export type CloudWakeProgressEvent = {
export type CloudOomEvent = {
workspaceId: string
/** Kernel OOM-kill counter after this notice (monotonic per VM boot). */
/** Kernel plus memory-guard kills after this notice (monotonic per VM boot). */
oomKills: number
victim: string | null
/** MEMGUARD: Orca's memory guard stopped it (the workspace stayed responsive), not the kernel. */
stoppedBy?: 'guard'
at: number
}
@@ -2,6 +2,7 @@
// The Worker's tests parse its responses with these schemas, so either side drifting fails a test.
import { z } from 'zod'
import { HostedEnvironmentHeldSchema } from './hosted-environment-held-wire'
import { HostedVmMetricsSchema, HostedWorkspaceMemorySchema } from './hosted-workspace-vm-wire'
/**
* Minutes of no agent work and no input before a cloud sandbox sleeps. The one source for every
@@ -239,24 +240,6 @@ export const HOSTED_WORKSPACE_STATES = [
] as const
export type HostedWorkspaceState = (typeof HOSTED_WORKSPACE_STATES)[number]
const Bytes = z.number().int().nonnegative()
export const HostedVmMetricsSchema = z
.object({
cpuPercent: z.number().int().min(0).max(100),
cpus: z.number().int().min(1),
load1: z.number().min(0),
memTotalBytes: Bytes,
memUsedBytes: Bytes,
diskTotalBytes: Bytes,
diskUsedBytes: Bytes,
uptimeSeconds: z.number().int().nonnegative(),
oomKills: z.number().int().nonnegative(),
lastOomVictim: z.string().max(64).nullable(),
orcadAlive: z.boolean()
})
.strict()
export type HostedVmMetrics = z.infer<typeof HostedVmMetricsSchema>
export const HostedWorkspaceStatusSchema = z
.object({
state: z.enum(HOSTED_WORKSPACE_STATES),
@@ -272,6 +255,8 @@ export const HostedWorkspaceStatusSchema = z
archive: z.object({ archivedAt: Millis, expiresAt: Millis.nullable() }).strict().nullable(),
// Optional: older control planes omit it; the desktop then uses its 15 min fallback.
idleSleepMinutes: z.number().int().positive().max(1440).optional(),
// Optional: older control planes omit it (MEMGUARD).
memory: HostedWorkspaceMemorySchema.nullable().optional(),
// Optional (feature `wake-hints`): predictive wake's inputs. The last sleep someone chose, from
// any client (not the idle timer), and sandboxes running against the owner's seat and account.
lastUserSleepAt: Millis.nullable().optional(),
@@ -0,0 +1,37 @@
import { z } from 'zod'
// The VM's live metrics (status panel, OOM banner), parsed strictly: a field a desktop doesn't know
// fails the read, so new VM facts go beside it (like `memory`), not in it.
const Bytes = z.number().int().nonnegative()
export const HostedVmMetricsSchema = z
.object({
cpuPercent: z.number().int().min(0).max(100),
cpus: z.number().int().min(1),
load1: z.number().min(0),
memTotalBytes: Bytes,
memUsedBytes: Bytes,
diskTotalBytes: Bytes,
diskUsedBytes: Bytes,
uptimeSeconds: z.number().int().nonnegative(),
oomKills: z.number().int().nonnegative(),
lastOomVictim: z.string().max(64).nullable(),
orcadAlive: z.boolean()
})
.strict()
export type HostedVmMetrics = z.infer<typeof HostedVmMetricsSchema>
/**
* MEMGUARD: memory pressure in a cloud VM and the processes its memory guard stopped this boot
* (kernel OOM kills stay in the status's `metrics.oomKills`). Pressure without a guard (an older
* image) comes from MemAvailable alone, and `guarded` is false.
*/
export const HostedWorkspaceMemorySchema = z
.object({
pressure: z.enum(['ok', 'high', 'critical']),
guardKills: z.number().int().nonnegative(),
lastGuardVictim: z.string().max(64).nullable(),
lastGuardKillAt: z.number().int().nonnegative().nullable(),
guarded: z.boolean()
})
.strip()
export type HostedWorkspaceMemory = z.infer<typeof HostedWorkspaceMemorySchema>