fix(cloud): a boot whose member identity install fails records it too (VM git-setup.failed + gitSetup), so Restart can't hide a failure

Boot-time installs were swallowed and the boot cleared gitSetup, so a restart whose install failed again dropped the desktop strip while git still lacked identities. Every boot install now goes through settleMemberGit: failure marks the VM and the row, a later success (or a boot with nobody to install) clears them. The git shim refuses on a failed mark even with nothing pending. Restart is the retry (it re-runs the install before it answers).
This commit is contained in:
Neil
2026-11-11 19:29:18 -08:00
parent 8bb65d69b0
commit 4bcf815aa7
6 changed files with 36 additions and 14 deletions
File diff suppressed because one or more lines are too long
+2 -1
View File
@@ -19,7 +19,8 @@ done
# Read-only git and fetches never wait (they use the workspace's own credentials, set before ready).
hold_for_member_setup() {
local state="${HOME:-/nonexistent}/.orca-sandbox" ticks=0 since
[ -f "$state/git-setup.pending" ] || return 0
# A failed install on any boot (create or a later restart) refuses even with nothing pending.
[ -f "$state/git-setup.pending" ] || [ -f "$state/git-setup.failed" ] || return 0
since="$(cat "$state/git-setup.pending" 2>/dev/null || echo 0)"
case "$since" in '' | *[!0-9]*) since=0 ;; esac
while [ -f "$state/git-setup.pending" ] && [ ! -f "$state/git-setup.ready" ] && [ ! -f "$state/git-setup.failed" ]; do
+2 -2
View File
@@ -11,7 +11,7 @@ import { z } from 'zod'
import type { SandboxCaller } from './auth'
import { isDeviceSessionLive } from './device-sessions'
import { markRevoked, runVmRevoke, type GrantRow } from './grant-revokes'
import { LifecycleError, finishMemberGitAfterCreate, resumeSandbox, syncMemberGit, type LifecycleDeps, type Ready } from './lifecycle'
import { LifecycleError, settleMemberGit, resumeSandbox, syncMemberGit, type LifecycleDeps, type Ready } from './lifecycle'
import { ensureEntryPoints } from './golden-images'
import type { CommandResult } from './sandbox-provider'
import { ORCAD_PORT } from './sandbox-vm-commands'
@@ -328,7 +328,7 @@ export async function recordOwnerDeviceForCreate(deps: LifecycleDeps, ready: Rea
}
// Always one full install: the boot skipped its own (connected members included), so even a retried
// create whose device was already recorded must install, and the install is what marks setup ready.
const work = finishMemberGitAfterCreate(deps, ready.row, sessionId, ready.row.status === 'running')
const work = settleMemberGit(deps, ready.row, sessionId, ready.row.status === 'running')
if (deps.background) {
deps.background(work)
} else {
+15 -9
View File
@@ -429,8 +429,6 @@ async function boot(
archived_at: null,
archive_expires_at: null,
retention_warned_at: null,
// A boot that installs identities before it answers starts the VM's setup clean (CREATE-GIT-ASYNC).
...(gitSetupPending ? {} : { git_setup_failed_at: null }),
github_token_expires_at: git?.expiresAt ?? null,
egress_secret_hosts: routed.egress_secret_hosts,
egress_allow_hosts: routed.egress_allow_hosts,
@@ -439,8 +437,15 @@ async function boot(
})
// Per-member git identities for `orca-git-as` (policy routes were refreshed above). A create that
// defers them (CREATE-GIT-ASYNC) installs them after it answers, once the owner device is recorded.
if (!gitSetupPending && deps.memberIdentity && (await connectedMembers(deps.db, running)).length > 0) {
await syncMemberGit(deps, running, state.session.id)
if (!gitSetupPending && memberGitApplies(deps, running)) {
if ((await connectedMembers(deps.db, running)).length > 0) {
// Recorded like a create's: a failure marks the VM (its git refuses commits and pushes) and the row
// (gitSetup 'failed'); only a later successful install clears them. Restart re-runs this.
await settleMemberGit(deps, running, state.session.id, true)
} else if (running.git_setup_failed_at) {
// Nobody's identity to install on this boot: nothing is failing any more.
await deps.db.prepare('UPDATE sandboxes SET git_setup_failed_at = NULL WHERE id = ?').bind(running.id).run()
}
}
await deps.hooks?.afterBoot?.(running, state.session.id).catch(() => undefined)
// Secrets saved while this VM booted missed prep's env; re-check once it is running (env-refresh.ts).
@@ -562,7 +567,7 @@ async function installMemberGit(deps: LifecycleDeps, row: SandboxRow, sessionId:
/** Marks this boot's member setup in the VM (its `git` reads it); best effort, a lost mark times out there. */
export const GIT_SETUP_MARK_SCRIPT =
'state="$HOME/.orca-sandbox"; mkdir -p "$state" && rm -f "$state/git-setup.pending" && date +%s >"$state/git-setup.$1"'
'state="$HOME/.orca-sandbox"; mkdir -p "$state" && rm -f "$state"/git-setup.pending "$state"/git-setup.ready "$state"/git-setup.failed && date +%s >"$state/git-setup.$1"'
async function markGitSetup(deps: LifecycleDeps, sessionId: string, outcome: 'ready' | 'failed'): Promise<void> {
await deps.provider
@@ -571,11 +576,12 @@ async function markGitSetup(deps: LifecycleDeps, sessionId: string, outcome: 're
}
/**
* CREATE-GIT-ASYNC: after a create answered, install member identities (`install` false: no new
* device, nothing to install) and settle the VM's pending setup either way. A failure stays on the
* row (gitSetup 'failed') until a later sync succeeds.
* Installs member identities and records the outcome (CREATE-GIT-ASYNC): after a create answered, and on
* every later boot that installs them. `install` false (the VM is not running) only marks it. A failure
* marks the VM (`git-setup.failed`, its git refuses commits and pushes) and the row (gitSetup 'failed')
* until a later install succeeds.
*/
export async function finishMemberGitAfterCreate(
export async function settleMemberGit(
deps: LifecycleDeps,
row: SandboxRow,
sessionId: string,
@@ -118,7 +118,14 @@ describe('create answers before member git setup', () => {
// Desktops that do not send the feature parse the view strictly and never see the field.
const strict = await t.h.call('GET', `/v1/sandboxes/${id}`, { bearer: await t.as('alice') })
expect(strict.body.sandbox.gitSetup).toBeUndefined()
// Restart installs identities before it answers (unchanged); a success clears the failure.
// A restart whose install fails again keeps the failure (VM mark and flag): never a silent broken state.
const marksBefore = t.marks().length
const again = await t.h.call('POST', `/v1/sandboxes/${id}/restart`, { bearer: await t.as('alice') })
expect(again.status).toBe(200)
expect(t.marks().slice(marksBefore)).toEqual(['failed'])
const still = await t.h.call('GET', `/v1/sandboxes/${id}`, { bearer: await t.as('alice'), headers: GIT_SETUP_AWARE })
expect(still.body.sandbox.gitSetup).toBe('failed')
// Restart is the retry: it installs identities before it answers, and a success clears the failure.
t.failMembers(0)
const restarted = await t.h.call('POST', `/v1/sandboxes/${id}/restart`, { bearer: await t.as('alice') })
expect(restarted.status).toBe(200)
@@ -80,6 +80,14 @@ describe('the git shim while member setup is pending', () => {
expect(reachedRealGit(run.stderr)).toBe(false)
})
it('refuses after a failed install on a later boot (restart), with nothing pending', () => {
writeFileSync(join(state, 'git-setup.failed'), `${now()}\n`)
const run = git(['push'])
expect(run.status).toBe(1)
expect(run.stderr).toContain("Orca couldn't set up your GitHub identity")
expect(run.ms).toBeLessThan(2_000)
})
it('gives up after its wait instead of blocking forever, and says so', () => {
writeFileSync(join(state, 'git-setup.pending'), `${now()}\n`)
const run = git(['push'], 5)