diff --git a/cloud/apps/sandbox-control/migrations/0047_agent_credential_checks.sql b/cloud/apps/sandbox-control/migrations/0047_agent_credential_checks.sql new file mode 100644 index 00000000000..0f7b6673e5f --- /dev/null +++ b/cloud/apps/sandbox-control/migrations/0047_agent_credential_checks.sql @@ -0,0 +1,17 @@ +-- A provider 401 on an Orca-account request no longer revokes on its own: the control plane confirms +-- with one cheap server-side auth check first (agent-credential-checks.ts). checked_at rate-limits those +-- checks per person and agent; every outcome is audited. +ALTER TABLE agent_credentials ADD COLUMN checked_at INTEGER; + +CREATE TABLE agent_credential_audit ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + user_id TEXT NOT NULL, + agent TEXT NOT NULL, + -- revoked | kept | skipped + action TEXT NOT NULL, + reason TEXT NOT NULL, + sandbox_id TEXT, + upstream_path TEXT, + created_at INTEGER NOT NULL +); +CREATE INDEX agent_credential_audit_user ON agent_credential_audit (user_id, created_at); diff --git a/cloud/apps/sandbox-control/src/account-providers.ts b/cloud/apps/sandbox-control/src/account-providers.ts index f0ddd66de92..e8f8a7a513f 100644 --- a/cloud/apps/sandbox-control/src/account-providers.ts +++ b/cloud/apps/sandbox-control/src/account-providers.ts @@ -30,6 +30,47 @@ export type AccountProvider = { onUnauthorized?(ctx: { db: D1Database; keyring: Keyring; userId: string; now: number; waitUntil: (work: Promise) => void }): Promise /** The real value for this person, or null (none, expired, revoked). */ credential(db: D1Database, keyring: Keyring, userId: string, now: number): Promise + /** + * One cheap server-side auth check with the real credential (spends no model tokens), run before a + * provider 401 may revoke it (agent-credential-checks.ts). 'invalid' only for a definite auth failure. + */ + verify?(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'> +} + +/** Anthropic's token counter: authenticated like Messages, free, and nothing is generated. */ +const CLAUDE_CHECK_URL = 'https://api.anthropic.com/v1/messages/count_tokens' +const CLAUDE_CHECK_MODEL = 'claude-sonnet-4-5' + +async function verifyClaudeToken(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'> { + let response: Response + try { + response = await fetchImpl(CLAUDE_CHECK_URL, { + method: 'POST', + headers: { + authorization: `Bearer ${credential}`, + 'anthropic-beta': 'oauth-2025-04-20', + 'anthropic-version': '2023-06-01', + 'content-type': 'application/json' + }, + body: JSON.stringify({ model: CLAUDE_CHECK_MODEL, messages: [{ role: 'user', content: 'ok' }] }), + signal: AbortSignal.timeout(8_000) + }) + } catch { + return 'unknown' + } + if (response.ok) { + return 'valid' + } + if (response.status !== 401) { + // Rate limits, overload, a model name the account can't use, permissions: none of these say the token is dead. + return 'unknown' + } + const body: unknown = await response.json().catch(() => null) + const type = + body && typeof body === 'object' && 'error' in body && body.error && typeof body.error === 'object' && 'type' in body.error + ? body.error.type + : null + return type === 'authentication_error' ? 'invalid' : 'unknown' } /** D13: a workspace shared with teammates never uses a personal account. Fails closed: only a row @@ -46,7 +87,8 @@ export const claudeAccount: AccountProvider = { sessionEnv: (row) => ({ CLAUDE_CODE_OAUTH_TOKEN: `orca-ph-claude-account-${row.id}` }), header: 'authorization', format: (credential) => `Bearer ${credential}`, - credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now) + credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now), + verify: verifyClaudeToken } export const ACCOUNT_PROVIDERS: AccountProvider[] = [claudeAccount, codexAccount] diff --git a/cloud/apps/sandbox-control/src/agent-credential-checks.ts b/cloud/apps/sandbox-control/src/agent-credential-checks.ts new file mode 100644 index 00000000000..0dc07a408bf --- /dev/null +++ b/cloud/apps/sandbox-control/src/agent-credential-checks.ts @@ -0,0 +1,68 @@ +// A provider 401 on an Orca-account request (the proxy swapped the real credential in) is a signal, not a +// verdict: one transient auth error must not sign a person out of every workspace. Before revoking, the +// control plane makes one cheap server-side auth check with the real credential (no tokens spent) and +// revokes only when that also says the credential is dead. Checks are rate-limited per person and agent +// (agent_credentials.checked_at), and every decision is audited (agent_credential_audit). +import type { AccountProvider } from './account-providers' +import { markAgentCredentialRevoked, readUsableAgentCredential } from './agent-credentials' +import type { Keyring } from './secrets-crypto' + +/** At most one auth check (and so one revoke) per person and agent per minute. */ +export const CREDENTIAL_CHECK_INTERVAL_MS = 60_000 + +export type CredentialVerdict = 'valid' | 'invalid' | 'unknown' +export type UnauthorizedOutcome = 'revoked' | 'kept' | 'skipped' + +async function audit( + db: D1Database, + entry: { userId: string; agent: string; action: 'revoked' | 'kept'; reason: string; sandboxId: string; path: string; now: number } +): Promise { + await db + .prepare( + `INSERT INTO agent_credential_audit (user_id, agent, action, reason, sandbox_id, upstream_path, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ) + .bind(entry.userId, entry.agent, entry.action, entry.reason, entry.sandboxId, entry.path.split('?')[0]!.slice(0, 200), entry.now) + .run() + console.log(JSON.stringify({ event: `agent_credential_${entry.action}`, agent: entry.agent, by: 'egress_401', reason: entry.reason, sandboxId: entry.sandboxId })) +} + +export async function handleAccountUnauthorized(ctx: { + db: D1Database + keyring: Keyring + provider: Pick + userId: string + sandboxId: string + /** The upstream path that answered 401 (audit only; no query string). */ + path: string + now: number + fetch: typeof fetch +}): Promise { + const { db, provider, userId, now } = ctx + const agent = provider.agent + // Claim the check slot: a burst of parallel 401s, or a flapping provider, makes one check a minute. + const claimed = await db + .prepare( + `UPDATE agent_credentials SET checked_at = ? + WHERE user_id = ? AND agent = ? AND state = 'active' AND (checked_at IS NULL OR checked_at <= ?)` + ) + .bind(now, userId, agent, now - CREDENTIAL_CHECK_INTERVAL_MS) + .run() + if (!claimed.meta.changes) { + console.log(JSON.stringify({ event: 'agent_credential_401_skipped', agent, sandboxId: ctx.sandboxId })) + return 'skipped' + } + const real = await readUsableAgentCredential(db, ctx.keyring, userId, agent, now) + if (!real) { + return 'skipped' + } + const verdict: CredentialVerdict = provider.verify ? await provider.verify(real, ctx.fetch) : 'unknown' + const base = { userId, agent, sandboxId: ctx.sandboxId, path: ctx.path, now } + if (verdict !== 'invalid') { + await audit(db, { ...base, action: 'kept', reason: verdict === 'valid' ? 'check_passed' : 'check_inconclusive' }) + return 'kept' + } + await markAgentCredentialRevoked(db, userId, agent, now) + await audit(db, { ...base, action: 'revoked', reason: 'provider_401_confirmed_by_check' }) + return 'revoked' +} diff --git a/cloud/apps/sandbox-control/src/egress-proxy.ts b/cloud/apps/sandbox-control/src/egress-proxy.ts index 261d07d7f11..f249bffd20b 100644 --- a/cloud/apps/sandbox-control/src/egress-proxy.ts +++ b/cloud/apps/sandbox-control/src/egress-proxy.ts @@ -40,7 +40,7 @@ export type ProxiedRoute = { /** Extra headers set with the credential (e.g. an account id the client can't know). */ extraHeaders?: () => Record /** Sees the provider's status (e.g. a 401 revokes an account credential). Never sees the body. */ - onUpstreamStatus?: (status: number) => void + onUpstreamStatus?: (status: number, request: { method: string; path: string }) => void /** Narrows where this credential may be used (member tokens: the workspace repository only). */ pathAllowed?: (method: string, path: string) => boolean /** The reply must be a JSON object; only these top-level fields pass (no refresh token for the VM). */ @@ -167,7 +167,7 @@ export async function handleEgressProxy(request: Request, deps: EgressProxyDeps) console.log( JSON.stringify({ event: 'egress_swap', sandboxId: row.id, host, credential: route.label ?? route.header, status: upstream.status }) ) - route.onUpstreamStatus?.(upstream.status) + route.onUpstreamStatus?.(upstream.status, { method: request.method, path }) if (route.keepResponseFields) { return onlyFields(upstream, route.keepResponseFields) } diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 086f62003a4..88fd55ca1bc 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -114,6 +114,7 @@ import { egressAccountRoutes, egressSecretHosts, placeholderValues, secretPlaceh import { SecretError, handleSecretsRoute, keyringFor, type SecretChange } from './secrets' import { refreshAfterSecretChange, refreshSandboxEnv } from './env-refresh' import { accountProvider, isSharedWorkspace } from './account-providers' +import { handleAccountUnauthorized } from './agent-credential-checks' import { codexBrokerTick, handleCodexBrokerRoute, type BrokerDeps } from './codex-broker' import { refreshCodexLeases } from './codex-credentials' import { @@ -123,7 +124,6 @@ import { deleteAgentCredential, providerEnabled, putClaudeCredential, - markAgentCredentialRevoked, readUsableAgentCredential } from './agent-credentials' import { @@ -371,16 +371,35 @@ function egressRoutes(env: Env, deps: Deps): (row: SandboxRow, host: string) => return provider.format(real) }, ...(provider.extraHeaders ? { extraHeaders: () => (real ? provider.extraHeaders!(real) : {}) } : {}), - onUpstreamStatus: (status) => { + // Reached only after the real credential was swapped in and sent (denied or unswapped requests + // never get here). + onUpstreamStatus: (status, request) => { if (status === 401) { - // The provider no longer accepts it (revoked at the source): stop using it until renewed. const now = (deps.now ?? Date.now)() const waitUntil = (work: Promise) => (deps.waitUntil ? deps.waitUntil(work.catch(() => undefined)) : void work.catch(() => undefined)) - const handled = provider.onUnauthorized - ? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil })) - : markAgentCredentialRevoked(env.SANDBOX_DB, row.user_id, agent, now) + // Codex refreshes its lease (which marks the login dead itself); everyone else confirms with one + // auth check before revoking, so a transient 401 never signs the person out everywhere. + const handled: Promise = provider.onUnauthorized + ? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil })).then(() => true) + : keyring + .then((ring) => + handleAccountUnauthorized({ + db: env.SANDBOX_DB, + keyring: ring, + provider, + userId: row.user_id, + sandboxId: row.id, + path: request.path, + now, + fetch: deps.egressFetch ?? fetch + }) + ) + .then((outcome) => outcome === 'revoked') // Every running sandbox of this person drops the dead placeholder, not only this one. - const work = handled.then(async () => { + const work = handled.then(async (changed) => { + if (!changed) { + return + } forgetAccountCredential(row.user_id, agent) await refreshOwnersSandboxes(env, deps, row.user_id) }) diff --git a/cloud/apps/sandbox-control/test/agent-accounts.test.ts b/cloud/apps/sandbox-control/test/agent-accounts.test.ts index 6b14debc3ae..7adc26fb5d7 100644 --- a/cloud/apps/sandbox-control/test/agent-accounts.test.ts +++ b/cloud/apps/sandbox-control/test/agent-accounts.test.ts @@ -42,12 +42,19 @@ async function setup(overrides: Record = {}) { const jwks = { keys: [{ ...(await exportJWK(publicKey)), kid: 'v1', alg: 'RS256' }] } const upstream: { url: string; headers: Headers }[] = [] const reply = { status: 200 } + // The control plane's own auth check (count_tokens) before a 401 may revoke; default: the token is dead. + const check = { status: 401, body: JSON.stringify({ type: 'error', error: { type: 'authentication_error' } }), calls: 0 } const h = await createHarness( { SECRETS_KEK: KEK, EGRESS_PROXY_URL: PROXY_URL, SECRETS_EGRESS_PLACEHOLDERS: 'on', AGENT_ACCOUNTS_CLAUDE: 'on', ...overrides }, { egressKeysForIssuer: () => createLocalJWKSet(jwks), credentials: { authorizationFor: async () => null }, egressFetch: async (input, init) => { + if (String(input) === 'https://api.anthropic.com/v1/messages/count_tokens') { + check.calls++ + expect(new Headers(init?.headers).get('authorization')).toBe(`Bearer ${SETUP_TOKEN}`) + return new Response(check.body, { status: check.status }) + } upstream.push({ url: String(input), headers: new Headers(init?.headers) }) return new Response('{}', { status: reply.status }) } @@ -79,7 +86,7 @@ async function setup(overrides: Record = {}) { ) return { status: response.status, body: await response.text() } } - return { h, as, lastEnv, proxied, upstream, reply } + return { h, as, lastEnv, proxied, upstream, reply, check } } describe('Orca-account credentials (D11)', () => { @@ -185,7 +192,8 @@ describe('egress-time Source (D10) and shared workspaces (D13)', () => { const otherSession: string = other.body.sandbox.session.id reply.status = 401 expect((await proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` })).status).toBe(401) - expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked') + // Confirmed by the control plane's own auth check (dead token), then revoked. + await vi.waitFor(async () => expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked')) // Already off the account: no placeholder in new shells, and the proxy refuses with a clear reason. await vi.waitFor(() => expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined()) // The person's other running workspace drops the dead placeholder too. @@ -253,3 +261,76 @@ describe('secrets saved while a sandbox boots', () => { expect(h.vercel.commands.filter((c) => c.command === ENV_COMMAND)).toHaveLength(0) }) }) + +describe('a provider 401 revokes only when confirmed', () => { + async function running() { + const ctx = await setup() + const mia = await ctx.as('mia') + await ctx.h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } }) + const created = await ctx.h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + const id: string = created.body.sandbox.id + const state = async () => (await ctx.h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state + const send = () => ctx.proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` }) + const auditRows = async () => + (await ctx.h.env.SANDBOX_DB.prepare('SELECT action, reason, upstream_path FROM agent_credential_audit').all()).results + return { ...ctx, id, state, send, auditRows } + } + + it('a transient 401 whose auth check passes keeps the account (audited)', async () => { + const { reply, check, send, state, lastEnv, auditRows } = await running() + reply.status = 401 + check.status = 200 + check.body = '{"input_tokens":3}' + expect((await send()).status).toBe(401) + await vi.waitFor(async () => expect(await auditRows()).toEqual([{ action: 'kept', reason: 'check_passed', upstream_path: '/v1/messages' }])) + expect(await state()).toBe('active') + expect(lastEnv(START_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toMatch(/^orca-ph-claude-account-/) + }) + + it('a check that is not a definite auth failure keeps the account', async () => { + const cases: [number, string][] = [ + [429, '{"type":"error","error":{"type":"rate_limit_error"}}'], + [500, 'oops'], + [401, '{"type":"error","error":{"type":"permission_error"}}'] + ] + for (const [status, body] of cases) { + const { reply, check, send, state, auditRows } = await running() + reply.status = 401 + check.status = status + check.body = body + await send() + await vi.waitFor(async () => expect((await auditRows())[0]).toMatchObject({ action: 'kept', reason: 'check_inconclusive' })) + expect(await state()).toBe('active') + } + }) + + it('checks at most once a minute per person: a burst of 401s makes one check', async () => { + const { reply, check, send, auditRows } = await running() + reply.status = 401 + check.status = 200 + check.body = '{}' + await Promise.all([send(), send(), send()]) + await vi.waitFor(async () => expect(await auditRows()).toHaveLength(1)) + await new Promise((resolve) => setTimeout(resolve, 30)) + expect(check.calls).toBe(1) + }) + + it('requests that never carried the real credential never trigger a check', async () => { + const { reply, check, proxied, id, state } = await running() + reply.status = 401 + // Wrong placeholder (another sandbox's) and no placeholder: refused before any swap. + expect((await proxied(id, 'api.anthropic.com', { authorization: 'Bearer orca-ph-claude-account-sbx_other' })).status).toBe(403) + expect((await proxied(id, 'api.anthropic.com', {})).status).toBe(403) + await new Promise((resolve) => setTimeout(resolve, 30)) + expect(check.calls).toBe(0) + expect(await state()).toBe('active') + }) + + it('a confirmed dead token is revoked once and audited', async () => { + const { reply, send, state, auditRows } = await running() + reply.status = 401 + await send() + await vi.waitFor(async () => expect(await state()).toBe('revoked')) + expect(await auditRows()).toEqual([{ action: 'revoked', reason: 'provider_401_confirmed_by_check', upstream_path: '/v1/messages' }]) + }) +})