mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 11:54:46 +00:00
fix(cloud): confirm a provider 401 with one auth check before revoking an Orca account
A single 401 on an Orca-account request no longer revokes the person's Claude account everywhere. The hook only fires after the real credential was swapped in and sent. The control plane then makes one free auth check with the real token (Anthropic count_tokens, oauth beta) and revokes only on a definite authentication_error; anything else (passes, 429, 5xx, other 401 types, network) keeps the account. Checks are rate-limited to one per person and agent per minute (agent_credentials.checked_at), and every keep/revoke is audited (agent_credential_audit, migration 0047). Codex keeps its lease-refresh path.
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
-- A provider 401 on an Orca-account request no longer revokes on its own: the control plane confirms
|
||||
-- with one cheap server-side auth check first (agent-credential-checks.ts). checked_at rate-limits those
|
||||
-- checks per person and agent; every outcome is audited.
|
||||
ALTER TABLE agent_credentials ADD COLUMN checked_at INTEGER;
|
||||
|
||||
CREATE TABLE agent_credential_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
agent TEXT NOT NULL,
|
||||
-- revoked | kept | skipped
|
||||
action TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
sandbox_id TEXT,
|
||||
upstream_path TEXT,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX agent_credential_audit_user ON agent_credential_audit (user_id, created_at);
|
||||
@@ -30,6 +30,47 @@ export type AccountProvider = {
|
||||
onUnauthorized?(ctx: { db: D1Database; keyring: Keyring; userId: string; now: number; waitUntil: (work: Promise<unknown>) => void }): Promise<void>
|
||||
/** The real value for this person, or null (none, expired, revoked). */
|
||||
credential(db: D1Database, keyring: Keyring, userId: string, now: number): Promise<string | null>
|
||||
/**
|
||||
* One cheap server-side auth check with the real credential (spends no model tokens), run before a
|
||||
* provider 401 may revoke it (agent-credential-checks.ts). 'invalid' only for a definite auth failure.
|
||||
*/
|
||||
verify?(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'>
|
||||
}
|
||||
|
||||
/** Anthropic's token counter: authenticated like Messages, free, and nothing is generated. */
|
||||
const CLAUDE_CHECK_URL = 'https://api.anthropic.com/v1/messages/count_tokens'
|
||||
const CLAUDE_CHECK_MODEL = 'claude-sonnet-4-5'
|
||||
|
||||
async function verifyClaudeToken(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'> {
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchImpl(CLAUDE_CHECK_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${credential}`,
|
||||
'anthropic-beta': 'oauth-2025-04-20',
|
||||
'anthropic-version': '2023-06-01',
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ model: CLAUDE_CHECK_MODEL, messages: [{ role: 'user', content: 'ok' }] }),
|
||||
signal: AbortSignal.timeout(8_000)
|
||||
})
|
||||
} catch {
|
||||
return 'unknown'
|
||||
}
|
||||
if (response.ok) {
|
||||
return 'valid'
|
||||
}
|
||||
if (response.status !== 401) {
|
||||
// Rate limits, overload, a model name the account can't use, permissions: none of these say the token is dead.
|
||||
return 'unknown'
|
||||
}
|
||||
const body: unknown = await response.json().catch(() => null)
|
||||
const type =
|
||||
body && typeof body === 'object' && 'error' in body && body.error && typeof body.error === 'object' && 'type' in body.error
|
||||
? body.error.type
|
||||
: null
|
||||
return type === 'authentication_error' ? 'invalid' : 'unknown'
|
||||
}
|
||||
|
||||
/** D13: a workspace shared with teammates never uses a personal account. Fails closed: only a row
|
||||
@@ -46,7 +87,8 @@ export const claudeAccount: AccountProvider = {
|
||||
sessionEnv: (row) => ({ CLAUDE_CODE_OAUTH_TOKEN: `orca-ph-claude-account-${row.id}` }),
|
||||
header: 'authorization',
|
||||
format: (credential) => `Bearer ${credential}`,
|
||||
credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now)
|
||||
credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now),
|
||||
verify: verifyClaudeToken
|
||||
}
|
||||
|
||||
export const ACCOUNT_PROVIDERS: AccountProvider[] = [claudeAccount, codexAccount]
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
// A provider 401 on an Orca-account request (the proxy swapped the real credential in) is a signal, not a
|
||||
// verdict: one transient auth error must not sign a person out of every workspace. Before revoking, the
|
||||
// control plane makes one cheap server-side auth check with the real credential (no tokens spent) and
|
||||
// revokes only when that also says the credential is dead. Checks are rate-limited per person and agent
|
||||
// (agent_credentials.checked_at), and every decision is audited (agent_credential_audit).
|
||||
import type { AccountProvider } from './account-providers'
|
||||
import { markAgentCredentialRevoked, readUsableAgentCredential } from './agent-credentials'
|
||||
import type { Keyring } from './secrets-crypto'
|
||||
|
||||
/** At most one auth check (and so one revoke) per person and agent per minute. */
|
||||
export const CREDENTIAL_CHECK_INTERVAL_MS = 60_000
|
||||
|
||||
export type CredentialVerdict = 'valid' | 'invalid' | 'unknown'
|
||||
export type UnauthorizedOutcome = 'revoked' | 'kept' | 'skipped'
|
||||
|
||||
async function audit(
|
||||
db: D1Database,
|
||||
entry: { userId: string; agent: string; action: 'revoked' | 'kept'; reason: string; sandboxId: string; path: string; now: number }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO agent_credential_audit (user_id, agent, action, reason, sandbox_id, upstream_path, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.bind(entry.userId, entry.agent, entry.action, entry.reason, entry.sandboxId, entry.path.split('?')[0]!.slice(0, 200), entry.now)
|
||||
.run()
|
||||
console.log(JSON.stringify({ event: `agent_credential_${entry.action}`, agent: entry.agent, by: 'egress_401', reason: entry.reason, sandboxId: entry.sandboxId }))
|
||||
}
|
||||
|
||||
export async function handleAccountUnauthorized(ctx: {
|
||||
db: D1Database
|
||||
keyring: Keyring
|
||||
provider: Pick<AccountProvider, 'agent' | 'verify'>
|
||||
userId: string
|
||||
sandboxId: string
|
||||
/** The upstream path that answered 401 (audit only; no query string). */
|
||||
path: string
|
||||
now: number
|
||||
fetch: typeof fetch
|
||||
}): Promise<UnauthorizedOutcome> {
|
||||
const { db, provider, userId, now } = ctx
|
||||
const agent = provider.agent
|
||||
// Claim the check slot: a burst of parallel 401s, or a flapping provider, makes one check a minute.
|
||||
const claimed = await db
|
||||
.prepare(
|
||||
`UPDATE agent_credentials SET checked_at = ?
|
||||
WHERE user_id = ? AND agent = ? AND state = 'active' AND (checked_at IS NULL OR checked_at <= ?)`
|
||||
)
|
||||
.bind(now, userId, agent, now - CREDENTIAL_CHECK_INTERVAL_MS)
|
||||
.run()
|
||||
if (!claimed.meta.changes) {
|
||||
console.log(JSON.stringify({ event: 'agent_credential_401_skipped', agent, sandboxId: ctx.sandboxId }))
|
||||
return 'skipped'
|
||||
}
|
||||
const real = await readUsableAgentCredential(db, ctx.keyring, userId, agent, now)
|
||||
if (!real) {
|
||||
return 'skipped'
|
||||
}
|
||||
const verdict: CredentialVerdict = provider.verify ? await provider.verify(real, ctx.fetch) : 'unknown'
|
||||
const base = { userId, agent, sandboxId: ctx.sandboxId, path: ctx.path, now }
|
||||
if (verdict !== 'invalid') {
|
||||
await audit(db, { ...base, action: 'kept', reason: verdict === 'valid' ? 'check_passed' : 'check_inconclusive' })
|
||||
return 'kept'
|
||||
}
|
||||
await markAgentCredentialRevoked(db, userId, agent, now)
|
||||
await audit(db, { ...base, action: 'revoked', reason: 'provider_401_confirmed_by_check' })
|
||||
return 'revoked'
|
||||
}
|
||||
@@ -40,7 +40,7 @@ export type ProxiedRoute = {
|
||||
/** Extra headers set with the credential (e.g. an account id the client can't know). */
|
||||
extraHeaders?: () => Record<string, string>
|
||||
/** Sees the provider's status (e.g. a 401 revokes an account credential). Never sees the body. */
|
||||
onUpstreamStatus?: (status: number) => void
|
||||
onUpstreamStatus?: (status: number, request: { method: string; path: string }) => void
|
||||
/** Narrows where this credential may be used (member tokens: the workspace repository only). */
|
||||
pathAllowed?: (method: string, path: string) => boolean
|
||||
/** The reply must be a JSON object; only these top-level fields pass (no refresh token for the VM). */
|
||||
@@ -167,7 +167,7 @@ export async function handleEgressProxy(request: Request, deps: EgressProxyDeps)
|
||||
console.log(
|
||||
JSON.stringify({ event: 'egress_swap', sandboxId: row.id, host, credential: route.label ?? route.header, status: upstream.status })
|
||||
)
|
||||
route.onUpstreamStatus?.(upstream.status)
|
||||
route.onUpstreamStatus?.(upstream.status, { method: request.method, path })
|
||||
if (route.keepResponseFields) {
|
||||
return onlyFields(upstream, route.keepResponseFields)
|
||||
}
|
||||
|
||||
@@ -114,6 +114,7 @@ import { egressAccountRoutes, egressSecretHosts, placeholderValues, secretPlaceh
|
||||
import { SecretError, handleSecretsRoute, keyringFor, type SecretChange } from './secrets'
|
||||
import { refreshAfterSecretChange, refreshSandboxEnv } from './env-refresh'
|
||||
import { accountProvider, isSharedWorkspace } from './account-providers'
|
||||
import { handleAccountUnauthorized } from './agent-credential-checks'
|
||||
import { codexBrokerTick, handleCodexBrokerRoute, type BrokerDeps } from './codex-broker'
|
||||
import { refreshCodexLeases } from './codex-credentials'
|
||||
import {
|
||||
@@ -123,7 +124,6 @@ import {
|
||||
deleteAgentCredential,
|
||||
providerEnabled,
|
||||
putClaudeCredential,
|
||||
markAgentCredentialRevoked,
|
||||
readUsableAgentCredential
|
||||
} from './agent-credentials'
|
||||
import {
|
||||
@@ -371,16 +371,35 @@ function egressRoutes(env: Env, deps: Deps): (row: SandboxRow, host: string) =>
|
||||
return provider.format(real)
|
||||
},
|
||||
...(provider.extraHeaders ? { extraHeaders: () => (real ? provider.extraHeaders!(real) : {}) } : {}),
|
||||
onUpstreamStatus: (status) => {
|
||||
// Reached only after the real credential was swapped in and sent (denied or unswapped requests
|
||||
// never get here).
|
||||
onUpstreamStatus: (status, request) => {
|
||||
if (status === 401) {
|
||||
// The provider no longer accepts it (revoked at the source): stop using it until renewed.
|
||||
const now = (deps.now ?? Date.now)()
|
||||
const waitUntil = (work: Promise<unknown>) => (deps.waitUntil ? deps.waitUntil(work.catch(() => undefined)) : void work.catch(() => undefined))
|
||||
const handled = provider.onUnauthorized
|
||||
? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil }))
|
||||
: markAgentCredentialRevoked(env.SANDBOX_DB, row.user_id, agent, now)
|
||||
// Codex refreshes its lease (which marks the login dead itself); everyone else confirms with one
|
||||
// auth check before revoking, so a transient 401 never signs the person out everywhere.
|
||||
const handled: Promise<boolean> = provider.onUnauthorized
|
||||
? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil })).then(() => true)
|
||||
: keyring
|
||||
.then((ring) =>
|
||||
handleAccountUnauthorized({
|
||||
db: env.SANDBOX_DB,
|
||||
keyring: ring,
|
||||
provider,
|
||||
userId: row.user_id,
|
||||
sandboxId: row.id,
|
||||
path: request.path,
|
||||
now,
|
||||
fetch: deps.egressFetch ?? fetch
|
||||
})
|
||||
)
|
||||
.then((outcome) => outcome === 'revoked')
|
||||
// Every running sandbox of this person drops the dead placeholder, not only this one.
|
||||
const work = handled.then(async () => {
|
||||
const work = handled.then(async (changed) => {
|
||||
if (!changed) {
|
||||
return
|
||||
}
|
||||
forgetAccountCredential(row.user_id, agent)
|
||||
await refreshOwnersSandboxes(env, deps, row.user_id)
|
||||
})
|
||||
|
||||
@@ -42,12 +42,19 @@ async function setup(overrides: Record<string, string> = {}) {
|
||||
const jwks = { keys: [{ ...(await exportJWK(publicKey)), kid: 'v1', alg: 'RS256' }] }
|
||||
const upstream: { url: string; headers: Headers }[] = []
|
||||
const reply = { status: 200 }
|
||||
// The control plane's own auth check (count_tokens) before a 401 may revoke; default: the token is dead.
|
||||
const check = { status: 401, body: JSON.stringify({ type: 'error', error: { type: 'authentication_error' } }), calls: 0 }
|
||||
const h = await createHarness(
|
||||
{ SECRETS_KEK: KEK, EGRESS_PROXY_URL: PROXY_URL, SECRETS_EGRESS_PLACEHOLDERS: 'on', AGENT_ACCOUNTS_CLAUDE: 'on', ...overrides },
|
||||
{
|
||||
egressKeysForIssuer: () => createLocalJWKSet(jwks),
|
||||
credentials: { authorizationFor: async () => null },
|
||||
egressFetch: async (input, init) => {
|
||||
if (String(input) === 'https://api.anthropic.com/v1/messages/count_tokens') {
|
||||
check.calls++
|
||||
expect(new Headers(init?.headers).get('authorization')).toBe(`Bearer ${SETUP_TOKEN}`)
|
||||
return new Response(check.body, { status: check.status })
|
||||
}
|
||||
upstream.push({ url: String(input), headers: new Headers(init?.headers) })
|
||||
return new Response('{}', { status: reply.status })
|
||||
}
|
||||
@@ -79,7 +86,7 @@ async function setup(overrides: Record<string, string> = {}) {
|
||||
)
|
||||
return { status: response.status, body: await response.text() }
|
||||
}
|
||||
return { h, as, lastEnv, proxied, upstream, reply }
|
||||
return { h, as, lastEnv, proxied, upstream, reply, check }
|
||||
}
|
||||
|
||||
describe('Orca-account credentials (D11)', () => {
|
||||
@@ -185,7 +192,8 @@ describe('egress-time Source (D10) and shared workspaces (D13)', () => {
|
||||
const otherSession: string = other.body.sandbox.session.id
|
||||
reply.status = 401
|
||||
expect((await proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` })).status).toBe(401)
|
||||
expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked')
|
||||
// Confirmed by the control plane's own auth check (dead token), then revoked.
|
||||
await vi.waitFor(async () => expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked'))
|
||||
// Already off the account: no placeholder in new shells, and the proxy refuses with a clear reason.
|
||||
await vi.waitFor(() => expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined())
|
||||
// The person's other running workspace drops the dead placeholder too.
|
||||
@@ -253,3 +261,76 @@ describe('secrets saved while a sandbox boots', () => {
|
||||
expect(h.vercel.commands.filter((c) => c.command === ENV_COMMAND)).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('a provider 401 revokes only when confirmed', () => {
|
||||
async function running() {
|
||||
const ctx = await setup()
|
||||
const mia = await ctx.as('mia')
|
||||
await ctx.h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
const created = await ctx.h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const id: string = created.body.sandbox.id
|
||||
const state = async () => (await ctx.h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state
|
||||
const send = () => ctx.proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` })
|
||||
const auditRows = async () =>
|
||||
(await ctx.h.env.SANDBOX_DB.prepare('SELECT action, reason, upstream_path FROM agent_credential_audit').all()).results
|
||||
return { ...ctx, id, state, send, auditRows }
|
||||
}
|
||||
|
||||
it('a transient 401 whose auth check passes keeps the account (audited)', async () => {
|
||||
const { reply, check, send, state, lastEnv, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = 200
|
||||
check.body = '{"input_tokens":3}'
|
||||
expect((await send()).status).toBe(401)
|
||||
await vi.waitFor(async () => expect(await auditRows()).toEqual([{ action: 'kept', reason: 'check_passed', upstream_path: '/v1/messages' }]))
|
||||
expect(await state()).toBe('active')
|
||||
expect(lastEnv(START_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toMatch(/^orca-ph-claude-account-/)
|
||||
})
|
||||
|
||||
it('a check that is not a definite auth failure keeps the account', async () => {
|
||||
const cases: [number, string][] = [
|
||||
[429, '{"type":"error","error":{"type":"rate_limit_error"}}'],
|
||||
[500, 'oops'],
|
||||
[401, '{"type":"error","error":{"type":"permission_error"}}']
|
||||
]
|
||||
for (const [status, body] of cases) {
|
||||
const { reply, check, send, state, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = status
|
||||
check.body = body
|
||||
await send()
|
||||
await vi.waitFor(async () => expect((await auditRows())[0]).toMatchObject({ action: 'kept', reason: 'check_inconclusive' }))
|
||||
expect(await state()).toBe('active')
|
||||
}
|
||||
})
|
||||
|
||||
it('checks at most once a minute per person: a burst of 401s makes one check', async () => {
|
||||
const { reply, check, send, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = 200
|
||||
check.body = '{}'
|
||||
await Promise.all([send(), send(), send()])
|
||||
await vi.waitFor(async () => expect(await auditRows()).toHaveLength(1))
|
||||
await new Promise((resolve) => setTimeout(resolve, 30))
|
||||
expect(check.calls).toBe(1)
|
||||
})
|
||||
|
||||
it('requests that never carried the real credential never trigger a check', async () => {
|
||||
const { reply, check, proxied, id, state } = await running()
|
||||
reply.status = 401
|
||||
// Wrong placeholder (another sandbox's) and no placeholder: refused before any swap.
|
||||
expect((await proxied(id, 'api.anthropic.com', { authorization: 'Bearer orca-ph-claude-account-sbx_other' })).status).toBe(403)
|
||||
expect((await proxied(id, 'api.anthropic.com', {})).status).toBe(403)
|
||||
await new Promise((resolve) => setTimeout(resolve, 30))
|
||||
expect(check.calls).toBe(0)
|
||||
expect(await state()).toBe('active')
|
||||
})
|
||||
|
||||
it('a confirmed dead token is revoked once and audited', async () => {
|
||||
const { reply, send, state, auditRows } = await running()
|
||||
reply.status = 401
|
||||
await send()
|
||||
await vi.waitFor(async () => expect(await state()).toBe('revoked'))
|
||||
expect(await auditRows()).toEqual([{ action: 'revoked', reason: 'provider_401_confirmed_by_check', upstream_path: '/v1/messages' }])
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user