mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
Merge pull request #324 from nwparker/cloud/w6-revoke-confirm
fix(cloud): confirm a provider 401 before revoking an Orca account (migration 0047)
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
-- A provider 401 on an Orca-account request no longer revokes on its own: the control plane confirms
|
||||
-- with one cheap server-side auth check first (agent-credential-checks.ts). checked_at rate-limits those
|
||||
-- checks per person and agent; every outcome is audited.
|
||||
ALTER TABLE agent_credentials ADD COLUMN checked_at INTEGER;
|
||||
|
||||
CREATE TABLE agent_credential_audit (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
user_id TEXT NOT NULL,
|
||||
agent TEXT NOT NULL,
|
||||
-- revoked | kept | skipped
|
||||
action TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
sandbox_id TEXT,
|
||||
upstream_path TEXT,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX agent_credential_audit_user ON agent_credential_audit (user_id, created_at);
|
||||
@@ -30,6 +30,47 @@ export type AccountProvider = {
|
||||
onUnauthorized?(ctx: { db: D1Database; keyring: Keyring; userId: string; now: number; waitUntil: (work: Promise<unknown>) => void }): Promise<void>
|
||||
/** The real value for this person, or null (none, expired, revoked). */
|
||||
credential(db: D1Database, keyring: Keyring, userId: string, now: number): Promise<string | null>
|
||||
/**
|
||||
* One cheap server-side auth check with the real credential (spends no model tokens), run before a
|
||||
* provider 401 may revoke it (agent-credential-checks.ts). 'invalid' only for a definite auth failure.
|
||||
*/
|
||||
verify?(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'>
|
||||
}
|
||||
|
||||
/** Anthropic's token counter: authenticated like Messages, free, and nothing is generated. */
|
||||
const CLAUDE_CHECK_URL = 'https://api.anthropic.com/v1/messages/count_tokens'
|
||||
const CLAUDE_CHECK_MODEL = 'claude-sonnet-4-5'
|
||||
|
||||
async function verifyClaudeToken(credential: string, fetchImpl: typeof fetch): Promise<'valid' | 'invalid' | 'unknown'> {
|
||||
let response: Response
|
||||
try {
|
||||
response = await fetchImpl(CLAUDE_CHECK_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
authorization: `Bearer ${credential}`,
|
||||
'anthropic-beta': 'oauth-2025-04-20',
|
||||
'anthropic-version': '2023-06-01',
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ model: CLAUDE_CHECK_MODEL, messages: [{ role: 'user', content: 'ok' }] }),
|
||||
signal: AbortSignal.timeout(8_000)
|
||||
})
|
||||
} catch {
|
||||
return 'unknown'
|
||||
}
|
||||
if (response.ok) {
|
||||
return 'valid'
|
||||
}
|
||||
if (response.status !== 401) {
|
||||
// Rate limits, overload, a model name the account can't use, permissions: none of these say the token is dead.
|
||||
return 'unknown'
|
||||
}
|
||||
const body: unknown = await response.json().catch(() => null)
|
||||
const type =
|
||||
body && typeof body === 'object' && 'error' in body && body.error && typeof body.error === 'object' && 'type' in body.error
|
||||
? body.error.type
|
||||
: null
|
||||
return type === 'authentication_error' ? 'invalid' : 'unknown'
|
||||
}
|
||||
|
||||
/** D13: a workspace shared with teammates never uses a personal account. Fails closed: only a row
|
||||
@@ -46,7 +87,8 @@ export const claudeAccount: AccountProvider = {
|
||||
sessionEnv: (row) => ({ CLAUDE_CODE_OAUTH_TOKEN: `orca-ph-claude-account-${row.id}` }),
|
||||
header: 'authorization',
|
||||
format: (credential) => `Bearer ${credential}`,
|
||||
credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now)
|
||||
credential: (db, keyring, userId, now) => readUsableAgentCredential(db, keyring, userId, 'claude', now),
|
||||
verify: verifyClaudeToken
|
||||
}
|
||||
|
||||
export const ACCOUNT_PROVIDERS: AccountProvider[] = [claudeAccount, codexAccount]
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
// A provider 401 on an Orca-account request (the proxy swapped the real credential in) is a signal, not a
|
||||
// verdict: one transient auth error must not sign a person out of every workspace. Before revoking, the
|
||||
// control plane makes one cheap server-side auth check with the real credential (no tokens spent) and
|
||||
// revokes only when that also says the credential is dead. Checks are rate-limited per person and agent
|
||||
// (agent_credentials.checked_at), and every decision is audited (agent_credential_audit).
|
||||
import type { AccountProvider } from './account-providers'
|
||||
import { markAgentCredentialRevoked, readUsableAgentCredential } from './agent-credentials'
|
||||
import type { Keyring } from './secrets-crypto'
|
||||
|
||||
/** At most one auth check (and so one revoke) per person and agent per minute. */
|
||||
export const CREDENTIAL_CHECK_INTERVAL_MS = 60_000
|
||||
|
||||
export type CredentialVerdict = 'valid' | 'invalid' | 'unknown'
|
||||
export type UnauthorizedOutcome = 'revoked' | 'kept' | 'skipped'
|
||||
|
||||
async function audit(
|
||||
db: D1Database,
|
||||
entry: { userId: string; agent: string; action: 'revoked' | 'kept'; reason: string; sandboxId: string; path: string; now: number }
|
||||
): Promise<void> {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO agent_credential_audit (user_id, agent, action, reason, sandbox_id, upstream_path, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.bind(entry.userId, entry.agent, entry.action, entry.reason, entry.sandboxId, entry.path.split('?')[0]!.slice(0, 200), entry.now)
|
||||
.run()
|
||||
console.log(JSON.stringify({ event: `agent_credential_${entry.action}`, agent: entry.agent, by: 'egress_401', reason: entry.reason, sandboxId: entry.sandboxId }))
|
||||
}
|
||||
|
||||
export async function handleAccountUnauthorized(ctx: {
|
||||
db: D1Database
|
||||
keyring: Keyring
|
||||
provider: Pick<AccountProvider, 'agent' | 'verify'>
|
||||
userId: string
|
||||
sandboxId: string
|
||||
/** The upstream path that answered 401 (audit only; no query string). */
|
||||
path: string
|
||||
now: number
|
||||
fetch: typeof fetch
|
||||
}): Promise<UnauthorizedOutcome> {
|
||||
const { db, provider, userId, now } = ctx
|
||||
const agent = provider.agent
|
||||
// Claim the check slot: a burst of parallel 401s, or a flapping provider, makes one check a minute.
|
||||
const claimed = await db
|
||||
.prepare(
|
||||
`UPDATE agent_credentials SET checked_at = ?
|
||||
WHERE user_id = ? AND agent = ? AND state = 'active' AND (checked_at IS NULL OR checked_at <= ?)`
|
||||
)
|
||||
.bind(now, userId, agent, now - CREDENTIAL_CHECK_INTERVAL_MS)
|
||||
.run()
|
||||
if (!claimed.meta.changes) {
|
||||
console.log(JSON.stringify({ event: 'agent_credential_401_skipped', agent, sandboxId: ctx.sandboxId }))
|
||||
return 'skipped'
|
||||
}
|
||||
const real = await readUsableAgentCredential(db, ctx.keyring, userId, agent, now)
|
||||
if (!real) {
|
||||
return 'skipped'
|
||||
}
|
||||
const verdict: CredentialVerdict = provider.verify ? await provider.verify(real, ctx.fetch) : 'unknown'
|
||||
const base = { userId, agent, sandboxId: ctx.sandboxId, path: ctx.path, now }
|
||||
if (verdict !== 'invalid') {
|
||||
await audit(db, { ...base, action: 'kept', reason: verdict === 'valid' ? 'check_passed' : 'check_inconclusive' })
|
||||
return 'kept'
|
||||
}
|
||||
await markAgentCredentialRevoked(db, userId, agent, now)
|
||||
await audit(db, { ...base, action: 'revoked', reason: 'provider_401_confirmed_by_check' })
|
||||
return 'revoked'
|
||||
}
|
||||
@@ -40,7 +40,7 @@ export type ProxiedRoute = {
|
||||
/** Extra headers set with the credential (e.g. an account id the client can't know). */
|
||||
extraHeaders?: () => Record<string, string>
|
||||
/** Sees the provider's status (e.g. a 401 revokes an account credential). Never sees the body. */
|
||||
onUpstreamStatus?: (status: number) => void
|
||||
onUpstreamStatus?: (status: number, request: { method: string; path: string }) => void
|
||||
/** Narrows where this credential may be used (member tokens: the workspace repository only). */
|
||||
pathAllowed?: (method: string, path: string) => boolean
|
||||
/** The reply must be a JSON object; only these top-level fields pass (no refresh token for the VM). */
|
||||
@@ -167,7 +167,7 @@ export async function handleEgressProxy(request: Request, deps: EgressProxyDeps)
|
||||
console.log(
|
||||
JSON.stringify({ event: 'egress_swap', sandboxId: row.id, host, credential: route.label ?? route.header, status: upstream.status })
|
||||
)
|
||||
route.onUpstreamStatus?.(upstream.status)
|
||||
route.onUpstreamStatus?.(upstream.status, { method: request.method, path })
|
||||
if (route.keepResponseFields) {
|
||||
return onlyFields(upstream, route.keepResponseFields)
|
||||
}
|
||||
|
||||
@@ -114,6 +114,7 @@ import { egressAccountRoutes, egressSecretHosts, placeholderValues, secretPlaceh
|
||||
import { SecretError, handleSecretsRoute, keyringFor, type SecretChange } from './secrets'
|
||||
import { refreshAfterSecretChange, refreshSandboxEnv } from './env-refresh'
|
||||
import { accountProvider, isSharedWorkspace } from './account-providers'
|
||||
import { handleAccountUnauthorized } from './agent-credential-checks'
|
||||
import { codexBrokerTick, handleCodexBrokerRoute, type BrokerDeps } from './codex-broker'
|
||||
import { refreshCodexLeases } from './codex-credentials'
|
||||
import {
|
||||
@@ -123,7 +124,6 @@ import {
|
||||
deleteAgentCredential,
|
||||
providerEnabled,
|
||||
putClaudeCredential,
|
||||
markAgentCredentialRevoked,
|
||||
readUsableAgentCredential
|
||||
} from './agent-credentials'
|
||||
import {
|
||||
@@ -371,16 +371,35 @@ function egressRoutes(env: Env, deps: Deps): (row: SandboxRow, host: string) =>
|
||||
return provider.format(real)
|
||||
},
|
||||
...(provider.extraHeaders ? { extraHeaders: () => (real ? provider.extraHeaders!(real) : {}) } : {}),
|
||||
onUpstreamStatus: (status) => {
|
||||
// Reached only after the real credential was swapped in and sent (denied or unswapped requests
|
||||
// never get here).
|
||||
onUpstreamStatus: (status, request) => {
|
||||
if (status === 401) {
|
||||
// The provider no longer accepts it (revoked at the source): stop using it until renewed.
|
||||
const now = (deps.now ?? Date.now)()
|
||||
const waitUntil = (work: Promise<unknown>) => (deps.waitUntil ? deps.waitUntil(work.catch(() => undefined)) : void work.catch(() => undefined))
|
||||
const handled = provider.onUnauthorized
|
||||
? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil }))
|
||||
: markAgentCredentialRevoked(env.SANDBOX_DB, row.user_id, agent, now)
|
||||
// Codex refreshes its lease (which marks the login dead itself); everyone else confirms with one
|
||||
// auth check before revoking, so a transient 401 never signs the person out everywhere.
|
||||
const handled: Promise<boolean> = provider.onUnauthorized
|
||||
? keyring.then((ring) => provider.onUnauthorized!({ db: env.SANDBOX_DB, keyring: ring, userId: row.user_id, now, waitUntil })).then(() => true)
|
||||
: keyring
|
||||
.then((ring) =>
|
||||
handleAccountUnauthorized({
|
||||
db: env.SANDBOX_DB,
|
||||
keyring: ring,
|
||||
provider,
|
||||
userId: row.user_id,
|
||||
sandboxId: row.id,
|
||||
path: request.path,
|
||||
now,
|
||||
fetch: deps.egressFetch ?? fetch
|
||||
})
|
||||
)
|
||||
.then((outcome) => outcome === 'revoked')
|
||||
// Every running sandbox of this person drops the dead placeholder, not only this one.
|
||||
const work = handled.then(async () => {
|
||||
const work = handled.then(async (changed) => {
|
||||
if (!changed) {
|
||||
return
|
||||
}
|
||||
forgetAccountCredential(row.user_id, agent)
|
||||
await refreshOwnersSandboxes(env, deps, row.user_id)
|
||||
})
|
||||
|
||||
@@ -42,12 +42,19 @@ async function setup(overrides: Record<string, string> = {}) {
|
||||
const jwks = { keys: [{ ...(await exportJWK(publicKey)), kid: 'v1', alg: 'RS256' }] }
|
||||
const upstream: { url: string; headers: Headers }[] = []
|
||||
const reply = { status: 200 }
|
||||
// The control plane's own auth check (count_tokens) before a 401 may revoke; default: the token is dead.
|
||||
const check = { status: 401, body: JSON.stringify({ type: 'error', error: { type: 'authentication_error' } }), calls: 0 }
|
||||
const h = await createHarness(
|
||||
{ SECRETS_KEK: KEK, EGRESS_PROXY_URL: PROXY_URL, SECRETS_EGRESS_PLACEHOLDERS: 'on', AGENT_ACCOUNTS_CLAUDE: 'on', ...overrides },
|
||||
{
|
||||
egressKeysForIssuer: () => createLocalJWKSet(jwks),
|
||||
credentials: { authorizationFor: async () => null },
|
||||
egressFetch: async (input, init) => {
|
||||
if (String(input) === 'https://api.anthropic.com/v1/messages/count_tokens') {
|
||||
check.calls++
|
||||
expect(new Headers(init?.headers).get('authorization')).toBe(`Bearer ${SETUP_TOKEN}`)
|
||||
return new Response(check.body, { status: check.status })
|
||||
}
|
||||
upstream.push({ url: String(input), headers: new Headers(init?.headers) })
|
||||
return new Response('{}', { status: reply.status })
|
||||
}
|
||||
@@ -79,7 +86,7 @@ async function setup(overrides: Record<string, string> = {}) {
|
||||
)
|
||||
return { status: response.status, body: await response.text() }
|
||||
}
|
||||
return { h, as, lastEnv, proxied, upstream, reply }
|
||||
return { h, as, lastEnv, proxied, upstream, reply, check }
|
||||
}
|
||||
|
||||
describe('Orca-account credentials (D11)', () => {
|
||||
@@ -185,7 +192,8 @@ describe('egress-time Source (D10) and shared workspaces (D13)', () => {
|
||||
const otherSession: string = other.body.sandbox.session.id
|
||||
reply.status = 401
|
||||
expect((await proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` })).status).toBe(401)
|
||||
expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked')
|
||||
// Confirmed by the control plane's own auth check (dead token), then revoked.
|
||||
await vi.waitFor(async () => expect((await h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state).toBe('revoked'))
|
||||
// Already off the account: no placeholder in new shells, and the proxy refuses with a clear reason.
|
||||
await vi.waitFor(() => expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined())
|
||||
// The person's other running workspace drops the dead placeholder too.
|
||||
@@ -253,3 +261,76 @@ describe('secrets saved while a sandbox boots', () => {
|
||||
expect(h.vercel.commands.filter((c) => c.command === ENV_COMMAND)).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('a provider 401 revokes only when confirmed', () => {
|
||||
async function running() {
|
||||
const ctx = await setup()
|
||||
const mia = await ctx.as('mia')
|
||||
await ctx.h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
const created = await ctx.h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const id: string = created.body.sandbox.id
|
||||
const state = async () => (await ctx.h.call('GET', '/v1/agent-credentials', { bearer: mia })).body.claude.state
|
||||
const send = () => ctx.proxied(id, 'api.anthropic.com', { authorization: `Bearer orca-ph-claude-account-${id}` })
|
||||
const auditRows = async () =>
|
||||
(await ctx.h.env.SANDBOX_DB.prepare('SELECT action, reason, upstream_path FROM agent_credential_audit').all()).results
|
||||
return { ...ctx, id, state, send, auditRows }
|
||||
}
|
||||
|
||||
it('a transient 401 whose auth check passes keeps the account (audited)', async () => {
|
||||
const { reply, check, send, state, lastEnv, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = 200
|
||||
check.body = '{"input_tokens":3}'
|
||||
expect((await send()).status).toBe(401)
|
||||
await vi.waitFor(async () => expect(await auditRows()).toEqual([{ action: 'kept', reason: 'check_passed', upstream_path: '/v1/messages' }]))
|
||||
expect(await state()).toBe('active')
|
||||
expect(lastEnv(START_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toMatch(/^orca-ph-claude-account-/)
|
||||
})
|
||||
|
||||
it('a check that is not a definite auth failure keeps the account', async () => {
|
||||
const cases: [number, string][] = [
|
||||
[429, '{"type":"error","error":{"type":"rate_limit_error"}}'],
|
||||
[500, 'oops'],
|
||||
[401, '{"type":"error","error":{"type":"permission_error"}}']
|
||||
]
|
||||
for (const [status, body] of cases) {
|
||||
const { reply, check, send, state, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = status
|
||||
check.body = body
|
||||
await send()
|
||||
await vi.waitFor(async () => expect((await auditRows())[0]).toMatchObject({ action: 'kept', reason: 'check_inconclusive' }))
|
||||
expect(await state()).toBe('active')
|
||||
}
|
||||
})
|
||||
|
||||
it('checks at most once a minute per person: a burst of 401s makes one check', async () => {
|
||||
const { reply, check, send, auditRows } = await running()
|
||||
reply.status = 401
|
||||
check.status = 200
|
||||
check.body = '{}'
|
||||
await Promise.all([send(), send(), send()])
|
||||
await vi.waitFor(async () => expect(await auditRows()).toHaveLength(1))
|
||||
await new Promise((resolve) => setTimeout(resolve, 30))
|
||||
expect(check.calls).toBe(1)
|
||||
})
|
||||
|
||||
it('requests that never carried the real credential never trigger a check', async () => {
|
||||
const { reply, check, proxied, id, state } = await running()
|
||||
reply.status = 401
|
||||
// Wrong placeholder (another sandbox's) and no placeholder: refused before any swap.
|
||||
expect((await proxied(id, 'api.anthropic.com', { authorization: 'Bearer orca-ph-claude-account-sbx_other' })).status).toBe(403)
|
||||
expect((await proxied(id, 'api.anthropic.com', {})).status).toBe(403)
|
||||
await new Promise((resolve) => setTimeout(resolve, 30))
|
||||
expect(check.calls).toBe(0)
|
||||
expect(await state()).toBe('active')
|
||||
})
|
||||
|
||||
it('a confirmed dead token is revoked once and audited', async () => {
|
||||
const { reply, send, state, auditRows } = await running()
|
||||
reply.status = 401
|
||||
await send()
|
||||
await vi.waitFor(async () => expect(await state()).toBe('revoked'))
|
||||
expect(await auditRows()).toEqual([{ action: 'revoked', reason: 'provider_401_confirmed_by_check', upstream_path: '/v1/messages' }])
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user