diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts index ad0e750b21d..60c6f010c30 100644 --- a/cloud/apps/sandbox-control/src/env.ts +++ b/cloud/apps/sandbox-control/src/env.ts @@ -92,11 +92,19 @@ export type Env = { // Secret store (src/secrets.ts). SECRETS_KEK is a base64 32-byte AES key (secret); SECRETS_KEK_ID // names it (default `k1`). After a rotation, SECRETS_KEK_RETIRED (secret, JSON {"":""}) - // keeps older rows readable. Without SECRETS_KEK the /v1/secrets routes answer 503 and sessions - // start without injected secrets. + // keeps older rows readable until POST /v1/internal/secrets/rewrap has moved them to the active + // KEK; GET /v1/internal/secrets/kek says when a retired id is no longer referenced and can be + // dropped (src/secrets-rewrap.ts). Without SECRETS_KEK the /v1/secrets routes answer 503 and + // sessions start without injected secrets. SECRETS_KEK?: string SECRETS_KEK_ID?: string SECRETS_KEK_RETIRED?: string + /** + * `v1` (default) | `v2`: the row format new secrets are sealed in (src/secrets-crypto.ts). With + * v2 a KEK rotation re-wraps data keys only. Set it only once every Worker version that reads this + * database (previews and the rollback target included) can read v2: older code skips v2 rows. + */ + SECRETS_SEAL_FORMAT?: string /** * Model API keys (Anthropic, OpenAI, AI Gateway, custom provider) reach the VM as placeholders and * the egress proxy swaps the real key in (forwardURL path). Unset or `on` (default): every account; diff --git a/cloud/apps/sandbox-control/src/github-user-tokens.ts b/cloud/apps/sandbox-control/src/github-user-tokens.ts index 2b25e4e8a38..e50f91749b3 100644 --- a/cloud/apps/sandbox-control/src/github-user-tokens.ts +++ b/cloud/apps/sandbox-control/src/github-user-tokens.ts @@ -9,7 +9,7 @@ import { z } from 'zod' import type { SandboxCaller } from './auth' import { GitHubConnectError, authorizeUrl, signState, tokenRequest, type GitHubConnectConfig, type TokenSet } from './github-connect' -import type { Keyring, SealedSecret } from './secrets-crypto' +import type { Keyring, SealedSecret, SecretBinding } from './secrets-crypto' const STATE_TTL_MS = 10 * 60_000 /** Refresh a little before GitHub's 8 h expiry so a push never races it. */ @@ -31,11 +31,13 @@ export type GitHubUserTokenRow = { export type { TokenSet } -const binding = (row: { account_id: string; user_id: string }) => ({ +const binding = (row: { account_id: string; user_id: string }): SecretBinding => ({ scope: 'github_user', ownerId: `${row.account_id}#${row.user_id}`, name: 'oauth' }) +/** The AAD identity of a github_user_tokens row (the KEK re-wrap sweep opens rows by it). */ +export const githubUserTokenBinding = binding export async function createUserConnectState( db: D1Database, diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 869334bf4a6..0310886188a 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -1307,7 +1307,8 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise readJson(request) + readJson: () => readJson(request), + keyring: () => keyringFor(env) })) ?? fail('not_found', 404) ) } diff --git a/cloud/apps/sandbox-control/src/operator.ts b/cloud/apps/sandbox-control/src/operator.ts index a709679b12a..380e36822c4 100644 --- a/cloud/apps/sandbox-control/src/operator.ts +++ b/cloud/apps/sandbox-control/src/operator.ts @@ -9,6 +9,8 @@ // POST /v1/internal/members/removed {accountId, userId} revoke a departed member's keys, phones and workspace grants, disable their routines (auth calls this) // GET /v1/internal/fair-use?days=30 per-account CPU utilisation + fair-use flag (U22; internal only, never billed) // POST /v1/internal/session-cpu/sweep fetch CPU for recently stopped sessions now (the cron does this every 5 min) +// GET /v1/internal/secrets/kek which KEK ids stored secrets reference; which retired ids can be dropped +// POST /v1/internal/secrets/rewrap {limit?, cursor?, dryRun?} move one batch of rows to the active KEK (secrets-rewrap.ts) import { z } from 'zod' import { audit } from './api/audit' import { syncMemberRole } from './api/member-role-sync' @@ -20,6 +22,8 @@ import { guardrails, type LifecycleDeps } from './lifecycle' import { drainAll, housekeeping } from './retention' import { memberChangeTime, recordMemberChange } from './member-changes' import { removeMember } from './member-removal' +import type { Keyring } from './secrets-crypto' +import { kekStatus, rewrapBatch, RewrapError, REWRAP_MAX_LIMIT } from './secrets-rewrap' import { fairUseReport, FAIR_USE_WINDOW_DAYS, sweepSessionCpu } from './session-cpu' import { listByStatus } from './store' @@ -38,6 +42,9 @@ const BindingBody = z const ChangedAt = z.number().int().positive().optional() const MemberRemovedBody = z.object({ accountId: AccountId, userId: z.string().min(1).max(128), changedAt: ChangedAt }).strict() const RoleChangedBody = z.object({ accountId: AccountId, userId: z.string().min(1).max(128), role: z.enum(['owner', 'admin', 'member']), changedAt: ChangedAt }).strict() +const RewrapBody = z + .object({ limit: z.number().int().min(1).max(REWRAP_MAX_LIMIT).optional(), cursor: z.string().max(2048).nullable().optional(), dryRun: z.boolean().optional() }) + .strict() const UnbindBody = z.object({ installationId: z.number().int().positive(), accountId: AccountId.optional() }).strict() async function digest(value: string): Promise { @@ -55,7 +62,13 @@ export async function isOperator(request: Request, operatorToken: string | undef export async function handleOperator( request: Request, path: string, - deps: { life: LifecycleDeps; github?: GitHubAppMinter; readJson: () => Promise } + deps: { + life: LifecycleDeps + github?: GitHubAppMinter + readJson: () => Promise + /** The Worker's keyring; null when SECRETS_KEK is not configured. Throws on a malformed key config. */ + keyring?: () => Promise | null + } ): Promise { const { life } = deps const now = life.now() @@ -153,6 +166,35 @@ export async function handleOperator( } case 'POST /v1/internal/session-cpu/sweep': return Response.json({ sweep: await sweepSessionCpu(life.db, life.provider, now) }) + case 'GET /v1/internal/secrets/kek': + case 'POST /v1/internal/secrets/rewrap': { + const noStore = { 'cache-control': 'no-store' } + let keyring: Keyring | null + try { + keyring = (await deps.keyring?.()) ?? null + } catch { + // A malformed SECRETS_KEK* value. The cause is not echoed: it could quote key material. + return Response.json({ error: 'secrets_misconfigured' }, { status: 503, headers: noStore }) + } + if (!keyring) { + return Response.json({ error: 'secrets_unconfigured' }, { status: 503, headers: noStore }) + } + if (request.method === 'GET') { + return Response.json(await kekStatus(life.db, keyring), { headers: noStore }) + } + const body = RewrapBody.safeParse(await deps.readJson()) + if (!body.success) { + return Response.json({ error: 'invalid_request' }, { status: 400, headers: noStore }) + } + try { + return Response.json(await rewrapBatch(life.db, keyring, body.data), { headers: noStore }) + } catch (error) { + if (error instanceof RewrapError) { + return Response.json({ error: 'invalid_request', reason: error.reason }, { status: 400, headers: noStore }) + } + throw error + } + } default: return null } diff --git a/cloud/apps/sandbox-control/src/secrets-crypto.ts b/cloud/apps/sandbox-control/src/secrets-crypto.ts index 8c3c7fdc671..656aebfd81e 100644 --- a/cloud/apps/sandbox-control/src/secrets-crypto.ts +++ b/cloud/apps/sandbox-control/src/secrets-crypto.ts @@ -2,19 +2,46 @@ // the DEK is wrapped by the key-encryption key (KEK), which exists only as a Worker secret. Both // layers authenticate the row's (scope, owner, name) as AAD, so a ciphertext copied onto another // row (another org, another name) fails to open instead of decrypting under the wrong identity. +// +// Two row formats: +// v1 (no prefix): the KEK id is in the AAD of BOTH layers, so moving a row to another KEK means +// decrypting and re-encrypting the value. +// v2 (`v2.` prefix on wrapped_dek and ciphertext): the KEK id is in the DEK layer's AAD only, so a +// KEK rotation re-wraps the 32-byte DEK and never touches (or decrypts) the value. +// Every Worker version that can read the database must understand v2 before SECRETS_SEAL_FORMAT=v2 +// is set: an older version skips v2 rows as unreadable. export type SecretBinding = { scope: string; ownerId: string; name: string } export type SealedSecret = { kekId: string; wrappedDek: string; ciphertext: string } +export type SealFormat = 'v1' | 'v2' +export type Rewrapped = { + sealed: SealedSecret + /** `rewrapped`: only the DEK was re-wrapped (v2 rows). `resealed`: the value was decrypted and sealed again (v1 rows). */ + mode: 'rewrapped' | 'resealed' +} export type Keyring = { activeId: string + /** The format new rows are sealed in. */ + sealFormat: SealFormat + /** Ids of the retired KEKs this keyring can still open. */ + retiredIds: string[] seal(binding: SecretBinding, value: string): Promise /** Throws on a tampered row, a swapped binding or an unknown key id. */ open(binding: SecretBinding, sealed: SealedSecret): Promise + /** True when the row is under the active KEK and needs no format upgrade. */ + isCurrent(sealed: SealedSecret): boolean + /** + * Moves a row to the active KEK (and to v2 when that is the seal format); null when it is already + * current. The result is opened again before it is returned, so a caller never stores a row this + * keyring cannot read. Throws like `open` on a tampered row or an unknown key id. + */ + rewrap(binding: SecretBinding, sealed: SealedSecret): Promise } const IV_BYTES = 12 const KEY_ID = /^[A-Za-z0-9_-]{1,32}$/ +const V2_PREFIX = 'v2.' function toBase64(bytes: Uint8Array): string { let binary = '' @@ -37,9 +64,31 @@ function utf8(value: string): Uint8Array { return new Uint8Array(new TextEncoder().encode(value)) } -function aad(binding: SecretBinding, layer: 'dek' | 'value', kekId: string): Uint8Array { +/** Throws when the two columns disagree about the format (a spliced row). */ +export function sealedFormat(sealed: Pick): SealFormat { + const dek = sealed.wrappedDek.startsWith(V2_PREFIX) + if (dek !== sealed.ciphertext.startsWith(V2_PREFIX)) { + throw new Error('mixed secret formats') + } + return dek ? 'v2' : 'v1' +} + +function aad(format: SealFormat, binding: SecretBinding, layer: 'dek' | 'value', kekId: string): Uint8Array { // JSON keeps field boundaries unambiguous (a name can't bleed into the owner id). - return utf8(JSON.stringify(['orca-secret-v1', layer, kekId, binding.scope, binding.ownerId, binding.name])) + const identity = [binding.scope, binding.ownerId, binding.name] + if (format === 'v1') { + return utf8(JSON.stringify(['orca-secret-v1', layer, kekId, ...identity])) + } + // v2: the value is bound to its row and its DEK, not to whichever KEK currently wraps that DEK. + return utf8(JSON.stringify(layer === 'dek' ? ['orca-secret-v2', 'dek', kekId, ...identity] : ['orca-secret-v2', 'value', ...identity])) +} + +function sameBytes(a: Uint8Array, b: Uint8Array): boolean { + let diff = a.length ^ b.length + for (let i = 0; i < a.length && i < b.length; i++) { + diff |= a[i]! ^ b[i]! + } + return diff === 0 } async function encrypt(key: CryptoKey, plaintext: Uint8Array, additionalData: Uint8Array) { @@ -63,19 +112,30 @@ function importAes(raw: Uint8Array): Promise { /** * `active` is the base64 32-byte KEK new rows are sealed with; `retired` maps older key ids to - * their KEKs so rows written before a rotation still open (re-sealed on their next write). + * their KEKs so rows written before a rotation still open until the re-wrap sweep + * (src/secrets-rewrap.ts) has moved them to the active KEK. */ -export async function createKeyring(active: { id: string; key: string }, retired: Record = {}): Promise { +export async function createKeyring( + active: { id: string; key: string }, + retired: Record = {}, + options: { sealFormat?: SealFormat } = {} +): Promise { + const sealFormat = options.sealFormat ?? 'v1' const keys = new Map() - for (const [id, b64] of [[active.id, active.key] as const, ...Object.entries(retired)]) { + // The active entry goes last so it wins if `retired` repeats its id. + for (const [id, b64] of [...Object.entries(retired), [active.id, active.key] as const]) { if (!KEY_ID.test(id)) { throw new Error('invalid KEK id') } + if (typeof b64 !== 'string') { + throw new Error('KEK must be base64') + } const raw = fromBase64(b64) if (raw.length !== 32) { throw new Error('KEK must be 32 bytes') } keys.set(id, await importAes(raw)) + raw.fill(0) } const kek = (id: string) => { const key = keys.get(id) @@ -84,21 +144,71 @@ export async function createKeyring(active: { id: string; key: string }, retired } return key } + const prefix = (format: SealFormat) => (format === 'v2' ? V2_PREFIX : '') + const body = (format: SealFormat, sealed: string) => sealed.slice(prefix(format).length) + const unwrapDek = (binding: SecretBinding, sealed: SealedSecret, format: SealFormat) => + decrypt(kek(sealed.kekId), body(format, sealed.wrappedDek), aad(format, binding, 'dek', sealed.kekId)) + const wrapDek = async (binding: SecretBinding, dekRaw: Uint8Array, format: SealFormat) => + prefix(format) + (await encrypt(kek(active.id), dekRaw, aad(format, binding, 'dek', active.id))) + + async function seal(binding: SecretBinding, value: string): Promise { + const dekRaw = crypto.getRandomValues(new Uint8Array(32)) + try { + const dek = await importAes(dekRaw) + const ciphertext = prefix(sealFormat) + (await encrypt(dek, utf8(value), aad(sealFormat, binding, 'value', active.id))) + return { kekId: active.id, wrappedDek: await wrapDek(binding, dekRaw, sealFormat), ciphertext } + } finally { + dekRaw.fill(0) + } + } + + async function open(binding: SecretBinding, sealed: SealedSecret): Promise { + const format = sealedFormat(sealed) + const dekRaw = await unwrapDek(binding, sealed, format) + try { + const dek = await importAes(dekRaw) + return new TextDecoder().decode(await decrypt(dek, body(format, sealed.ciphertext), aad(format, binding, 'value', sealed.kekId))) + } finally { + dekRaw.fill(0) + } + } + + const isCurrent = (sealed: SealedSecret) => sealed.kekId === active.id && (sealFormat === 'v1' || sealedFormat(sealed) === 'v2') + return { activeId: active.id, - async seal(binding, value) { - const dekRaw = crypto.getRandomValues(new Uint8Array(32)) - const dek = await importAes(dekRaw) - const ciphertext = await encrypt(dek, utf8(value), aad(binding, 'value', active.id)) - const wrappedDek = await encrypt(kek(active.id), dekRaw, aad(binding, 'dek', active.id)) - dekRaw.fill(0) - return { kekId: active.id, wrappedDek, ciphertext } - }, - async open(binding, sealed) { - const dekRaw = await decrypt(kek(sealed.kekId), sealed.wrappedDek, aad(binding, 'dek', sealed.kekId)) - const dek = await importAes(dekRaw) - dekRaw.fill(0) - return new TextDecoder().decode(await decrypt(dek, sealed.ciphertext, aad(binding, 'value', sealed.kekId))) + sealFormat, + retiredIds: Object.keys(retired).filter((id) => id !== active.id), + seal, + open, + isCurrent, + async rewrap(binding, sealed) { + if (isCurrent(sealed)) { + return null + } + if (sealedFormat(sealed) === 'v1') { + // A v1 value is authenticated against its KEK id, so it has to be sealed again. The plaintext + // stays inside this function, as it does when a session start opens the row. + const value = await open(binding, sealed) + const next = await seal(binding, value) + if ((await open(binding, next)) !== value) { + throw new Error('reseal verification failed') + } + return { sealed: next, mode: 'resealed' } + } + const dekRaw = await unwrapDek(binding, sealed, 'v2') + try { + const next: SealedSecret = { kekId: active.id, wrappedDek: await wrapDek(binding, dekRaw, 'v2'), ciphertext: sealed.ciphertext } + const check = await unwrapDek(binding, next, 'v2') + const same = sameBytes(check, dekRaw) + check.fill(0) + if (!same) { + throw new Error('rewrap verification failed') + } + return { sealed: next, mode: 'rewrapped' } + } finally { + dekRaw.fill(0) + } } } } diff --git a/cloud/apps/sandbox-control/src/secrets-rewrap.ts b/cloud/apps/sandbox-control/src/secrets-rewrap.ts new file mode 100644 index 00000000000..e470e2fe557 --- /dev/null +++ b/cloud/apps/sandbox-control/src/secrets-rewrap.ts @@ -0,0 +1,293 @@ +// KEK rotation sweep: moves every envelope-encrypted row from a retired KEK to the active one, so +// the retired KEK can be dropped from SECRETS_KEK_RETIRED. Operator routes (operator.ts): +// GET /v1/internal/secrets/kek -> which KEK ids rows still reference, and which retired ids are droppable +// POST /v1/internal/secrets/rewrap {limit?, cursor?, dryRun?} -> one batch; repeat with `cursor` until `done` +// Idempotent and resumable: a batch selects only rows that are not current, so a run that stops +// anywhere is continued by calling again (with or without the cursor). Each write is conditional on +// the row still holding the bytes that were read, so a secret saved mid-sweep is never overwritten. +// v2 rows have only their DEK re-wrapped; v1 rows are decrypted and sealed again inside the keyring. +// Logs and responses carry counts, table names and KEK ids only: never a value, a key or ciphertext. +import type { CredentialAgent } from './agent-credential-names' +import { credentialBinding } from './agent-credentials' +import { githubUserTokenBinding } from './github-user-tokens' +import type { Keyring, SecretBinding } from './secrets-crypto' + +type Sealed = { kek_id: string; wrapped_dek: string; ciphertext: string } +type Row = Sealed & Record + +type SealedTable = { + table: string + /** Primary-key columns (all TEXT), in key order. */ + key: readonly string[] + /** The AAD identity the row was sealed under. */ + binding(row: Record): SecretBinding +} + +/** + * Every table that stores a KEK-wrapped DEK. A table missing here would keep referencing a retired + * KEK after a sweep reports done; test/secrets-rewrap.test.ts checks this list against the schema. + */ +export const SEALED_TABLES: readonly SealedTable[] = [ + { table: 'secrets', key: ['scope', 'owner_id', 'name'], binding: (row) => ({ scope: row.scope!, ownerId: row.owner_id!, name: row.name! }) }, + { table: 'agent_credentials', key: ['user_id', 'agent'], binding: (row) => credentialBinding(row.user_id!, row.agent as CredentialAgent) }, + { table: 'github_user_tokens', key: ['account_id', 'user_id'], binding: (row) => githubUserTokenBinding({ account_id: row.account_id!, user_id: row.user_id! }) } +] + +export const REWRAP_DEFAULT_LIMIT = 200 +export const REWRAP_MAX_LIMIT = 1000 +const WRITE_CHUNK = 50 + +/** Rows a sweep still has to move: under another KEK, or v1 while the seal format is v2. */ +function pendingWhere(keyring: Keyring): string { + return keyring.sealFormat === 'v2' ? `(kek_id != ? OR wrapped_dek NOT LIKE 'v2.%')` : '(kek_id != ?)' +} + +type Cursor = { table: number; after: string[] | null } + +function encodeCursor(cursor: Cursor): string { + return btoa(JSON.stringify([cursor.table, cursor.after])) +} + +/** null for anything that is not a cursor this module produced. */ +function decodeCursor(value: string): Cursor | null { + try { + const parsed: unknown = JSON.parse(atob(value)) + if (!Array.isArray(parsed) || parsed.length !== 2) { + return null + } + const [table, after] = parsed as [unknown, unknown] + const spec = typeof table === 'number' ? SEALED_TABLES[table] : undefined + if (!spec) { + return null + } + if (after === null) { + return { table: table as number, after: null } + } + if (!Array.isArray(after) || after.length !== spec.key.length || !after.every((part) => typeof part === 'string')) { + return null + } + return { table: table as number, after: after as string[] } + } catch { + return null + } +} + +export type RewrapFailure = { table: string; kekId: string; reason: 'unknown_kek' | 'unreadable'; rows: number } + +export type RewrapResult = { + dryRun: boolean + /** Rows read this batch. */ + scanned: number + /** v2 rows whose DEK was re-wrapped; the value was not decrypted. */ + rewrapped: number + /** v1 rows decrypted and sealed again under the active KEK. */ + resealed: number + /** Rows that changed between the read and the write (someone saved them); they are current now or picked up next run. */ + raced: number + /** Rows this keyring cannot open. They stay as they are and keep their KEK id in use. */ + failed: number + failures: RewrapFailure[] + /** Pass back to continue; null once every table has been walked. */ + cursor: string | null + done: boolean + /** Rows still not current after this batch (0 means every retired KEK is unreferenced). */ + pending: number +} + +export class RewrapError extends Error { + constructor(readonly reason: 'invalid_cursor' | 'invalid_limit') { + super(reason) + this.name = 'RewrapError' + } +} + +async function pendingCount(db: D1Database, keyring: Keyring): Promise { + let pending = 0 + for (const spec of SEALED_TABLES) { + const row = await db + .prepare(`SELECT COUNT(*) AS n FROM ${spec.table} WHERE ${pendingWhere(keyring)}`) + .bind(keyring.activeId) + .first<{ n: number }>() + pending += row?.n ?? 0 + } + return pending +} + +/** One batch of the sweep. Call again with the returned cursor until `done`. */ +export async function rewrapBatch( + db: D1Database, + keyring: Keyring, + options: { limit?: number; cursor?: string | null; dryRun?: boolean } = {} +): Promise { + const limit = options.limit ?? REWRAP_DEFAULT_LIMIT + if (!Number.isInteger(limit) || limit < 1 || limit > REWRAP_MAX_LIMIT) { + throw new RewrapError('invalid_limit') + } + let position: Cursor = { table: 0, after: null } + if (options.cursor) { + const decoded = decodeCursor(options.cursor) + if (!decoded) { + throw new RewrapError('invalid_cursor') + } + position = decoded + } + const dryRun = options.dryRun === true + const result = { scanned: 0, rewrapped: 0, resealed: 0, raced: 0, failed: 0 } + const failures = new Map() + let done = false + + while (result.scanned < limit) { + const spec = SEALED_TABLES[position.table] + if (!spec) { + done = true + break + } + const columns = spec.key.join(', ') + const afterKey = position.after ? ` AND (${columns}) > (${spec.key.map(() => '?').join(', ')})` : '' + const take = limit - result.scanned + const { results: rows } = await db + .prepare( + `SELECT ${columns}, kek_id, wrapped_dek, ciphertext FROM ${spec.table} + WHERE ${pendingWhere(keyring)}${afterKey} ORDER BY ${columns} LIMIT ?` + ) + .bind(keyring.activeId, ...(position.after ?? []), take) + .all() + + const writes: { statement: D1PreparedStatement; mode: 'rewrapped' | 'resealed' }[] = [] + for (const row of rows) { + result.scanned++ + const before = { kekId: row.kek_id, wrappedDek: row.wrapped_dek, ciphertext: row.ciphertext } + let moved: Awaited> + try { + moved = await keyring.rewrap(spec.binding(row), before) + } catch (error) { + // The error text is one of this module's own fixed messages or a WebCrypto OperationError; it + // is classified, never logged or returned. + const reason = error instanceof Error && error.message === 'unknown KEK id' ? 'unknown_kek' : 'unreadable' + const id = `${spec.table}\n${row.kek_id}\n${reason}` + const entry = failures.get(id) ?? { table: spec.table, kekId: row.kek_id, reason, rows: 0 } + entry.rows++ + failures.set(id, entry) + result.failed++ + continue + } + if (!moved) { + continue + } + if (dryRun) { + result[moved.mode]++ + continue + } + writes.push({ + mode: moved.mode, + // Conditional on the bytes that were read: a concurrent save (always under the active KEK) wins. + statement: db + .prepare( + `UPDATE ${spec.table} SET kek_id = ?, wrapped_dek = ?, ciphertext = ? + WHERE ${spec.key.map((column) => `${column} = ?`).join(' AND ')} AND kek_id = ? AND wrapped_dek = ? AND ciphertext = ?` + ) + .bind( + moved.sealed.kekId, + moved.sealed.wrappedDek, + moved.sealed.ciphertext, + ...spec.key.map((column) => row[column]!), + before.kekId, + before.wrappedDek, + before.ciphertext + ) + }) + } + for (let at = 0; at < writes.length; at += WRITE_CHUNK) { + const chunk = writes.slice(at, at + WRITE_CHUNK) + const outcomes = await db.batch(chunk.map((write) => write.statement)) + outcomes.forEach((outcome, index) => { + if (outcome.meta.changes === 1) { + result[chunk[index]!.mode]++ + } else { + result.raced++ + } + }) + } + + const last = rows.at(-1) + position = + rows.length < take || !last + ? { table: position.table + 1, after: null } + : { table: position.table, after: spec.key.map((column) => last[column]!) } + } + if (!SEALED_TABLES[position.table]) { + done = true + } + + const summary: RewrapResult = { + dryRun, + ...result, + failures: [...failures.values()], + cursor: done ? null : encodeCursor(position), + done, + pending: await pendingCount(db, keyring) + } + console.log( + JSON.stringify({ + event: 'secrets_rewrap', + activeKekId: keyring.activeId, + sealFormat: keyring.sealFormat, + dryRun, + scanned: summary.scanned, + rewrapped: summary.rewrapped, + resealed: summary.resealed, + raced: summary.raced, + failed: summary.failed, + failures: summary.failures, + pending: summary.pending, + done + }) + ) + return summary +} + +export type KekStatus = { + activeKekId: string + sealFormat: 'v1' | 'v2' + /** Retired KEK ids this Worker is configured with. */ + retiredKekIds: string[] + /** Per table, per KEK id: rows and how many of them are still v1. */ + tables: Record> + /** Rows a sweep would still move. */ + pending: number + /** Retired ids rows still reference: keep them configured. */ + retiredInUse: string[] + /** Retired ids no row references: safe to remove from SECRETS_KEK_RETIRED. */ + retiredDroppable: string[] + /** KEK ids rows reference that this Worker has no key for: those rows cannot be opened. */ + unknownKekIds: string[] +} + +/** What rows reference which KEK. Ids and counts only. */ +export async function kekStatus(db: D1Database, keyring: Keyring): Promise { + const tables: KekStatus['tables'] = {} + const referenced = new Set() + for (const spec of SEALED_TABLES) { + const { results } = await db + .prepare( + `SELECT kek_id, COUNT(*) AS n, SUM(CASE WHEN wrapped_dek LIKE 'v2.%' THEN 0 ELSE 1 END) AS v1 + FROM ${spec.table} GROUP BY kek_id ORDER BY kek_id` + ) + .all<{ kek_id: string; n: number; v1: number }>() + tables[spec.table] = Object.fromEntries(results.map((row) => [row.kek_id, { rows: row.n, v1: row.v1 }])) + for (const row of results) { + referenced.add(row.kek_id) + } + } + const known = new Set([keyring.activeId, ...keyring.retiredIds]) + return { + activeKekId: keyring.activeId, + sealFormat: keyring.sealFormat, + retiredKekIds: keyring.retiredIds, + tables, + pending: await pendingCount(db, keyring), + retiredInUse: keyring.retiredIds.filter((id) => referenced.has(id)), + retiredDroppable: keyring.retiredIds.filter((id) => !referenced.has(id)), + unknownKekIds: [...referenced].filter((id) => !known.has(id)).sort() + } +} diff --git a/cloud/apps/sandbox-control/src/secrets.ts b/cloud/apps/sandbox-control/src/secrets.ts index 81b765b1d5d..24152fd6d3e 100644 --- a/cloud/apps/sandbox-control/src/secrets.ts +++ b/cloud/apps/sandbox-control/src/secrets.ts @@ -416,12 +416,14 @@ export function keyringFor(env: Env): Promise | null { if (!env.SECRETS_KEK) { return null } - const source = `${env.SECRETS_KEK_ID ?? 'k1'}\n${env.SECRETS_KEK}\n${env.SECRETS_KEK_RETIRED ?? ''}` + // Anything but the exact opt-in stays v1, the format every deployed version reads. + const sealFormat = env.SECRETS_SEAL_FORMAT === 'v2' ? 'v2' : 'v1' + const source = `${env.SECRETS_KEK_ID ?? 'k1'}\n${env.SECRETS_KEK}\n${env.SECRETS_KEK_RETIRED ?? ''}\n${sealFormat}` if (cachedKeyring?.source !== source) { const retired = env.SECRETS_KEK_RETIRED ? (JSON.parse(env.SECRETS_KEK_RETIRED) as Record) : {} cachedKeyring = { source, - keyring: createKeyring({ id: env.SECRETS_KEK_ID ?? 'k1', key: env.SECRETS_KEK }, retired) + keyring: createKeyring({ id: env.SECRETS_KEK_ID ?? 'k1', key: env.SECRETS_KEK }, retired, { sealFormat }) } } return cachedKeyring.keyring diff --git a/cloud/apps/sandbox-control/test/secrets-rewrap.test.ts b/cloud/apps/sandbox-control/test/secrets-rewrap.test.ts new file mode 100644 index 00000000000..34d6829abed --- /dev/null +++ b/cloud/apps/sandbox-control/test/secrets-rewrap.test.ts @@ -0,0 +1,517 @@ +import { createCipheriv, createDecipheriv, randomBytes } from 'node:crypto' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { credentialBinding } from '../src/agent-credentials' +import { githubUserTokenBinding } from '../src/github-user-tokens' +import { createKeyring, sealedFormat, type Keyring, type SealedSecret, type SecretBinding } from '../src/secrets-crypto' +import { kekStatus, rewrapBatch, RewrapError, SEALED_TABLES } from '../src/secrets-rewrap' +import { createFakeD1 } from './d1-fake' +import { createHarness } from './harness' + +// Fake keys and values only; the log check looks for every one of them. +const K0 = Buffer.alloc(32, 3).toString('base64') +const K1 = Buffer.alloc(32, 7).toString('base64') +const K2 = Buffer.alloc(32, 9).toString('base64') +const OP = 'o'.repeat(40) +const VALUES = ['org-value-aaaa-1111', 'personal-value-bbbb-2222', 'repo-value-cccc-3333', 'claude-token-dddd-4444', 'gh-token-eeee-5555'] +const NEVER_LOGGED = [...VALUES, K0, K1, K2] + +let logs: string[] = [] +beforeEach(() => { + logs = [] + for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) { + vi.spyOn(console, method).mockImplementation((...args: unknown[]) => { + logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')) + }) + } +}) +afterEach(() => { + // Whatever the test did, no value and no key reached a log line. + for (const secret of NEVER_LOGGED) { + expect(logs.join('\n')).not.toContain(secret) + } + vi.restoreAllMocks() +}) + +type Fake = ReturnType +type Stored = { table: string; binding: SecretBinding; value: string; where: string } + +async function insertSecret(fake: Fake, keyring: Keyring, scope: string, owner: string, name: string, value: string): Promise { + const binding = { scope, ownerId: owner, name } + const sealed = await keyring.seal(binding, value) + fake.raw + .prepare( + `INSERT INTO secrets (scope, owner_id, name, kek_id, wrapped_dek, ciphertext, hint, created_at, updated_at, updated_by) + VALUES (?, ?, ?, ?, ?, ?, NULL, 1, 1, 'alice')` + ) + .run(scope, owner, name, sealed.kekId, sealed.wrappedDek, sealed.ciphertext) + return { table: 'secrets', binding, value, where: `scope = '${scope}' AND owner_id = '${owner}' AND name = '${name}'` } +} + +async function insertCredential(fake: Fake, keyring: Keyring, userId: string, value: string): Promise { + const binding = credentialBinding(userId, 'claude') + const sealed = await keyring.seal(binding, value) + fake.raw + .prepare( + `INSERT INTO agent_credentials (user_id, account_id, agent, kek_id, wrapped_dek, ciphertext, created_at, updated_at) + VALUES (?, ?, 'claude', ?, ?, ?, 1, 1)` + ) + .run(userId, `user:${userId}`, sealed.kekId, sealed.wrappedDek, sealed.ciphertext) + return { table: 'agent_credentials', binding, value, where: `user_id = '${userId}' AND agent = 'claude'` } +} + +async function insertGitHubToken(fake: Fake, keyring: Keyring, accountId: string, userId: string, value: string): Promise { + const binding = githubUserTokenBinding({ account_id: accountId, user_id: userId }) + const sealed = await keyring.seal(binding, value) + fake.raw + .prepare( + `INSERT INTO github_user_tokens (account_id, user_id, github_user_id, github_login, kek_id, wrapped_dek, ciphertext, updated_at) + VALUES (?, ?, 1, 'octo', ?, ?, ?, 1)` + ) + .run(accountId, userId, sealed.kekId, sealed.wrappedDek, sealed.ciphertext) + return { table: 'github_user_tokens', binding, value, where: `account_id = '${accountId}' AND user_id = '${userId}'` } +} + +function read(fake: Fake, stored: Stored): SealedSecret & { updated_at: number } { + const row = fake.raw.prepare(`SELECT kek_id, wrapped_dek, ciphertext, updated_at FROM ${stored.table} WHERE ${stored.where}`).get() as { + kek_id: string + wrapped_dek: string + ciphertext: string + updated_at: number + } + return { kekId: row.kek_id, wrappedDek: row.wrapped_dek, ciphertext: row.ciphertext, updated_at: row.updated_at } +} + +/** One row in each sealed table plus two more secrets, all under `keyring`. */ +async function seed(fake: Fake, keyring: Keyring): Promise { + return [ + await insertSecret(fake, keyring, 'org', 'org:acme', 'ORG_VAR', VALUES[0]!), + await insertSecret(fake, keyring, 'personal', 'user:alice', 'MY_VAR', VALUES[1]!), + await insertSecret(fake, keyring, 'repo', 'org:acme#github.com/acme/app', 'REPO_VAR', VALUES[2]!), + await insertCredential(fake, keyring, 'alice', VALUES[3]!), + await insertGitHubToken(fake, keyring, 'org:acme', 'alice', VALUES[4]!) + ] +} + +async function sweep(fake: Fake, keyring: Keyring, limit: number) { + const batches = [] + let cursor: string | null = null + do { + const batch = await rewrapBatch(fake.db, keyring, { limit, cursor }) + batches.push(batch) + cursor = batch.cursor + } while (cursor) + return batches +} + +/** + * The v1 row layout as rows written before this change have it, built with node:crypto instead of + * the keyring: base64(iv || ciphertext || tag) on both columns, AAD ["orca-secret-v1", layer, kekId, scope, owner, name]. + */ +function legacyV1Row(kekB64: string, kekId: string, binding: SecretBinding, value: string): SealedSecret { + const encrypt = (key: Buffer, plaintext: Buffer, layer: 'dek' | 'value') => { + const iv = randomBytes(12) + const cipher = createCipheriv('aes-256-gcm', key, iv) + cipher.setAAD(Buffer.from(JSON.stringify(['orca-secret-v1', layer, kekId, binding.scope, binding.ownerId, binding.name]))) + return Buffer.concat([iv, cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]).toString('base64') + } + const dek = randomBytes(32) + return { kekId, ciphertext: encrypt(dek, Buffer.from(value, 'utf8'), 'value'), wrappedDek: encrypt(Buffer.from(kekB64, 'base64'), dek, 'dek') } +} + +describe('rows written before this change (v1)', () => { + const binding = { scope: 'org', ownerId: 'org:acme', name: 'ORG_VAR' } + + it('open unchanged with SECRETS_SEAL_FORMAT unset, under the active or a retired KEK', async () => { + const row = legacyV1Row(K1, 'k1', binding, VALUES[0]!) + const keyring = await createKeyring({ id: 'k1', key: K1 }) + expect(sealedFormat(row)).toBe('v1') + expect(await keyring.open(binding, row)).toBe(VALUES[0]) + expect(keyring.isCurrent(row)).toBe(true) + expect(await keyring.rewrap(binding, row)).toBeNull() + // New rows are still written in exactly that layout: the independent reader opens them. + const fresh = await keyring.seal(binding, VALUES[1]!) + const open = (key: Buffer, sealed: string, layer: 'dek' | 'value') => { + const bytes = Buffer.from(sealed, 'base64') + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)) + decipher.setAAD(Buffer.from(JSON.stringify(['orca-secret-v1', layer, 'k1', binding.scope, binding.ownerId, binding.name]))) + decipher.setAuthTag(bytes.subarray(bytes.length - 16)) + return Buffer.concat([decipher.update(bytes.subarray(12, bytes.length - 16)), decipher.final()]) + } + expect(open(open(Buffer.from(K1, 'base64'), fresh.wrappedDek, 'dek'), fresh.ciphertext, 'value').toString('utf8')).toBe(VALUES[1]) + + const retired = legacyV1Row(K0, 'k0', binding, VALUES[2]!) + expect(await (await createKeyring({ id: 'k1', key: K1 }, { k0: K0 })).open(binding, retired)).toBe(VALUES[2]) + // Also with v2 switched on: old rows stay readable before any sweep has run. + expect(await (await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }, { sealFormat: 'v2' })).open(binding, retired)).toBe(VALUES[2]) + }) + + it('are left byte-for-byte alone by the store and the sweep while the format is unset', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K1 }) + const owner = { scope: 'personal', ownerId: 'user:alice', name: 'OLD_VAR' } + const row = legacyV1Row(K1, 'k1', owner, VALUES[0]!) + h.raw + .prepare( + `INSERT INTO secrets (scope, owner_id, name, kek_id, wrapped_dek, ciphertext, hint, created_at, updated_at, updated_by) + VALUES ('personal', 'user:alice', 'OLD_VAR', 'k1', ?, ?, NULL, 1, 1, 'alice')` + ) + .run(row.wrappedDek, row.ciphertext) + const listed = await h.call('GET', '/v1/secrets?scope=personal', { bearer: await h.token('alice') }) + expect(listed.body.secrets.map((s: { name: string }) => s.name)).toEqual(['OLD_VAR']) + expect((await h.call('GET', '/v1/internal/secrets/kek', { bearer: OP })).body).toMatchObject({ sealFormat: 'v1', pending: 0, unknownKekIds: [] }) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: {} })).body).toMatchObject({ scanned: 0, done: true }) + // A dry run under a pretend rotation proves the row opens, without writing. + Object.assign(h.env, { SECRETS_KEK: K2, SECRETS_KEK_ID: 'k2', SECRETS_KEK_RETIRED: JSON.stringify({ k1: K1 }) }) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: { dryRun: true } })).body).toMatchObject({ scanned: 1, resealed: 1, failed: 0 }) + const after = h.raw.prepare(`SELECT kek_id, wrapped_dek, ciphertext FROM secrets WHERE name = 'OLD_VAR'`).get() + expect(after).toEqual({ kek_id: 'k1', wrapped_dek: row.wrappedDek, ciphertext: row.ciphertext }) + }) +}) + +describe('v2 envelope format', () => { + const binding = { scope: 'org', ownerId: 'org:acme', name: 'ORG_VAR' } + const flip = (sealed: string) => { + const at = sealed.startsWith('v2.') ? 3 : 0 + const bytes = Buffer.from(sealed.slice(at), 'base64') + bytes[bytes.length - 1]! ^= 1 + return sealed.slice(0, at) + bytes.toString('base64') + } + + it('round-trips, refuses tampering, a swapped identity and a stripped format marker, and is readable by a v1-sealing keyring', async () => { + const v2 = await createKeyring({ id: 'k1', key: K1 }, {}, { sealFormat: 'v2' }) + const sealed = await v2.seal(binding, VALUES[0]!) + expect(sealedFormat(sealed)).toBe('v2') + expect(await v2.open(binding, sealed)).toBe(VALUES[0]) + // A Worker still sealing v1 (the default) reads v2 rows: readers ship before the format is switched on. + const v1 = await createKeyring({ id: 'k1', key: K1 }) + expect(v1.sealFormat).toBe('v1') + expect(sealedFormat(await v1.seal(binding, VALUES[0]!))).toBe('v1') + expect(await v1.open(binding, sealed)).toBe(VALUES[0]) + + await expect(v2.open(binding, { ...sealed, ciphertext: flip(sealed.ciphertext) })).rejects.toThrow() + await expect(v2.open(binding, { ...sealed, wrappedDek: flip(sealed.wrappedDek) })).rejects.toThrow() + await expect(v2.open({ ...binding, ownerId: 'org:evil' }, sealed)).rejects.toThrow() + await expect(v2.open({ ...binding, name: 'OTHER' }, sealed)).rejects.toThrow() + // Downgrading the marker on one or both columns never opens. + await expect(v2.open(binding, { ...sealed, ciphertext: sealed.ciphertext.slice(3) })).rejects.toThrow('mixed secret formats') + await expect(v2.open(binding, { ...sealed, wrappedDek: sealed.wrappedDek.slice(3), ciphertext: sealed.ciphertext.slice(3) })).rejects.toThrow() + // A DEK wrapped for one row cannot be moved under another row's ciphertext. + const other = await v2.seal({ ...binding, name: 'OTHER' }, 'x') + await expect(v2.open(binding, { ...sealed, wrappedDek: other.wrappedDek })).rejects.toThrow() + }) + + it('re-wraps only the data key of a v2 row: the ciphertext is untouched and the retired KEK is no longer needed', async () => { + const old = await createKeyring({ id: 'k0', key: K0 }, {}, { sealFormat: 'v2' }) + const sealed = await old.seal(binding, VALUES[0]!) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }, { sealFormat: 'v2' }) + const moved = await rotated.rewrap(binding, sealed) + expect(moved?.mode).toBe('rewrapped') + expect(moved!.sealed.kekId).toBe('k1') + expect(moved!.sealed.ciphertext).toBe(sealed.ciphertext) + expect(moved!.sealed.wrappedDek).not.toBe(sealed.wrappedDek) + const onlyNew = await createKeyring({ id: 'k1', key: K1 }) + expect(await onlyNew.open(binding, moved!.sealed)).toBe(VALUES[0]) + await expect(old.open(binding, moved!.sealed)).rejects.toThrow('unknown KEK id') + // Already current: nothing to do. A v2 row also stays v2 when the Worker seals v1. + expect(await rotated.rewrap(binding, moved!.sealed)).toBeNull() + const v1Sealer = await createKeyring({ id: 'k2', key: K2 }, { k1: K1 }) + const again = await v1Sealer.rewrap(binding, moved!.sealed) + expect(again?.mode).toBe('rewrapped') + expect(again!.sealed.ciphertext).toBe(sealed.ciphertext) + }) + + it('re-seals a v1 row (its value is bound to the KEK id) and refuses rows it cannot authenticate', async () => { + const old = await createKeyring({ id: 'k0', key: K0 }) + const sealed = await old.seal(binding, VALUES[0]!) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + const moved = await rotated.rewrap(binding, sealed) + expect(moved?.mode).toBe('resealed') + expect(sealedFormat(moved!.sealed)).toBe('v1') + expect(await (await createKeyring({ id: 'k1', key: K1 })).open(binding, moved!.sealed)).toBe(VALUES[0]) + // With v2 on, the same move upgrades the row, and a v1 row under the active KEK is upgraded too. + const upgrading = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }, { sealFormat: 'v2' }) + expect(sealedFormat((await upgrading.rewrap(binding, sealed))!.sealed)).toBe('v2') + expect(upgrading.isCurrent(moved!.sealed)).toBe(false) + expect(sealedFormat((await upgrading.rewrap(binding, moved!.sealed))!.sealed)).toBe('v2') + + await expect(rotated.rewrap({ ...binding, name: 'OTHER' }, sealed)).rejects.toThrow() + await expect(rotated.rewrap(binding, { ...sealed, wrappedDek: flip(sealed.wrappedDek) })).rejects.toThrow() + await expect((await createKeyring({ id: 'k1', key: K1 })).rewrap(binding, sealed)).rejects.toThrow('unknown KEK id') + }) +}) + +describe('KEK re-wrap sweep', () => { + it('covers every table that stores a wrapped data key', () => { + const { raw } = createFakeD1() + const tables = raw + .prepare(`SELECT m.name AS name FROM sqlite_master m JOIN pragma_table_info(m.name) c WHERE m.type = 'table' AND c.name = 'wrapped_dek' ORDER BY m.name`) + .all() + .map((row) => (row as { name: string }).name) + expect(SEALED_TABLES.map((spec) => spec.table).sort()).toEqual(tables) + for (const spec of SEALED_TABLES) { + const pk = raw + .prepare(`SELECT name FROM pragma_table_info('${spec.table}') WHERE pk > 0 ORDER BY pk`) + .all() + .map((row) => (row as { name: string }).name) + expect([...spec.key]).toEqual(pk) + } + }) + + it('moves every row to the active KEK in resumable batches, after which the retired KEK can be dropped', async () => { + const fake = createFakeD1() + const stored = await seed(fake, await createKeyring({ id: 'k0', key: K0 })) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + + const before = await kekStatus(fake.db, rotated) + expect(before).toMatchObject({ activeKekId: 'k1', sealFormat: 'v1', retiredKekIds: ['k0'], pending: 5, retiredInUse: ['k0'], retiredDroppable: [], unknownKekIds: [] }) + expect(before.tables).toEqual({ + secrets: { k0: { rows: 3, v1: 3 } }, + agent_credentials: { k0: { rows: 1, v1: 1 } }, + github_user_tokens: { k0: { rows: 1, v1: 1 } } + }) + + const batches = await sweep(fake, rotated, 2) + expect(batches.length).toBeGreaterThan(2) + expect(batches.slice(0, -1).every((batch) => !batch.done && batch.cursor)).toBe(true) + expect(batches.at(-1)).toMatchObject({ done: true, cursor: null, pending: 0, failed: 0 }) + expect(batches.reduce((n, batch) => n + batch.resealed, 0)).toBe(5) + expect(batches.map((batch) => batch.pending)).toEqual([...batches.map((batch) => batch.pending)].sort((a, b) => b - a)) + + // The retired key is gone from the keyring and every row still opens to its value. + const onlyNew = await createKeyring({ id: 'k1', key: K1 }) + for (const row of stored) { + const sealed = read(fake, row) + expect(sealed.kekId).toBe('k1') + expect(sealed.updated_at).toBe(1) + expect(await onlyNew.open(row.binding, sealed)).toBe(row.value) + } + expect(await kekStatus(fake.db, rotated)).toMatchObject({ pending: 0, retiredInUse: [], retiredDroppable: ['k0'] }) + + // Idempotent: a second sweep finds nothing and changes nothing. + const snapshot = stored.map((row) => read(fake, row)) + expect(await rewrapBatch(fake.db, rotated)).toMatchObject({ scanned: 0, rewrapped: 0, resealed: 0, done: true, pending: 0 }) + expect(stored.map((row) => read(fake, row))).toEqual(snapshot) + // Progress lines carry counts and key ids. + const progress = logs.filter((line) => line.includes('"secrets_rewrap"')).map((line) => JSON.parse(line)) + expect(progress.length).toBe(batches.length + 1) + expect(progress[0]).toMatchObject({ event: 'secrets_rewrap', activeKekId: 'k1', scanned: 2, resealed: 2, pending: 3, done: false }) + }) + + it('continues an interrupted sweep without its cursor', async () => { + const fake = createFakeD1() + await seed(fake, await createKeyring({ id: 'k0', key: K0 })) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + expect(await rewrapBatch(fake.db, rotated, { limit: 2 })).toMatchObject({ scanned: 2, resealed: 2, pending: 3, done: false }) + // The caller died here. Starting over picks up exactly the rows that are left. + expect(await rewrapBatch(fake.db, rotated, { limit: 10 })).toMatchObject({ scanned: 3, resealed: 3, pending: 0, done: true }) + }) + + it('with v2 on, re-wraps data keys only: stored ciphertext does not change', async () => { + const fake = createFakeD1() + const stored = await seed(fake, await createKeyring({ id: 'k0', key: K0 }, {}, { sealFormat: 'v2' })) + // One legacy v1 row under the same retired key. + const legacy = await insertSecret(fake, await createKeyring({ id: 'k0', key: K0 }), 'personal', 'user:bob', 'LEGACY', 'legacy-value') + const ciphertexts = stored.map((row) => read(fake, row).ciphertext) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }, { sealFormat: 'v2' }) + + const batches = await sweep(fake, rotated, 4) + expect(batches.reduce((n, batch) => n + batch.rewrapped, 0)).toBe(5) + expect(batches.reduce((n, batch) => n + batch.resealed, 0)).toBe(1) + expect(stored.map((row) => read(fake, row).ciphertext)).toEqual(ciphertexts) + const onlyNew = await createKeyring({ id: 'k1', key: K1 }) + for (const row of [...stored, legacy]) { + const sealed = read(fake, row) + expect([sealed.kekId, sealedFormat(sealed)]).toEqual(['k1', 'v2']) + expect(await onlyNew.open(row.binding, sealed)).toBe(row.value) + } + }) + + it('upgrades v1 rows under the active KEK once v2 is on, so the next rotation never decrypts a value', async () => { + const fake = createFakeD1() + const stored = await seed(fake, await createKeyring({ id: 'k1', key: K1 })) + const v1 = await createKeyring({ id: 'k1', key: K1 }) + expect(await rewrapBatch(fake.db, v1)).toMatchObject({ scanned: 0, pending: 0, done: true }) + const v2 = await createKeyring({ id: 'k1', key: K1 }, {}, { sealFormat: 'v2' }) + expect((await kekStatus(fake.db, v2)).pending).toBe(5) + expect(await rewrapBatch(fake.db, v2)).toMatchObject({ scanned: 5, resealed: 5, pending: 0, done: true }) + expect((await kekStatus(fake.db, v2)).tables.secrets).toEqual({ k1: { rows: 3, v1: 0 } }) + + const next = await createKeyring({ id: 'k2', key: K2 }, { k1: K1 }, { sealFormat: 'v2' }) + expect(await rewrapBatch(fake.db, next)).toMatchObject({ scanned: 5, rewrapped: 5, resealed: 0, pending: 0, done: true }) + const onlyNext = await createKeyring({ id: 'k2', key: K2 }) + for (const row of stored) { + expect(await onlyNext.open(row.binding, read(fake, row))).toBe(row.value) + } + }) + + it('leaves unreadable rows alone, reports them by table and key id, and does not let them block the rest', async () => { + const fake = createFakeD1() + const old = await createKeyring({ id: 'k0', key: K0 }) + // Sorted first in `secrets`, so a sweep that retried failures forever would never get past them. + const lost = await insertSecret(fake, await createKeyring({ id: 'kx', key: K2 }), 'org', 'org:aaa', 'LOST', 'lost-value') + const tampered = await insertSecret(fake, old, 'org', 'org:aab', 'TAMPERED', 'tampered-value') + fake.raw.prepare(`UPDATE secrets SET owner_id = 'org:aac' WHERE name = 'TAMPERED'`).run() + tampered.where = `name = 'TAMPERED'` + const stored = await seed(fake, old) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + const untouched = [read(fake, lost), read(fake, tampered)] + + const batches = await sweep(fake, rotated, 2) + expect(batches.reduce((n, batch) => n + batch.failed, 0)).toBe(2) + expect(batches[0]!.failures).toEqual([ + { table: 'secrets', kekId: 'kx', reason: 'unknown_kek', rows: 1 }, + { table: 'secrets', kekId: 'k0', reason: 'unreadable', rows: 1 } + ]) + expect(batches.at(-1)).toMatchObject({ done: true, pending: 2 }) + expect([read(fake, lost), read(fake, tampered)]).toEqual(untouched) + for (const row of stored) { + expect(read(fake, row).kekId).toBe('k1') + } + // k0 is still referenced by the row that would not open, so it is not reported droppable. + expect(await kekStatus(fake.db, rotated)).toMatchObject({ pending: 2, retiredInUse: ['k0'], retiredDroppable: [], unknownKekIds: ['kx'] }) + // No identifier of a failing row, and nothing of its contents, is logged. + expect(logs.join('\n')).not.toContain('org:aaa') + expect(logs.join('\n')).not.toContain(untouched[0]!.ciphertext) + }) + + it('never overwrites a secret saved between its read and its write', async () => { + const fake = createFakeD1() + const [target] = await seed(fake, await createKeyring({ id: 'k0', key: K0 })) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + const fresh = await rotated.seal(target!.binding, 'saved-mid-sweep') + const racing = { + prepare: (sql: string) => fake.db.prepare(sql), + batch: (statements: D1PreparedStatement[]) => { + // A user saves the row after the sweep read it and before the sweep writes. + fake.raw.prepare(`UPDATE secrets SET kek_id = ?, wrapped_dek = ?, ciphertext = ? WHERE ${target!.where}`).run(fresh.kekId, fresh.wrappedDek, fresh.ciphertext) + return fake.db.batch(statements) + } + } as unknown as D1Database + + expect(await rewrapBatch(racing, rotated)).toMatchObject({ scanned: 5, resealed: 4, raced: 1, pending: 0, done: true }) + expect(await rotated.open(target!.binding, read(fake, target!))).toBe('saved-mid-sweep') + }) + + it('dry run reports what would move and whether every row opens, and writes nothing', async () => { + const fake = createFakeD1() + const stored = await seed(fake, await createKeyring({ id: 'k0', key: K0 })) + const rotated = await createKeyring({ id: 'k1', key: K1 }, { k0: K0 }) + const snapshot = stored.map((row) => read(fake, row)) + const batches = [] + let cursor: string | null = null + do { + const batch = await rewrapBatch(fake.db, rotated, { limit: 2, cursor, dryRun: true }) + batches.push(batch) + cursor = batch.cursor + } while (cursor) + expect(batches.reduce((n, batch) => n + batch.resealed, 0)).toBe(5) + expect(batches.at(-1)).toMatchObject({ dryRun: true, done: true, pending: 5, failed: 0 }) + expect(stored.map((row) => read(fake, row))).toEqual(snapshot) + }) + + it('rejects a bad limit or a cursor it did not issue', async () => { + const fake = createFakeD1() + const keyring = await createKeyring({ id: 'k1', key: K1 }) + await expect(rewrapBatch(fake.db, keyring, { limit: 0 })).rejects.toThrow(RewrapError) + await expect(rewrapBatch(fake.db, keyring, { limit: 100_000 })).rejects.toThrow(RewrapError) + for (const cursor of ['nope', btoa('[9,null]'), btoa('[0,["only-one"]]'), btoa('[0,["a","b",3]]'), btoa('{"table":0}')]) { + await expect(rewrapBatch(fake.db, keyring, { cursor })).rejects.toThrow('invalid_cursor') + } + }) +}) + +describe('operator rotation routes', () => { + it('are operator-only and need a configured KEK', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K1 }) + const alice = await h.token('alice', 'acme', 'sandbox', { orgRole: 'owner' }) + expect((await h.call('GET', '/v1/internal/secrets/kek', { bearer: alice })).status).toBe(404) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: alice, body: {} })).status).toBe(404) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: 'x'.repeat(40), body: {} })).status).toBe(404) + + const unconfigured = await createHarness({ OPERATOR_TOKEN: OP }) + expect(await unconfigured.call('GET', '/v1/internal/secrets/kek', { bearer: OP })).toEqual({ status: 503, body: { error: 'secrets_unconfigured' } }) + // A broken key config answers without echoing what was wrong with it. + const broken = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K1, SECRETS_KEK_RETIRED: `{"k0":"${K0}"` }) + const refused = await broken.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: {} }) + expect(refused.status).toBeGreaterThanOrEqual(400) + expect(JSON.stringify(refused.body)).not.toContain(K0) + const short = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K1, SECRETS_KEK_RETIRED: JSON.stringify({ k0: 'c2hvcnQ=' }) }) + expect(await short.call('GET', '/v1/internal/secrets/kek', { bearer: OP })).toEqual({ status: 503, body: { error: 'secrets_misconfigured' } }) + + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: { limit: 0 } })).status).toBe(400) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: { nope: true } })).status).toBe(400) + expect(await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: { cursor: 'nope' } })).toEqual({ + status: 400, + body: { error: 'invalid_request', reason: 'invalid_cursor' } + }) + }) + + it('rotates a live store end to end: save under k0, rotate, sweep, drop k0, and the value is still usable', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K0, SECRETS_KEK_ID: 'k0' }) + const alice = await h.token('alice') + for (const [name, value] of [['FIRST_VAR', VALUES[0]!], ['SECOND_VAR', VALUES[1]!], ['THIRD_VAR', VALUES[2]!]] as const) { + expect((await h.call('PUT', `/v1/secrets/personal/${name}`, { bearer: alice, body: { value } })).status).toBe(200) + } + + // Rotation step 1: the new key becomes active and the old one is kept as retired. + Object.assign(h.env, { SECRETS_KEK: K1, SECRETS_KEK_ID: 'k1', SECRETS_KEK_RETIRED: JSON.stringify({ k0: K0 }) }) + const before = await h.call('GET', '/v1/internal/secrets/kek', { bearer: OP }) + expect(before.body).toMatchObject({ activeKekId: 'k1', pending: 3, retiredInUse: ['k0'], retiredDroppable: [] }) + // A save after the rotation lands under the new key on its own. + await h.call('PUT', '/v1/secrets/personal/FIRST_VAR', { bearer: alice, body: { value: VALUES[3]! } }) + + // Step 2: sweep until done. + const responses = [] + let cursor: string | null = null + do { + const res = await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: { limit: 1, cursor } }) + expect(res.status).toBe(200) + responses.push(res.body) + cursor = res.body.cursor + } while (cursor) + expect(responses.reduce((n, body) => n + body.resealed, 0)).toBe(2) + const after = await h.call('GET', '/v1/internal/secrets/kek', { bearer: OP }) + expect(after.body).toMatchObject({ pending: 0, retiredInUse: [], retiredDroppable: ['k0'], tables: { secrets: { k1: { rows: 3, v1: 3 } } } }) + // Responses carry counts and ids, never a value, a key or stored bytes. + const stored = h.raw.prepare('SELECT name, kek_id, wrapped_dek, ciphertext FROM secrets ORDER BY name').all() as { + name: string + kek_id: string + wrapped_dek: string + ciphertext: string + }[] + const wire = JSON.stringify([before.body, after.body, responses]) + for (const leaked of [...NEVER_LOGGED, ...stored.flatMap((row) => [row.wrapped_dek, row.ciphertext])]) { + expect(wire).not.toContain(leaked) + } + + // Step 3: the retired key is dropped; the store reads with the new key alone. + delete h.env.SECRETS_KEK_RETIRED + expect((await h.call('GET', '/v1/internal/secrets/kek', { bearer: OP })).body).toMatchObject({ retiredKekIds: [], pending: 0, unknownKekIds: [] }) + const onlyNew = await createKeyring({ id: 'k1', key: K1 }) + const opened = await Promise.all( + stored.map((row) => onlyNew.open({ scope: 'personal', ownerId: 'user:alice', name: row.name }, { kekId: row.kek_id, wrappedDek: row.wrapped_dek, ciphertext: row.ciphertext })) + ) + expect(opened).toEqual([VALUES[3], VALUES[1], VALUES[2]]) + }) + + it('seals v2 only when SECRETS_SEAL_FORMAT=v2', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OP, SECRETS_KEK: K1 }) + const alice = await h.token('alice') + const formatOf = (name: string) => { + const row = h.raw.prepare('SELECT wrapped_dek, ciphertext FROM secrets WHERE name = ?').get(name) as { wrapped_dek: string; ciphertext: string } + return sealedFormat({ wrappedDek: row.wrapped_dek, ciphertext: row.ciphertext }) + } + await h.call('PUT', '/v1/secrets/personal/BEFORE_VAR', { bearer: alice, body: { value: VALUES[0]! } }) + expect(formatOf('BEFORE_VAR')).toBe('v1') + for (const ignored of ['V2', 'v3', '']) { + h.env.SECRETS_SEAL_FORMAT = ignored + await h.call('PUT', '/v1/secrets/personal/BEFORE_VAR', { bearer: alice, body: { value: VALUES[0]! } }) + expect(formatOf('BEFORE_VAR')).toBe('v1') + } + h.env.SECRETS_SEAL_FORMAT = 'v2' + await h.call('PUT', '/v1/secrets/personal/AFTER_VAR', { bearer: alice, body: { value: VALUES[1]! } }) + expect(formatOf('AFTER_VAR')).toBe('v2') + expect((await h.call('GET', '/v1/internal/secrets/kek', { bearer: OP })).body).toMatchObject({ sealFormat: 'v2', pending: 1 }) + expect((await h.call('POST', '/v1/internal/secrets/rewrap', { bearer: OP, body: {} })).body).toMatchObject({ resealed: 1, pending: 0, done: true }) + expect(formatOf('BEFORE_VAR')).toBe('v2') + }) +}) diff --git a/cloud/docs/sec-kms-design.md b/cloud/docs/sec-kms-design.md new file mode 100644 index 00000000000..dd5bbf2a6ac --- /dev/null +++ b/cloud/docs/sec-kms-design.md @@ -0,0 +1,132 @@ +# SEC-3: moving the secrets-store KEK behind a KMS (design, not implemented) + +Status: design and recommendation, 2026-11-12. Nothing here has been built or provisioned. No prod, no Terraform. +Prices, latencies and vendor feature claims are from memory of public docs and are **unmeasured**; check each before committing. + +## Where we are after the SEC-3 PR + +- Per-row data keys (DEK, AES-256-GCM) wrapped by one key-encryption key (KEK). The KEK is a Cloudflare Worker secret (`SECRETS_KEK`), with a copy in GCP Secret Manager (launch runbook 1.6a). +- Three tables hold wrapped DEKs: `secrets`, `agent_credentials`, `github_user_tokens`. +- Rotation now re-encrypts: `POST /v1/internal/secrets/rewrap` moves every row to the active KEK, and `GET /v1/internal/secrets/kek` says when a retired KEK is unreferenced. +- A v2 row format (off by default, `SECRETS_SEAL_FORMAT=v2`) lets rotation re-wrap the 32-byte DEK without decrypting the value. + +What is still true: the KEK is a static, exportable 32-byte value. Anyone who can read the Worker secret and export D1 can decrypt every stored secret offline, with no log entry and no way to revoke after the fact. + +## Threat model + +| Threat | Today | KMS-wrapped KEK (phase 1) | Per-org keys under KMS (phase 2) | +|---|---|---|---| +| D1 export alone (backup leak, SQL access) | Safe: ciphertext only | Safe | Safe | +| Cloudflare account access: read Worker secrets + D1 | **Decrypts everything, offline, silently** | Attacker gets a wrapped KEK and a GCP credential. Must call KMS: every call is in Cloud Audit Logs, and disabling the service account or key version stops it. Data exfiltrated before revocation is still lost | Same, and each org's key is a separate logged unwrap, so the logs show which orgs were touched | +| Code execution in the Worker (malicious deploy, dependency) | Decrypts whatever it asks for | Same. A KMS does not help when the legitimate caller is compromised | Same, limited to the orgs it unwraps while it runs | +| Leaked KEK copy (laptop, transcript, password-manager export) | Decrypts everything until a rotation plus sweep completes | The KEK plaintext exists only in Worker memory and in the escrow copy, so there are fewer places to leak from | No long-lived exportable root key exists at all | +| KEK lost | Every secret unrecoverable; recovery depends on the Secret Manager copy | The KMS key is the durable copy (scheduled-destroy window before deletion); escrow copy still advised | Same | +| Insider at Orca with Cloudflare admin | Can decrypt | Can decrypt, but leaves an audit trail in a system they may not administer | Same | + +So a KMS does **not** make the store zero-knowledge and the white paper must not say so. What it buys: an audit trail on key use, fast revocation, a non-exportable root, native rotation, and separation of duties between whoever runs Cloudflare and whoever runs GCP IAM. + +## Options + +### A. GCP Cloud KMS (recommended) + +A symmetric `ENCRYPT_DECRYPT` key in the GCP project that already holds Secret Manager. The Worker calls `cryptoKeys.decrypt` over REST. + +- Worker auth to GCP: a service-account key held as a Worker secret, used to sign a JWT with WebCrypto and exchange it for a one-hour access token (cached). The account gets `roles/cloudkms.cryptoKeyDecrypter` on that one key and nothing else. If Cloudflare offers workload identity (an OIDC token GCP can federate) by the time this is built, use that instead and hold no long-lived GCP credential. I have not confirmed that it exists. +- Turn on Data Access audit logs for Cloud KMS, or the decrypt calls are not logged. Alert on decrypt volume. +- Protection level: software is enough to start. HSM (FIPS 140-2 Level 3) is a per-key setting and a later switch if a customer asks. + +### B. Cloudflare + +- **Worker secrets** (today): encrypted at rest by Cloudflare, value delivered in plaintext to the Worker, usable by anyone who can deploy to the Worker. +- **Secrets Store**: account-level secrets with their own roles, bound into Workers. Better access control and one place to rotate, but the Worker still receives the raw key, there is no unwrap API and no per-use audit. It does not change the table above. +- I know of no Cloudflare product that offers a general encrypt/decrypt API with a non-exportable key. Keyless SSL and Geo Key Manager cover TLS keys only. + +Cloudflare alone can tidy access control but cannot close the gap. + +### C. AWS KMS + +Same shape as A. Orca Cloud has no AWS footprint, so it adds a vendor for no gain. + +### D. Fetch the KEK from GCP Secret Manager at cold start + +Cheap, since the secret already exists. It gives an access log and revocation, but the KEK stays exportable to every principal with the accessor role, and it is a dead end for phase 2. Not recommended over A. + +## Recommended design + +### Phase 1: KMS-wrapped KEK, unwrapped once per isolate + +- The KEK stays a 32-byte AES key and the row format does not change. Instead of `SECRETS_KEK` (plaintext), the Worker gets `SECRETS_KEK_WRAPPED` (the KEK encrypted by the KMS key). +- On first use in an isolate the Worker calls KMS decrypt once, imports the result as a non-extractable `CryptoKey`, zeroes the bytes and caches the keyring in memory. `keyringFor(env)` in `secrets.ts` already caches per isolate and already returns a promise, so the seam exists. +- Retired KEKs are wrapped the same way. +- Row DEKs are still unwrapped locally, so session start costs no extra KMS calls. + +This is the smallest change that moves the long-lived plaintext key out of Cloudflare config. Its limit: one successful KMS call yields the KEK, which opens everything. The call is logged and revocable, but the blast radius of a single call is the whole store. + +### Phase 2: per-org keys, KMS as the root + +- Add a key per owner account (org or user), wrapped by KMS and stored in D1. Row DEKs are wrapped by their owner's key. +- The Worker unwraps an org key on demand and caches it: one KMS call per org per isolate per cache window. +- No exportable key covers the whole store. The audit log shows per-org access. It also opens the door to customer-managed keys (a customer supplies the KMS key for their org). +- With v2 rows this migration is a DEK re-wrap done by the existing sweep. No value is decrypted. + +Do phase 2 only when an enterprise customer asks for per-tenant keys or customer-managed keys. Phase 1 addresses the gap in the white paper. + +### Why not call KMS per row + +A session start opens up to about 100 secrets per owner across three scopes. One KMS call each would add seconds to start and make every start depend on KMS. Per-row DEKs under a cached wrapping key keep KMS off the hot path. + +## Latency (expected, unmeasured) + +- Cloud KMS decrypt is typically tens of milliseconds from a nearby region. Workers run at the user's edge, so a distant PoP could see 100 to 300 ms. Use a multi-region key location (`us` or `global`). +- The first call in an isolate also pays the token exchange, roughly another 100 ms. +- The cost is paid once per isolate per cache window, on the first secrets read or write. Session start already takes seconds, so this is not visible there. The secrets settings page would see it on a cold isolate only. +- Measure in dev before deciding the cache window. + +## Cost (list prices from memory) + +- Software key version: about $0.06 per month. HSM: about $1 to $2.50 per month. +- Operations: about $0.03 per 10,000. +- With per-isolate caching this is a few dollars per month at most. Audit log storage is the larger line and is still small. + +## Failure modes + +| Failure | Behaviour to build | +|---|---| +| KMS unreachable, isolate already has the keyring | Keep serving from cache. Extend the cache past its window while KMS is down (stale-on-error), with a ceiling of a few hours | +| KMS unreachable, cold isolate | Secrets routes answer 503. Session start must **not** proceed silently without secrets, which is what a missing `SECRETS_KEK` does today: retry with backoff, then fail the start with a clear error | +| Service-account key revoked or expired | Same as unreachable. Alert on the first failed unwrap | +| KMS key version disabled (deliberate revocation) | Cached isolates keep working until the cache window ends. For an emergency, also redeploy the Worker, which drops every isolate | +| KMS key destroyed | Scheduled destruction has a waiting period (30 days by default) in which it can be restored. Keep the escrow copy below as a second line | +| Wrong wrapped blob deployed | `GET /v1/internal/secrets/kek` shows rows under an unknown KEK, and a dry-run rewrap reports rows that will not open. Run both after every key change | + +Cache window: the trade is revocation speed against KMS dependence. Start at 15 minutes with stale-on-error. + +## Backup and escrow + +- Phase 1: the KEK still exists as bytes. Keep the Secret Manager copy from runbook 1.6a as escrow, with read access cut down to the break-glass group. The wrapped blob is not sensitive without KMS access. +- Phase 2: org keys live in D1 wrapped by KMS. The KMS key is the root and cannot be exported, so backup means protecting the key from deletion (IAM deny on destroy, the scheduled-destroy window, an org policy) and covering D1 with its normal backups. + +## Migration plan (phase 1) + +No step below has been run. Steps 1 and 2 are infra changes for a human to approve. + +1. Terraform, dev project first: key ring and key, service account with decrypt on that key only, Data Access audit logs for KMS, an alert on decrypt count. +2. Wrap the existing KEK: read it from Secret Manager and call KMS encrypt in one pipeline so it is never printed. Store the result as `SECRETS_KEK_WRAPPED`. Store the service-account credential as a Worker secret. +3. Ship the Worker change: prefer `SECRETS_KEK_WRAPPED` plus KMS, fall back to `SECRETS_KEK`. The KEK id stays the same, so no row changes. +4. Verify in dev: `GET /v1/internal/secrets/kek` shows no unknown ids, and `POST /v1/internal/secrets/rewrap {"dryRun":true}` run to completion reports `failed: 0`. +5. Delete the `SECRETS_KEK` Worker secret. The plaintext KEK is now only in Worker memory and in escrow. +6. Rotate once: generate a new KEK, wrap it, make it active, list the old one as retired, run the sweep, drop the retired entry. This retires the key that once sat in Cloudflare config as plaintext. The SEC-3 sweep is what makes this step cheap. +7. Repeat 1 to 6 for prod, when launch gating allows it. + +Rollback at any step before 5: remove `SECRETS_KEK_WRAPPED`; the Worker falls back to `SECRETS_KEK`. + +Estimated work: about 2 days of Worker code and tests (GCP token exchange, KMS client, cache with stale-on-error, fail-closed session start), plus the Terraform. + +## Why this was not implemented now + +It needs a KMS key, a service account and IAM bindings. Those are infra changes, which the brief ruled out. A prototype against a fake KMS would only test our own mock. The code seam is small and already in place (`keyringFor` is async and cached), so nothing in the SEC-3 PR has to be redone. + +## Before phase 1 + +- Turn `SECRETS_SEAL_FORMAT=v2` on (dev once every preview branch contains the SEC-3 PR; prod from day one) and run the sweep once so every row is v2. +- Confirm who can read the Secret Manager copy of the KEK and record it in the runbook. diff --git a/cloud/docs/vercel-driver-design.md b/cloud/docs/vercel-driver-design.md index 9211dca6ad8..b198f4e5c71 100644 --- a/cloud/docs/vercel-driver-design.md +++ b/cloud/docs/vercel-driver-design.md @@ -210,8 +210,11 @@ Product decision: "API-key injection" in the Conductor parity matrix (form 2 = t (`src/secrets-crypto.ts`): per-secret random AES-256-GCM DEK; DEK wrapped by the KEK (`SECRETS_KEK`, base64 32 bytes, Worker secret only; id `SECRETS_KEK_ID`, default `k1`). Both layers take `["orca-secret-v1", layer, kekId, scope, ownerId, name]` as AAD, so a row copied onto another owner/name fails to open (skipped at injection, - logged by name). Rotation: set the new KEK + id, move the old one into `SECRETS_KEK_RETIRED` (`{"k1":"…"}`); rows - re-seal on their next write. + logged by name). Rotation: set the new KEK + id, move the old one into `SECRETS_KEK_RETIRED` (`{"k1":"…"}`) in the + same deploy, then run the operator sweep `POST /v1/internal/secrets/rewrap` until `done` (`src/secrets-rewrap.ts`; + it also covers `agent_credentials` and `github_user_tokens`). `GET /v1/internal/secrets/kek` reports when the + retired id is unreferenced and can be dropped. With `SECRETS_SEAL_FORMAT=v2` rows drop the KEK id from the value + layer's AAD, so the sweep re-wraps DEKs without decrypting values. KMS plan: `cloud/docs/sec-kms-design.md`. - **API** (desktop JWT): `GET /v1/secrets?scope=org|personal|repo[&repo=owner/name]`, `PUT /v1/secrets/:scope/:NAME` `{value}`, `DELETE …`. Write-only values: responses carry name, scope, kind, updatedAt, updatedBy and a last-4 hint for values ≥16 chars. Owners: org = the org account (`org:`), personal = `user:`, repo =