From 0dfd746dc7fa13b3fac4fd246f24e7f11dc6231a Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Tue, 3 Nov 2026 06:50:19 +0100 Subject: [PATCH] feat: verify all stored credential domains from a bounded consistent database snapshot --- .../aws/runtime/checks/credentials.cjs | 35 ++++++++++++------- .../aws/runtime/checks/credentials.test.cjs | 29 +++++++++++++++ .../aws/runtime/checks/dependencies.cjs | 2 +- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/deploy/terraform/aws/runtime/checks/credentials.cjs b/deploy/terraform/aws/runtime/checks/credentials.cjs index 41280caed..049aaecf1 100644 --- a/deploy/terraform/aws/runtime/checks/credentials.cjs +++ b/deploy/terraform/aws/runtime/checks/credentials.cjs @@ -45,14 +45,17 @@ async function verifyStoredCredentials(db, kms, DecryptCommand, env) { try { const wrapped = await db.query("SELECT organization_id,encrypted_data_key FROM organization_encrypted_keys"); if (!wrapped.rowCount) throw new Error("No data key available to verify"); - for (const row of wrapped.rows) { - const out = await kms.send(new DecryptCommand({ CiphertextBlob: Buffer.from(row.encrypted_data_key, "base64") })); - try { - if (out.Plaintext?.length !== 32) throw new Error("Invalid data key"); - organizationKeys.set(row.organization_id, Buffer.from(out.Plaintext)); - } finally { out.Plaintext?.fill(0); } + for (let offset = 0; offset < wrapped.rowCount; offset += 5) { + const batch = await Promise.allSettled(wrapped.rows.slice(offset, offset + 5).map(async (row) => { + const out = await kms.send(new DecryptCommand({ CiphertextBlob: Buffer.from(row.encrypted_data_key, "base64") })); + try { + if (out.Plaintext?.length !== 32) throw new Error("Invalid data key"); + organizationKeys.set(row.organization_id, Buffer.from(out.Plaintext)); + } finally { out.Plaintext?.fill(0); } + })); + if (batch.some((result) => result.status === "rejected")) throw new Error("Organization key verification failed"); } - const fields = await db.query(` + await db.query(`DECLARE credential_fields NO SCROLL CURSOR FOR SELECT v AS value,true AS mailbox,false AS signing_secret,a.organization_id FROM email_accounts_smtp_imap m JOIN email_accounts a ON a.id=m.email_account_id CROSS JOIN LATERAL (VALUES (smtp_host),(smtp_user),(smtp_password),(imap_host),(imap_user),(imap_password)) fields(v) @@ -65,16 +68,22 @@ async function verifyStoredCredentials(db, kms, DecryptCommand, env) { UNION ALL SELECT secret,false,true,NULL::uuid FROM webhook_endpoints WHERE secret IS NOT NULL AND secret<>'' UNION ALL SELECT webhook_secret,false,true,NULL::uuid FROM oauth_applications WHERE webhook_secret IS NOT NULL AND webhook_secret<>'' UNION ALL SELECT token,false,false,NULL::uuid FROM cloud_link WHERE token IS NOT NULL AND token<>'' - UNION ALL SELECT refresh_token,false,false,NULL::uuid FROM mailbox_erasures WHERE refresh_token IS NOT NULL AND refresh_token<>'' + UNION ALL SELECT refresh_token,true,false,organization_id FROM mailbox_erasures WHERE refresh_token IS NOT NULL AND refresh_token<>'' `); - let legacySigningFields = 0; - for (const row of fields.rows) { - if (row.signing_secret && row.value.startsWith("whsec_")) legacySigningFields++; - else verifyCredentialField(row, keys, organizationKeys); + let legacySigningFields = 0, credentialFields = 0; + for (;;) { + const fields = await db.query("FETCH FORWARD 1000 FROM credential_fields"); + if (!fields.rowCount) break; + credentialFields += fields.rowCount; + for (const row of fields.rows) { + if (row.signing_secret && row.value.startsWith("whsec_")) legacySigningFields++; + else verifyCredentialField(row, keys, organizationKeys); + } } + await db.query("CLOSE credential_fields"); const mfa = await db.query("SELECT totp_secret_sealed FROM user_totp_settings WHERE totp_secret_sealed IS NOT NULL AND totp_secret_sealed<>''"); for (const row of mfa.rows) verifyCredentialCipher(mfaKeys, row.totp_secret_sealed, "base64"); - return { credential_fields: fields.rowCount - legacySigningFields, legacy_signing_fields: legacySigningFields, mfa_enrollments: mfa.rowCount, organization_keys: wrapped.rowCount, kms_decrypt_as_task_role: true }; + return { credential_fields: credentialFields - legacySigningFields, legacy_signing_fields: legacySigningFields, mfa_enrollments: mfa.rowCount, organization_keys: wrapped.rowCount, kms_decrypt_as_task_role: true }; } finally { [...keys, ...mfaKeys, ...organizationKeys.values()].forEach((key) => key.fill(0)); } diff --git a/deploy/terraform/aws/runtime/checks/credentials.test.cjs b/deploy/terraform/aws/runtime/checks/credentials.test.cjs index 3b73f34e7..488e1e00f 100644 --- a/deploy/terraform/aws/runtime/checks/credentials.test.cjs +++ b/deploy/terraform/aws/runtime/checks/credentials.test.cjs @@ -31,7 +31,14 @@ test("stored validation unwraps keys, verifies both domains and MFA, and clears let plaintext; const kms = { async send() { plaintext = Buffer.from(org); return { Plaintext: plaintext }; } }; let corrupt = false; + let fetched = false; const db = { async query(sql) { + if (sql.startsWith("DECLARE")) { fetched = false; return { rows: [], rowCount: 0 }; } + if (sql.startsWith("CLOSE")) return { rows: [], rowCount: 0 }; + if (sql.startsWith("FETCH")) { + if (fetched) return { rows: [], rowCount: 0 }; + fetched = true; + } const rows = sql.includes("FROM organization_encrypted_keys") ? [{ organization_id: "org-a", encrypted_data_key: "Zml4dHVyZQ==" }] : sql.includes("FROM user_totp_settings") ? [{ totp_secret_sealed: seal(mfa, "fixture", "base64") }] : [{ mailbox: true, organization_id: "org-a", value: "dek1:" + seal(corrupt ? instance : org, "fixture", "base64") }, @@ -46,3 +53,25 @@ test("stored validation unwraps keys, verifies both domains and MFA, and clears await assert.rejects(verifyStoredCredentials(db, kms, DecryptCommand, env), /Ciphertext verification failed/); assert.ok(plaintext.every((byte) => byte === 0)); }); + +test("a failed KMS batch waits for every decrypt and clears returned plaintext", async () => { + const plaintexts = [], rows = Array.from({ length: 7 }, (_, i) => ({ organization_id: String(i), encrypted_data_key: Buffer.from(String(i)).toString("base64") })); + let active = 0, settled = 0, peak = 0; + const kms = { async send(command) { + peak = Math.max(peak, ++active); + try { + if (command.input.CiphertextBlob.toString() === "0") throw new Error("fixture KMS failure"); + await new Promise((resolve) => setTimeout(resolve, 10)); + const plaintext = crypto.randomBytes(32); + plaintexts.push(plaintext); + return { Plaintext: plaintext }; + } finally { active--; settled++; } + } }; + class DecryptCommand { constructor(input) { this.input = input; } } + await assert.rejects(verifyStoredCredentials({ async query() { return { rows, rowCount: rows.length }; } }, kms, DecryptCommand, + { CREDENTIALS_ENCRYPTION_KEY: crypto.randomBytes(32).toString("hex"), AUTH_SECRET: "fixture" }), /Organization key verification failed/); + assert.ok(peak <= 5); + assert.equal(settled, 5); + assert.equal(active, 0); + assert.ok(plaintexts.every((key) => key.every((byte) => byte === 0))); +}); diff --git a/deploy/terraform/aws/runtime/checks/dependencies.cjs b/deploy/terraform/aws/runtime/checks/dependencies.cjs index c789d0a99..e5e2e52b9 100644 --- a/deploy/terraform/aws/runtime/checks/dependencies.cjs +++ b/deploy/terraform/aws/runtime/checks/dependencies.cjs @@ -32,7 +32,7 @@ async function withDatabase(connectionString, fn) { client.on("error", () => {}); try { await client.connect(); - await client.query("BEGIN READ ONLY"); + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"); await client.query("SET LOCAL statement_timeout = '15s'"); return await fn(client); } finally {