From e73770c2478634a86c6360bbcd4c8b5444150eb5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 22 Apr 2026 13:05:08 -0700 Subject: [PATCH] fix: derive debug signing key deterministically from JWT_SECRET (#8917) Previously each API replica generated a random Ed25519 signing key at startup (unless DEBUG_SIGNING_KEY_SEED was set). In multi-replica deployments this caused "Invalid JWT signature" rejections in the multiplayer server: the browser could sign a token on pod A while `windmill-extra` had cached the JWKS public key from pod B. Derive the seed deterministically from the DB-backed JWT_SECRET using SHA-256 with a domain-separation tag so all pods agree without coordination. Re-derive on JWT_SECRET rotation. The DEBUG_SIGNING_KEY_SEED env var is still honored as an override. Co-authored-by: Claude Opus 4.7 (1M context) --- backend/Cargo.lock | 1 - backend/src/monitor.rs | 4 ++ backend/windmill-api-debug/Cargo.toml | 1 - backend/windmill-api-debug/src/lib.rs | 70 +++++++++++++++++++-------- backend/windmill-api/src/lib.rs | 2 + 5 files changed, 57 insertions(+), 21 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index da8ae12243..b703a76711 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16242,7 +16242,6 @@ dependencies = [ "ed25519-dalek", "hex", "lazy_static", - "rand 0.9.0", "serde", "serde_json", "sha2 0.10.9", diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 175965e4ad..c360bb7b8d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -3879,6 +3879,10 @@ pub async fn reload_jwt_secret_setting(db: &DB) -> error::Result<()> { JWT_SECRET.store(std::sync::Arc::new(jwt_secret)); + // The debug signing key is derived from JWT_SECRET, so re-derive it here so + // rotation propagates to /api/debug/* signing without requiring a restart. + windmill_api::reload_debug_signing_key().await; + Ok(()) } diff --git a/backend/windmill-api-debug/Cargo.toml b/backend/windmill-api-debug/Cargo.toml index de94a3aba3..8531407416 100644 --- a/backend/windmill-api-debug/Cargo.toml +++ b/backend/windmill-api-debug/Cargo.toml @@ -18,7 +18,6 @@ chrono.workspace = true ed25519-dalek.workspace = true hex.workspace = true lazy_static.workspace = true -rand.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true diff --git a/backend/windmill-api-debug/src/lib.rs b/backend/windmill-api-debug/src/lib.rs index baece9e8e9..61ec59be75 100644 --- a/backend/windmill-api-debug/src/lib.rs +++ b/backend/windmill-api-debug/src/lib.rs @@ -37,7 +37,7 @@ use tokio::sync::RwLock; use uuid::Uuid; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ - db::UserDB, error::JsonResult, jobs::JobKind, scripts::ScriptLang, + db::UserDB, error::JsonResult, jobs::JobKind, jwt::JWT_SECRET, scripts::ScriptLang, users::username_to_permissioned_as, }; @@ -48,35 +48,67 @@ pub const DEBUG_TOKEN_TTL_SECS: i64 = 60; lazy_static::lazy_static! { /// Ed25519 signing key for debug tokens. - /// Generated at startup if not provided via environment variable. + /// + /// Derived deterministically from the instance `JWT_SECRET` so all API + /// replicas agree on the same key without coordination. Refreshed via + /// [`reload_debug_signing_key`] when `JWT_SECRET` is reloaded. static ref DEBUG_SIGNING_KEY: Arc>> = Arc::new(RwLock::new(None)); } -/// Initialize the debug signing key. -/// Call this at server startup. -pub async fn init_debug_signing_key() { - let mut key_guard = DEBUG_SIGNING_KEY.write().await; +/// Domain-separation tag so the debug Ed25519 seed cannot be confused with +/// any other HMAC/HS256 usage of `JWT_SECRET`. +const DEBUG_KEY_DERIVATION_TAG: &[u8] = b"windmill-debug-signing-key:v1:"; - // Check if key is provided via environment variable (base64-encoded seed) +fn derive_signing_key_from_jwt_secret(jwt_secret: &str) -> SigningKey { + let mut hasher = Sha256::new(); + hasher.update(DEBUG_KEY_DERIVATION_TAG); + hasher.update(jwt_secret.as_bytes()); + let seed: [u8; 32] = hasher.finalize().into(); + SigningKey::from_bytes(&seed) +} + +fn compute_debug_signing_key() -> Option { + // Env var override: base64url-encoded 32-byte seed. Useful for tests or + // advanced deployments that want to pin the key independently. if let Ok(seed_b64) = std::env::var("DEBUG_SIGNING_KEY_SEED") { - if let Ok(seed_bytes) = URL_SAFE_NO_PAD.decode(&seed_b64) { - if seed_bytes.len() >= 32 { + match URL_SAFE_NO_PAD.decode(&seed_b64) { + Ok(seed_bytes) if seed_bytes.len() >= 32 => { let mut seed = [0u8; 32]; seed.copy_from_slice(&seed_bytes[..32]); - *key_guard = Some(SigningKey::from_bytes(&seed)); - tracing::info!("Debug signing key loaded from environment"); - return; + tracing::info!("Debug signing key loaded from DEBUG_SIGNING_KEY_SEED"); + return Some(SigningKey::from_bytes(&seed)); } + _ => tracing::warn!( + "Invalid DEBUG_SIGNING_KEY_SEED (expect base64url-encoded 32+ bytes); falling back to JWT_SECRET derivation" + ), } - tracing::warn!("Invalid DEBUG_SIGNING_KEY_SEED, generating new key"); } - // Generate a new random key using rand - let mut seed = [0u8; 32]; - rand::Rng::fill(&mut rand::rng(), &mut seed); - let signing_key = SigningKey::from_bytes(&seed); - tracing::info!("Generated new debug signing key"); - *key_guard = Some(signing_key); + let jwt_secret = JWT_SECRET.load(); + if jwt_secret.is_empty() { + return None; + } + Some(derive_signing_key_from_jwt_secret(&jwt_secret)) +} + +/// Initialize the debug signing key. Call once at server startup, after +/// `reload_jwt_secret_setting` so `JWT_SECRET` is populated. +pub async fn init_debug_signing_key() { + reload_debug_signing_key().await; +} + +/// Recompute and store the debug signing key. Call after `JWT_SECRET` is +/// (re)loaded so rotation propagates without a pod restart. +pub async fn reload_debug_signing_key() { + let key = compute_debug_signing_key(); + if key.is_none() { + tracing::warn!( + "Debug signing key not initialized: JWT_SECRET is empty and DEBUG_SIGNING_KEY_SEED is not set. /api/debug/* endpoints will return an error." + ); + } else { + tracing::info!("Debug signing key initialized from JWT_SECRET"); + } + *DEBUG_SIGNING_KEY.write().await = key; } pub fn global_service() -> Router { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index c11f89a1c4..042aa98a14 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -216,6 +216,8 @@ pub use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; pub use windmill_common::utils::{COOKIE_DOMAIN, IS_SECURE}; +pub use windmill_api_debug::reload_debug_signing_key; + #[cfg(feature = "oauth2")] pub use windmill_oauth::OAUTH_CLIENTS;