From 4fef1195adaa9fa036a219884bd6c996460ca37f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 2 Sep 2026 22:29:01 +0200 Subject: [PATCH] fix: apply object-storage test SSRF validation to all non-super-admins (#10933) * fix: apply object-storage test SSRF validation to all non-super-admins Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: name the job-token case in object-storage test rejections Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: run object-storage connection tests with a short-lived user token Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: test object-storage resources from the browser, mint a token only for the worker test Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: resolve variable and resource references before the browser-side object-storage test Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: bound the browser-side object-storage test to 15s Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: explain object-storage test rejections and name the way out Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg * fix: keep the server-resolved address out of the object-storage test rejection Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01FJLqsE5br9r5e7qy8ULwUg --------- Co-authored-by: Claude Fable 5.1 --- backend/tests/object_storage_test_ssrf.rs | 97 ++++++ backend/windmill-api-settings/src/lib.rs | 93 ++++-- .../ObjectStoreConfigSettings.svelte | 1 + .../src/lib/components/TestConnection.svelte | 286 ++++++++++++------ 4 files changed, 354 insertions(+), 123 deletions(-) create mode 100644 backend/tests/object_storage_test_ssrf.rs diff --git a/backend/tests/object_storage_test_ssrf.rs b/backend/tests/object_storage_test_ssrf.rs new file mode 100644 index 0000000000..aa9eaaf74c --- /dev/null +++ b/backend/tests/object_storage_test_ssrf.rs @@ -0,0 +1,97 @@ +//! `POST /api/settings/test_object_storage_config` runs the probe on the API server and reflects the +//! upstream response, so every non-super-admin must be rejected for private/loopback endpoints and +//! the Filesystem backend on every deployment (`CLOUD_HOSTED` is unset here), while a super admin's +//! Filesystem probe still round-trips. Requires the `parquet` feature, like the route. +#![cfg(feature = "parquet")] + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use windmill_test_utils::*; + +const SUPER_ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const USER_TOKEN: &str = "SECRET_TOKEN_2"; + +async fn test_object_storage( + url: &str, + token: &str, + body: serde_json::Value, +) -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .post(url) + .header("Authorization", format!("Bearer {token}")) + .json(&body) + .send() + .await?; + Ok((resp.status().as_u16(), resp.text().await?)) +} + +#[sqlx::test(fixtures("base"))] +async fn object_storage_test_is_restricted_for_non_super_admins_off_cloud( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let url = format!( + "http://localhost:{}/api/settings/test_object_storage_config", + server.addr.port() + ); + + // A loopback "S3 endpoint" standing in for an internal service: the probe must be rejected + // before the server opens a connection to it. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let internal_port = listener.local_addr()?.port(); + let connected = Arc::new(AtomicBool::new(false)); + tokio::spawn({ + let connected = connected.clone(); + async move { + while listener.accept().await.is_ok() { + connected.store(true, Ordering::SeqCst); + } + } + }); + let internal_s3 = json!({ + "type": "S3", + "bucket": "bucket", + "region": "us-east-1", + "access_key": "key", + "secret_key": "secret", + "endpoint": format!("http://127.0.0.1:{internal_port}"), + "allow_http": true, + "path_style": true, + }); + let (status, body) = test_object_storage(&url, USER_TOKEN, internal_s3).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a loopback endpoint (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + assert!( + !connected.load(Ordering::SeqCst), + "the server must not connect to the rejected endpoint" + ); + + let tmp = tempfile::tempdir()?; + let filesystem = json!({ "type": "Filesystem", "root_path": tmp.path().to_str().unwrap() }); + let (status, body) = test_object_storage(&url, USER_TOKEN, filesystem.clone()).await?; + assert_eq!( + status, 401, + "non-super-admin must be rejected for a Filesystem backend (got {status}): {body}" + ); + assert!( + body.contains("requires a super admin"), + "unexpected rejection: {body}" + ); + + // Super admins keep the unrestricted path. + let (status, body) = test_object_storage(&url, SUPER_ADMIN_TOKEN, filesystem).await?; + assert_eq!( + status, 200, + "super admin must be able to test a Filesystem backend (got {status}): {body}" + ); + Ok(()) +} diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 2cda80f458..18a8d52ac0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -284,15 +284,28 @@ pub async fn test_s3_bucket( use bytes::Bytes; use futures::StreamExt; - // The probe executes on the API server itself. On multi-tenant Cloud that is a shared control - // plane, so we constrain untrusted callers to remove the SSRF / credential-exfiltration / - // local-filesystem surface (see validate_object_storage_test). On self-hosted instances the - // object store usually lives on the local/private network and all authenticated users are - // trusted, so testing there stays unrestricted. Super admins keep the unrestricted path too. + // The probe executes on the API server itself and reflects the upstream response into the + // error, so any authenticated caller could otherwise use it as an SSRF / port-scan primitive + // against the server's network, exfiltrate its ambient credentials, or write to its local + // disk (see validate_object_storage_test). That holds on self-hosted instances as much as on + // Cloud, so only super admins get the unrestricted path. let is_super_admin = windmill_api_auth::is_super_admin_authed(&db, &authed).await?; - let restrict = !is_super_admin && *CLOUD_HOSTED; + let restrict = !is_super_admin; if restrict { - validate_object_storage_test(&test_s3_bucket).await?; + validate_object_storage_test(&test_s3_bucket) + .await + .map_err(|e| match e { + // A job token never counts as a super admin (it is capped at workspace admin), so + // a super admin calling this route from a script is told why rather than that + // they lack a privilege they hold. + error::Error::NotAuthorized(msg) if authed.job_id.is_some() => { + error::Error::NotAuthorized(format!( + "{msg} A job token ($WM_TOKEN) is never treated as a super admin; call \ + this route with a user token instead." + )) + } + e => e, + })?; } let client = build_object_store_from_settings(test_s3_bucket, Some(&db)) @@ -355,8 +368,8 @@ pub async fn test_s3_bucket( } } -// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller on -// Cloud. The probe runs on the shared API server, so without these constraints an authenticated +// Hardening for the object-storage connectivity test by an untrusted (non-super-admin) caller. +// The probe runs on the API server, so without these constraints an authenticated // user could coerce the server into connecting to arbitrary internal endpoints (SSRF), signing // requests with the instance role (credential exfiltration), or reading/writing the server's local // disk (filesystem object store). @@ -366,6 +379,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul opt.as_ref().is_some_and(|s| !s.is_empty()) } + // Every refusal names the way out: the resource usually works in jobs (workers reach the + // endpoint directly), so without it the refusal reads as a broken resource. + const ALTERNATIVE: &str = + "Ask a super admin to run it, or test the resource from a script, which runs on a worker."; + // Reject backends that rely on the server's identity or local filesystem, require explicit // credentials for the rest (so the server never falls back to its own ambient credentials), and // resolve the host the client will actually connect to. We derive the *effective* endpoint here @@ -376,20 +394,25 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul let effective_endpoint: Option = match settings { ObjectSettings::Filesystem(_) => { return Err(error::Error::NotAuthorized( - "Testing a local filesystem object store requires a super admin".to_string(), + "Testing a local filesystem object store requires a super admin: it runs on the \ + Windmill server and reads and writes the server's local disk. Ask a super admin \ + to run it." + .to_string(), )); } ObjectSettings::AwsOidc(_) => { - return Err(error::Error::NotAuthorized( - "Testing OIDC-based object storage requires a super admin".to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing OIDC-based object storage requires a super admin: it runs on the \ + Windmill server with the server's own identity. {ALTERNATIVE}" + ))); } ObjectSettings::S3(s3) => { if !(non_empty(&s3.access_key) && non_empty(&s3.secret_key)) { - return Err(error::Error::NotAuthorized( - "Testing S3 storage without explicit credentials requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing S3 storage without an explicit access key and secret key requires a \ + super admin: it runs on the Windmill server, which would use its own ambient \ + credentials. {ALTERNATIVE}" + ))); } let region = s3 .region @@ -413,10 +436,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul } ObjectSettings::Azure(azure) => { if !non_empty(&azure.access_key) { - return Err(error::Error::NotAuthorized( - "Testing Azure storage without an explicit access key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing Azure storage without an explicit access key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } Some( azure @@ -432,10 +456,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul // otherwise an untrusted caller could probe with the server's identity (the very // SSRF/credential-exfil this function guards against). if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { - return Err(error::Error::NotAuthorized( - "Testing GCS storage without a service account key requires a super admin" - .to_string(), - )); + return Err(error::Error::NotAuthorized(format!( + "Testing GCS storage without a service account key requires a super admin: \ + it runs on the Windmill server, which would use its own ambient credentials. \ + {ALTERNATIVE}" + ))); } // The service-account-key JSON can override the data-plane URL (`gcs_base_url`) and the // OAuth token endpoint (`token_uri`); the GCS client connects to whatever they point at. @@ -492,10 +517,15 @@ async fn validate_public_endpoint(endpoint: &str) -> error::Result<()> { // attempts (a name resolving to both a public and a private address). for addr in addrs { if is_forbidden_ip(addr.ip()) { - return Err(error::Error::NotAuthorized( - "Testing object storage at a private, loopback, or link-local endpoint requires a super admin" - .to_string(), - )); + // The resolved address stays out of the message: it is the server's resolver's + // answer, and this message is only ever shown to the caller being constrained. + return Err(error::Error::NotAuthorized(format!( + "Testing object storage at '{host}', which resolves to a private, loopback, or \ + link-local address, requires a super admin: this test runs on the Windmill \ + server, which is not allowed to probe internal addresses for non-super-admins. \ + Ask a super admin to run it, or test the resource from a script, which runs on \ + a worker." + ))); } } Ok(()) @@ -2008,7 +2038,10 @@ struct CachedResourceType { /// decodes the on-disk cache, where an absent key means "written before the /// column, leave the stored extension alone" and an explicit null means the hub /// dropped it. Plain serde folds both into `None`. - #[serde(default, deserialize_with = "windmill_common::more_serde::double_option")] + #[serde( + default, + deserialize_with = "windmill_common::more_serde::double_option" + )] format_extension: Option>, } diff --git a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte index c7f7065565..14658a4228 100644 --- a/frontend/src/lib/components/ObjectStoreConfigSettings.svelte +++ b/frontend/src/lib/components/ObjectStoreConfigSettings.svelte @@ -305,6 +305,7 @@ resourceType="s3_bucket" workspaceOverride="admins" buttonTextOverride="Test from a worker" + viaWorker /> diff --git a/frontend/src/lib/components/TestConnection.svelte b/frontend/src/lib/components/TestConnection.svelte index 37e3e152db..6a58af8031 100644 --- a/frontend/src/lib/components/TestConnection.svelte +++ b/frontend/src/lib/components/TestConnection.svelte @@ -1,5 +1,13 @@