diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index cdfda7e0d4..73f955131e 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -295,4 +295,4 @@ jobs: # never reaches it. Pin the target dir (matching the cache step above) so # its own tests run off this compile rather than a second bundled build. (cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal) - DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger --all -- --nocapture --test-threads=10 + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger,openidconnect --all -- --nocapture --test-threads=10 diff --git a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json b/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json deleted file mode 100644 index b647ec29fa..0000000000 --- a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "permissioned_as_email", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parent_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true, - false, - true - ] - }, - "hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496" -} diff --git a/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json new file mode 100644 index 0000000000..446e9da6df --- /dev/null +++ b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) ON CONFLICT (name) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97" +} diff --git a/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json new file mode 100644 index 0000000000..b3a2ea6a05 --- /dev/null +++ b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS (SELECT 1 FROM global_settings WHERE name = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a" +} diff --git a/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json new file mode 100644 index 0000000000..c8134447e6 --- /dev/null +++ b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now() WHERE global_settings.value IS DISTINCT FROM EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aa3d601492..610768134f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15536,6 +15536,7 @@ dependencies = [ "windmill-api-scripts", "windmill-api-settings", "windmill-api-users", + "windmill-audit", "windmill-autoscaling", "windmill-common", "windmill-dep-map", @@ -15658,12 +15659,14 @@ dependencies = [ "native-tls", "openidconnect", "openssl", + "p256", "pin-project", "postgres-native-tls 0.5.3", "prometheus", "quick-xml", "quick_cache", "rand 0.9.5", + "rcgen", "regex", "reqwest 0.13.5", "rsa", @@ -15684,6 +15687,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-postgres", + "tokio-rustls 0.26.6", "tokio-stream", "tokio-util", "tower 0.5.3", @@ -16346,6 +16350,7 @@ dependencies = [ "opentelemetry-otlp 0.30.0", "opentelemetry-semantic-conventions 0.30.0", "opentelemetry_sdk 0.30.0", + "p256", "pep440_rs", "phf 0.11.3", "pin-project-lite", @@ -16630,6 +16635,7 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "windmill-audit", "windmill-common", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index d627328e27..6f2420b8a3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -156,7 +156,7 @@ bedrock = ["windmill-ai/bedrock", "windmill-api/bedrock", "windmill-worker/bedro mqtt_trigger = ["windmill-api/mqtt_trigger"] amqp_trigger = ["windmill-api/amqp_trigger"] native_trigger = ["windmill-api/native_trigger"] -sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"] +sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "openidconnect"] gcp_trigger = ["windmill-api/gcp_trigger"] azure_trigger = ["windmill-api/azure_trigger"] smtp = ["windmill-api/smtp", "windmill-common/smtp", "windmill-queue/smtp"] @@ -281,6 +281,7 @@ windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-scripts.workspace = true windmill-api-settings.workspace = true +windmill-audit.workspace = true windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } @@ -594,6 +595,7 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +p256 = "0.13" spki = { version = "0.7", features = ["pem"] } pkcs1 = "0.7" aes-gcm = "0.10.3" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 51f87e8879..699243fc55 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -06083a1d8060b2eb92df954c0aef4a9e7d857e28 \ No newline at end of file +6dedb4687f591a88aba1503d55a001fcb0ab128a diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 10929e409a..341906a3e3 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -235,7 +235,7 @@ lazy_static::lazy_static! { // -- $name (type) = default static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \(([A-Za-z0-9_\[\]]+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); - static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -254,6 +254,12 @@ fn parsed_default(parsed_typ: &Typ, default: String) -> Option= 2 && default.starts_with("'") && default.ends_with("'") => { Some(json!(&default[1..default.len() - 1])) } + Typ::List(_) => Some( + serde_json::from_str::(&default) + .ok() + .filter(|v| v.is_array()) + .unwrap_or_else(|| json!(default)), + ), _ => Some(json!(default)), } } @@ -1039,7 +1045,11 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ { } pub fn parse_snowflake_typ(typ: &str) -> Typ { + if let Some(base_typ) = typ.strip_suffix("[]") { + return Typ::List(Box::new(parse_snowflake_typ(base_typ))); + } match typ { + "array" => Typ::List(Box::new(Typ::Unknown)), "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None), "binary" | "varbinary" => Typ::Bytes, "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz" @@ -1591,6 +1601,39 @@ SELECT ?; Ok(()) } + #[test] + fn test_parse_snowflake_array_types() -> anyhow::Result<()> { + let code = r#" +-- ? groups (varchar[]) +-- ? ids (int[]) = [1, 2] +-- ? anything (ARRAY) +SELECT 1; +"#; + let args = parse_snowflake_sig(code)?.args; + assert_eq!(args[1].default, Some(json!([1, 2]))); + let got = args + .iter() + .map(|a| (a.name.as_str(), a.otyp.as_deref(), &a.typ)) + .collect::>(); + assert_eq!( + got, + vec![ + ( + "groups", + Some("varchar[]"), + &Typ::List(Box::new(Typ::Str(None))) + ), + ("ids", Some("int[]"), &Typ::List(Box::new(Typ::Int))), + ( + "anything", + Some("array"), + &Typ::List(Box::new(Typ::Unknown)) + ), + ] + ); + Ok(()) + } + #[test] fn test_parse_mssql_sig() -> anyhow::Result<()> { let code = r#" diff --git a/backend/src/main.rs b/backend/src/main.rs index b8fdd9353f..fd953213a4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -437,6 +437,8 @@ fn print_help() { println!(" cache-rt Pre-cache hub resource types"); println!(" sync-config Sync instance config from a YAML file to the database"); println!(" operator Run the Kubernetes operator (watches a ConfigMap)"); + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + println!(" oidc-signer (ee) Check an external OIDC signer, or run a file-backed one for tests"); println!(); println!("Environment variables (name = default):"); println!(" DATABASE_URL = The Postgres database url."); @@ -471,6 +473,38 @@ fn print_help() { println!("- At startup, Windmill logs currently set configuration keys for visibility."); } +async fn otel_environment(conn: &Connection) -> String { + if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { + environment + } else { + load_base_url(conn) + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string() + } +} + +/// Tracing for a subcommand that writes instance config. It never runs the server startup +/// that loads the OTel settings, and its audit events are exported under them. +/// +/// The subscriber it installs depends on those settings, so `early_logs` is what prints +/// the events of connecting and loading them; it gives way once the real one is in. +async fn init_declarative_writer_tracing( + db: &windmill_common::DB, + service: &str, + early_logs: tracing::subscriber::DefaultGuard, +) { + load_otel(db).await; + let environment = otel_environment(&Connection::Sql(db.clone())).await; + drop(early_logs); + windmill_common::tracing_init::initialize_subcommand_tracing(service, &HOSTNAME, &environment); +} + async fn windmill_main() -> anyhow::Result<()> { let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -574,7 +608,6 @@ async fn windmill_main() -> anyhow::Result<()> { return Ok(()); } "sync-config" => { - tracing_subscriber::fmt::init(); let path = std::env::args().nth(2).unwrap_or_else(|| { eprintln!("Usage: windmill sync-config "); std::process::exit(1); @@ -588,15 +621,40 @@ async fn windmill_main() -> anyhow::Result<()> { windmill_common::instance_config::resolve_env_refs(&mut config.global_settings) .map_err(|var| anyhow::anyhow!("environment variable '{var}' not found"))?; + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Connecting to database..."); let db = crate::db_connect::initial_connection().await?; - config.sync_to_db(&db).await?; + init_declarative_writer_tracing(&db, "windmill-sync-config", early_logs).await; + + let author = windmill_audit::instance_config::declarative_sync_author(); + let synced = config + .sync_to_db( + &db, + &windmill_audit::instance_config::InstanceConfigAuditor { + db: &db, + author: &author, + source: Some( + windmill_audit::instance_config::DeclarativeSource::SyncConfig, + ), + }, + ) + .await; + // Also on failure: the settings written before it are already audited, and the + // process exits before the exporter's next scheduled batch. + windmill_common::otel_oss::flush_otel_logs().await; + synced?; tracing::info!("Synced instance config from {path}"); return Ok(()); } + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + "oidc-signer" => { + tracing_subscriber::fmt::init(); + windmill_api::oidc_signer_tools_ee::run_cli(std::env::args().skip(2).collect()).await?; + return Ok(()); + } #[cfg(feature = "operator")] "operator" => { - tracing_subscriber::fmt::init(); + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Starting Windmill Kubernetes operator..."); tracing::info!("Connecting to database..."); @@ -609,6 +667,7 @@ async fn windmill_main() -> anyhow::Result<()> { operator_killpill_rx, ) .await?; + init_declarative_writer_tracing(&db, "windmill-operator", early_logs).await; #[cfg(all(feature = "enterprise", feature = "private"))] tokio::spawn(async move { @@ -735,19 +794,7 @@ async fn windmill_main() -> anyhow::Result<()> { (Connection::Sql(db), None, None) }; - let environment = if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { - environment - } else { - load_base_url(&conn) - .await - .unwrap_or_else(|_| "local".to_string()) - .trim_start_matches("https://") - .trim_start_matches("http://") - .split(".") - .next() - .unwrap_or_else(|| "local") - .to_string() - }; + let environment = otel_environment(&conn).await; let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); @@ -1003,6 +1050,15 @@ Windmill Community Edition {GIT_VERSION} // empty script cache, refilled from the database. windmill_common::cache::script::clear(); + // Before the first load: it can sign a token for instance object storage. + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + windmill_common::oidc_ee::init_oidc_signing( + &conn, + server_mode, + killpill_rx.resubscribe(), + ) + .await?; + initial_load( &conn, killpill_tx.clone(), @@ -2095,6 +2151,9 @@ async fn process_notify_event( SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => { // Read by the daily resource type sync each time it runs } + "oidc_external_signer" => { + // Written by servers that sign with an external signer, read when signing + } "plain_emails_telemetry" => { let enabled = sqlx::query_scalar!( "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'" diff --git a/backend/tests/app_ctx_inputs.rs b/backend/tests/app_ctx_inputs.rs new file mode 100644 index 0000000000..cf386adc89 --- /dev/null +++ b/backend/tests/app_ctx_inputs.rs @@ -0,0 +1,88 @@ +//! A context input a raw app runnable declares (`fields. = { type: "ctx", .. }`) is +//! filled by the server from the viewer's session. Whatever the request sends under that +//! name is discarded: a runnable filters rows on it, so a viewer who could set it would +//! read someone else's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) +//! test-user-2 (developer, token SECRET_TOKEN_2) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const VIEWER_TOKEN: &str = "SECRET_TOKEN_2"; +const APP_PATH: &str = "u/test-user/ctx_app"; +const CONTENT: &str = "export function main(who: string) { return who }"; +// sha256 of CONTENT, the key an inline runnable is pinned under. +const CONTENT_SHA: &str = "6e198199621d57e4fed834510cb0c2692274aa4cbe976e1704fab2806be9d744"; + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn a_declared_ctx_input_ignores_the_value_the_viewer_sends( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + let client = reqwest::Client::new(); + + let resp = authed(client.post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "ctx inputs", + "value": { + "runnables": { + "whoami": { + "type": "inline", + "inlineScript": { "language": "bun", "content": CONTENT }, + "fields": { "who": { "type": "ctx", "ctx": "username" } } + } + } + }, + // The policy says nothing of context inputs: the server reads the + // declaration from the deployed runnable. + "policy": { + "execution_mode": "anonymous", + "triggerables_v2": { + format!("whoami:rawscript/{CONTENT_SHA}"): { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + + // Both spellings a viewer could send: a forged literal, and no argument at all. + for args in [json!({ "who": "test-user" }), json!({})] { + let resp = authed( + client.post(format!("{ws}/apps_u/execute_component/{APP_PATH}")), + VIEWER_TOKEN, + ) + .json(&json!({ + "component": "whoami", + "args": args, + "raw_code": { "language": "bun", "content": CONTENT } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let job_id: uuid::Uuid = resp.text().await?.parse()?; + + let queued: serde_json::Value = sqlx::query_scalar("SELECT args FROM v2_job WHERE id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + assert_eq!(queued["who"], json!("test-user-2"), "sent {args}"); + } + + Ok(()) +} diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 52ebbee7e7..24937a3ce5 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -237,7 +237,7 @@ async fn test_diff_global_settings_empty_string_routes_to_delete(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "key_a").await, @@ -468,7 +468,7 @@ async fn test_apply_settings_diff_deletes_only(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "to_delete_1").await.is_none()); assert!(get_global_setting(&db, "to_delete_2").await.is_none()); @@ -489,7 +489,7 @@ async fn test_apply_settings_diff_upserts_and_deletes(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "old_key").await.is_none()); assert_eq!( @@ -503,7 +503,7 @@ async fn test_apply_settings_diff_empty_noop(db: Pool) { insert_global_setting(&db, "preexisting", serde_json::json!("value")).await; let diff = SettingsDiff::default(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "preexisting").await, @@ -525,7 +525,7 @@ async fn test_apply_settings_diff_upsert_overwrites(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "overwrite_me").await, @@ -553,7 +553,7 @@ async fn test_apply_settings_diff_complex_json(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let stored = get_global_setting(&db, "complex_setting").await.unwrap(); assert_eq!(stored, complex_value); @@ -568,7 +568,7 @@ async fn test_apply_settings_diff_delete_nonexistent_is_noop(db: Pool) }; // Should not error - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); } // ======================================================================== @@ -589,7 +589,7 @@ async fn test_apply_configs_diff_upserts_with_prefix(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__mygroup").await.unwrap(); assert_eq!(stored["init_bash"], "echo hi"); @@ -601,7 +601,7 @@ async fn test_apply_configs_diff_deletes_with_prefix(db: Pool) { let diff = ConfigsDiff { upserts: BTreeMap::new(), deletes: vec!["to_remove".to_string()] }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__to_remove").await.is_none()); } @@ -619,7 +619,7 @@ async fn test_apply_configs_diff_upsert_overwrites(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__grp").await.unwrap(); assert_eq!(stored, serde_json::json!({"new": true})); @@ -630,7 +630,7 @@ async fn test_apply_configs_diff_empty_noop(db: Pool) { insert_config(&db, "worker__keep", serde_json::json!({"keep": true})).await; let diff = ConfigsDiff::default(); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__keep").await.is_some()); } @@ -641,7 +641,7 @@ async fn test_apply_configs_diff_does_not_touch_non_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("expose_metrics")); assert!(diff.deletes.is_empty()); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Step 5: Read back and verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -729,7 +729,7 @@ async fn test_roundtrip_replace_mode_deletes(db: Pool) { assert!(diff.deletes.contains(&"retention_period_secs".to_string())); assert!(diff.deletes.contains(&"expose_metrics".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "retention_period_secs") .await @@ -782,7 +782,7 @@ async fn test_roundtrip_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("gpu")); // new assert_eq!(diff.deletes, vec!["legacy".to_string()]); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); // Verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -820,7 +820,7 @@ async fn test_roundtrip_to_settings_map_from_db_consistency(db: Pool) let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let config = InstanceConfig::from_db(&db).await.unwrap(); assert_eq!(config.global_settings.base_url, original.base_url); @@ -868,8 +868,8 @@ async fn test_idempotent_apply(db: Pool) { }; // Apply twice - apply_settings_diff(&db, &diff).await.unwrap(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "idem_key").await, @@ -1069,7 +1069,7 @@ async fn test_diff_apply_only_touches_changed_rows(db: Pool) { assert!(!diff.upserts.contains_key("unchanged_1")); assert!(!diff.upserts.contains_key("unchanged_2")); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify all 4 settings present assert_eq!(count_global_settings(&db).await, 4); @@ -1148,7 +1148,7 @@ async fn test_replace_mode_protects_settings_in_integration(db: Pool) // But normal_setting should be deleted assert!(diff.deletes.contains(&"normal_setting".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify protected settings survived assert!(get_global_setting(&db, "ducklake_settings").await.is_some()); @@ -1196,7 +1196,7 @@ async fn test_jwt_secret_roundtrip_as_string_or_secret_ref(db: Pool) { let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB — jwt_secret should survive the roundtrip let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1239,7 +1239,7 @@ async fn test_rsa_keys_roundtrip_via_extra(db: Pool) { let map = settings.to_settings_map(); let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1296,7 +1296,7 @@ async fn test_replace_mode_protects_jwt_secret_and_rsa_keys(db: Pool) "normal_setting should be deleted" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Both protected settings survive assert_eq!( @@ -1371,7 +1371,7 @@ async fn test_alert_config_in_global_settings_roundtrip(db: Pool) { "alert_config change should be detected in diff" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Re-read and verify the update let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -1455,6 +1455,7 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a javascript: banner link must fail the sync"); @@ -1545,6 +1548,7 @@ async fn declarative_sync_rejects_a_malformed_max_token_expiration(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a non-hex accent color must fail the sync"); @@ -1681,7 +1691,109 @@ async fn declarative_sync_rejects_a_non_hex_accent_color(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect("a hex accent color must sync"); } + +/// A declarative sync has no user request behind it, so its observer is the only thing +/// that audits it: one entry per setting or worker config it actually writes, values +/// recorded under the API path's rule, and none for what it leaves as is. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_declarative_sync_audits_only_what_it_writes(db: Pool) { + use windmill_audit::instance_config::{ + declarative_sync_author, DeclarativeSource, InstanceConfigAuditor, + }; + + insert_global_setting(&db, "retention_period_secs", serde_json::json!(100)).await; + insert_global_setting(&db, "expose_metrics", serde_json::json!(true)).await; + insert_global_setting(&db, "scim_token", serde_json::json!("old-scim-secret")).await; + insert_global_setting( + &db, + "npm_config_registry", + serde_json::json!("https://u:p@npm"), + ) + .await; + insert_config( + &db, + "worker__kept", + serde_json::json!({"worker_tags": ["a"]}), + ) + .await; + insert_config( + &db, + "worker__dropped", + serde_json::json!({"worker_tags": ["b"]}), + ) + .await; + + let config: windmill_common::instance_config::InstanceConfig = + serde_json::from_value(serde_json::json!({ + "global_settings": { + "retention_period_secs": 200, + "expose_metrics": true, + "scim_token": "new-scim-secret", + }, + "worker_configs": { + "kept": {"worker_tags": ["a"]}, + "added": {"init_bash": "export TOKEN=init-secret"}, + }, + })) + .unwrap(); + let author = declarative_sync_author(); + config + .sync_to_db( + &db, + &InstanceConfigAuditor { + db: &db, + author: &author, + source: Some(DeclarativeSource::SyncConfig), + }, + ) + .await + .unwrap(); + + let entries: Vec<(String, String, serde_json::Value)> = sqlx::query_as( + "SELECT operation, resource, parameters FROM audit_partitioned \ + WHERE workspace_id = 'global' AND username = $1 AND resource = ANY($2) \ + ORDER BY resource", + ) + .bind(&author.username) + .bind(vec![ + "retention_period_secs", + "expose_metrics", + "scim_token", + "npm_config_registry", + "worker__kept", + "worker__dropped", + "worker__added", + ]) + .fetch_all(&db) + .await + .unwrap(); + + let source = serde_json::json!({"source": "sync-config"}); + let entry = |operation: &str, resource: &str, parameters: &serde_json::Value| { + ( + operation.to_string(), + resource.to_string(), + parameters.clone(), + ) + }; + assert_eq!( + entries, + vec![ + entry("global_settings.delete", "npm_config_registry", &source), + entry( + "global_settings.update", + "retention_period_secs", + &serde_json::json!({"source": "sync-config", "value": "200"}), + ), + entry("global_settings.update", "scim_token", &source), + entry("worker_config.update", "worker__added", &source), + entry("worker_config.delete", "worker__dropped", &source), + ] + ); +} diff --git a/backend/tests/mcp_header_var_refs.rs b/backend/tests/mcp_header_var_refs.rs new file mode 100644 index 0000000000..158bc71a5f --- /dev/null +++ b/backend/tests/mcp_header_var_refs.rs @@ -0,0 +1,110 @@ +//! An MCP resource header whose value is a `$var:` reference reaches the MCP +//! server as the variable's value, and only for a caller allowed to read it. +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use tokio::io::AsyncReadExt; +use windmill_common::client::AuthedClient; +use windmill_test_utils::*; +use windmill_worker::{load_mcp_tools, McpResourceConfig}; + +const SECRET_VALUE: &str = "S3CRET-MCP-HEADER-VALUE"; + +/// A secret variable and an MCP resource at `url` sending it as a header. +async fn seed(db: &Pool, url: &str) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'u/test-user/mcp_header', $1, true, '', '{}')", + ) + .bind(windmill_common::variables::encrypt(&mc, SECRET_VALUE)) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/mcp', $1, 'mcp', '{}', 'test-user')", + ) + .bind(serde_json::json!({ + "name": "mcp", + "url": url, + "headers": { "X-Api-Key": "$var:u/test-user/mcp_header", "X-Plain": "plain" } + })) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_is_resolved(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + std::env::set_var("ALLOW_PRIVATE_MCP_SERVER_URLS", "true"); + + // Not an MCP server: it captures the first request and hangs up, so loading + // fails at the connect step after the headers were sent. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let mcp_port = listener.local_addr()?.port(); + let first_request = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = vec![0u8; 16 * 1024]; + let n = socket.read(&mut buf).await?; + anyhow::Ok(String::from_utf8_lossy(&buf[..n]).to_lowercase()) + }); + seed(&db, &format!("http://127.0.0.1:{mcp_port}/mcp")).await?; + + let server = ApiServer::start(db.clone()).await?; + let client = AuthedClient::new( + format!("http://localhost:{}", server.addr.port()), + "test-workspace".to_string(), + "SECRET_TOKEN".to_string(), + None, + ); + let config = vec![McpResourceConfig { + resource_path: "$res:u/test-user/mcp".to_string(), + include_tools: None, + exclude_tools: None, + }]; + let _ = load_mcp_tools(&db, "test-workspace", config, &client).await; + + let request = first_request.await??; + assert!( + request.contains(&format!("x-api-key: {}", SECRET_VALUE.to_lowercase())), + "the header must carry the variable's value, not the reference: {request}" + ); + assert!(request.contains("x-plain: plain"), "{request}"); + + Ok(()) +} + +/// A token scoped to the resource alone must not have the API read the +/// variable a header points at. +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_requires_variable_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed(&db, "https://mcp.invalid.windmill.test").await?; + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('RES_ONLY_TOKEN'::bytea), 'hex'), 'RES_ONLY_T', 'RES_ONLY_TOKEN', + 'test@windmill.dev', 'resources only', false, '{resources:read}')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{}/api/w/test-workspace/resources/mcp_tools/u/test-user/mcp", + server.addr.port() + )) + .header("Authorization", "Bearer RES_ONLY_TOKEN") + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "{body}"); + assert!( + body.contains("variables:read:u/test-user/mcp_header"), + "{body}" + ); + + Ok(()) +} diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index f087090992..14df1a4c25 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -2,6 +2,36 @@ use sqlx::{Pool, Postgres}; use windmill_test_utils::*; +/// Both job-running tests seed `f/test/success_handler` with a different hash, each in its own +/// database, while the deployed-script caches are process-global and keyed by workspace and +/// path. With them on, one test resolves the handler to the other's hash and the handler push +/// 404s whenever that hash's info is not cached yet. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn disable_deployed_script_cache() { + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// The handler job is pushed after the job it handles is reported complete. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn wait_for_success_handler_job( + db: &Pool, +) -> anyhow::Result, Option)>> { + for _ in 0..100 { + let job = sqlx::query_as::<_, (Option, Option)>( + "SELECT runnable_path, parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(db) + .await?; + if job.is_some() { + return Ok(job); + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + Ok(None) +} + /// Test that the workspace success handler cache works correctly with 60s TTL #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -108,6 +138,7 @@ async fn test_success_handler_triggered_on_success(db: Pool) -> anyhow use windmill_common::scripts::{ScriptHash, ScriptLang}; initialize_tracing().await; + disable_deployed_script_cache(); let server = ApiServer::start(db.clone()).await?; @@ -191,49 +222,14 @@ export async function main(path: string, email: string, job_id: string, is_flow: let main_job_id = completed_job.id; - // Wait a short time for the success handler job to be created - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - - // Verify the success handler job was created (query by email since trigger is not set) - let success_handler_job = sqlx::query!( - r#" - SELECT - id, - runnable_path, - permissioned_as_email, - parent_job - FROM v2_job - WHERE workspace_id = 'test-workspace' - AND permissioned_as_email = 'success_handler@windmill.dev' - ORDER BY created_at DESC - LIMIT 1 - "# - ) - .fetch_optional(&db) - .await?; - - assert!( - success_handler_job.is_some(), - "Success handler job should have been created" - ); - - let handler_job = success_handler_job.unwrap(); - - // Verify the success handler job has correct parameters + let handler_job = wait_for_success_handler_job(&db).await?; assert_eq!( - handler_job.runnable_path.as_deref(), - Some("f/test/success_handler"), - "Success handler should run the configured script" - ); - assert_eq!( - handler_job.permissioned_as_email.as_str(), - "success_handler@windmill.dev", - "Success handler should run as success_handler user" - ); - assert_eq!( - handler_job.parent_job, - Some(main_job_id), - "Success handler should have main job as parent" + handler_job, + Some(( + Some("f/test/success_handler".to_string()), + Some(main_job_id) + )), + "Success handler should run the configured script with the main job as parent" ); // Note: root_job may be None when it equals parent_job (optimization in push function) @@ -250,6 +246,7 @@ async fn test_success_handler_triggered_on_hub_script_success( use windmill_common::jobs::JobPayload; initialize_tracing().await; + disable_deployed_script_cache(); let server = ApiServer::start(db.clone()).await?; @@ -285,21 +282,9 @@ async fn test_success_handler_triggered_on_hub_script_success( .await; assert!(completed_job.success, "Hub script should have succeeded"); - let mut handler_parent = None; - for _ in 0..50 { - handler_parent = sqlx::query_scalar::<_, Option>( - "SELECT parent_job FROM v2_job - WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", - ) - .fetch_optional(&db) - .await?; - if handler_parent.is_some() { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } + let handler_job = wait_for_success_handler_job(&db).await?; assert_eq!( - handler_parent, + handler_job.map(|(_, parent)| parent), Some(Some(completed_job.id)), "Workspace success handler should handle the successful hub script" ); diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index e591e85159..59733e352c 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -1,3 +1,4 @@ +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; @@ -533,7 +534,7 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none")] pub items: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub properties: Option>>, + pub properties: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub required: Option>, #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] @@ -560,9 +561,9 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none", rename = "$ref")] pub ref_path: Option, #[serde(skip_serializing_if = "Option::is_none", rename = "$defs")] - pub defs: Option>>, + pub defs: Option>>, #[serde(skip_serializing_if = "Option::is_none")] - pub definitions: Option>>, + pub definitions: Option>>, // Schema composition #[serde(skip_serializing_if = "Option::is_none", rename = "allOf")] @@ -829,7 +830,7 @@ impl OpenAPISchema { // Merge properties (other's properties are added if key doesn't exist) if let Some(ref other_props) = other.properties { - let props = self.properties.get_or_insert_with(HashMap::new); + let props = self.properties.get_or_insert_with(IndexMap::new); for (key, value) in other_props { props.entry(key.clone()).or_insert_with(|| value.clone()); } @@ -880,13 +881,13 @@ impl OpenAPISchema { // Merge $defs and definitions if let Some(ref other_defs) = other.defs { - let defs = self.defs.get_or_insert_with(HashMap::new); + let defs = self.defs.get_or_insert_with(IndexMap::new); for (key, value) in other_defs { defs.entry(key.clone()).or_insert_with(|| value.clone()); } } if let Some(ref other_definitions) = other.definitions { - let definitions = self.definitions.get_or_insert_with(HashMap::new); + let definitions = self.definitions.get_or_insert_with(IndexMap::new); for (key, value) in other_definitions { definitions .entry(key.clone()) @@ -981,7 +982,29 @@ pub struct S3ObjectWithType { #[cfg(test)] mod tests { use super::*; - use std::collections::HashMap; + + /// The schema is part of the prompt prefix the provider caches on, so its keys must + /// serialize in the same order on every request. + #[test] + fn strict_schema_keeps_the_defined_property_order() { + let keys = [ + "zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "kappa", + ]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let mut schema: OpenAPISchema = + serde_json::from_str(&format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + schema.make_strict(); + + let value = serde_json::to_value(&schema).unwrap(); + let serialized: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(serialized, keys); + assert_eq!(value["required"], serde_json::json!(keys)); + } /// Whichever provider served the request, `input_tokens` has to end up meaning the /// whole prompt. Leaving the Anthropic shape as reported under-states it by the @@ -1248,7 +1271,7 @@ mod tests { #[test] fn test_make_strict_defs() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1274,7 +1297,7 @@ mod tests { #[test] fn test_make_strict_definitions() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut definitions = HashMap::new(); + let mut definitions = IndexMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; @@ -1528,7 +1551,7 @@ mod tests { #[test] fn test_flatten_all_of_preserves_defs() { let def_schema = object_schema(vec![("field", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1642,7 +1665,7 @@ mod tests { schema_url: Some("http://json-schema.org/draft-07/schema#".to_string()), ..Default::default() }; - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 855f687431..b8ee9ded91 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1492,7 +1492,8 @@ async fn set_default_error_handler( } else { windmill_api_settings::delete_global_setting(&db, key.as_str()).await?; } - windmill_api_settings::audit_setting_write(&db, &authed, &key, value.as_ref()).await?; + windmill_audit::instance_config::audit_setting_write(&db, &authed, &key, value.as_ref(), None) + .await?; if payload.override_existing { // The rewrite and its history rows go in one transaction: on separate diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 116ff73457..1359e8c142 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -29,7 +29,11 @@ mod log_cleanup; mod storage_usage; use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_audit::{ + audit_oss::audit_log, + instance_config::{audit_setting_write, InstanceConfigAuditor}, + ActionKind, +}; use windmill_common::hub_resource_types::{self, SyncSource}; use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; use windmill_common::DB; @@ -833,50 +837,7 @@ pub async fn set_global_setting( require_super_admin(&db, &authed).await?; let value = value.value.unwrap_or(serde_json::Value::Null); set_global_setting_internal(&db, key.clone(), value.clone()).await?; - audit_setting_write(&db, &authed, &key, Some(&value)).await -} - -/// A setting is recorded by name, with its value only when that is a boolean or a number. -/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no -/// list of secret-bearing settings stays complete as settings are added. -/// -/// AUTHORIZATION: records a write, checks nothing. The caller must have established that -/// `authed` may change instance settings before performing the write this records. -pub async fn audit_setting_write( - db: &DB, - authed: &ApiAuthed, - key: &str, - value: Option<&serde_json::Value>, -) -> error::Result<()> { - let deleted = match value { - None | Some(serde_json::Value::Null) => true, - Some(serde_json::Value::String(s)) => s.trim().is_empty(), - Some(_) => false, - }; - let recorded = match value { - Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { - Some(v.to_string()) - } - _ => None, - }; - audit_log( - db, - authed, - if deleted { - "global_settings.delete" - } else { - "global_settings.update" - }, - if deleted { - ActionKind::Delete - } else { - ActionKind::Update - }, - "global", - Some(key), - recorded.as_deref().map(|v| [("value", v)].into()), - ) - .await + audit_setting_write(&db, &authed, &key, Some(&value), None).await } async fn audit_instance_action( @@ -1367,65 +1328,17 @@ async fn set_instance_config( run_setting_pre_write_hook(&db, key, value).await?; } } + let auditor = InstanceConfigAuditor { db: &db, author: &authed, source: None }; // The external-instance write removes its own key from the diff. - let external = settings_diff - .upserts - .get(EXTERNAL_INSTANCE_PG_SETTING) - .cloned() - .map(Some) - .or_else(|| { - settings_diff - .deletes - .iter() - .any(|k| k == EXTERNAL_INSTANCE_PG_SETTING) - .then_some(None) - }); windmill_common::external_instance_pg::write_external_instance_pg_from_diff( &db, &mut settings_diff, + &auditor, ) .await?; - if let Some(value) = external { - audit_setting_write(&db, &authed, EXTERNAL_INSTANCE_PG_SETTING, value.as_ref()).await?; - } - - // Applied and audited one setting at a time: the writes are not transactional, so a - // batch that fails midway must not leave the settings it did write unrecorded. - let instance_config::SettingsDiff { - upserts, - deletes, - mut previous_values, - unchanged_count, - } = settings_diff; - let steps = upserts - .into_iter() - .map(|(k, v)| (k, Some(v))) - .chain(deletes.into_iter().map(|k| (k, None))) - .map(Some) - .chain(std::iter::once(None)); - for step in steps { - let mut diff = instance_config::SettingsDiff::default(); - match &step { - Some((key, value)) => { - if let Some(previous) = previous_values.remove(key) { - diff.previous_values.insert(key.clone(), previous); - } - match value { - Some(value) => { - diff.upserts.insert(key.clone(), value.clone()); - } - None => diff.deletes.push(key.clone()), - } - } - None => diff.unchanged_count = unchanged_count, - } - instance_config::apply_settings_diff(&db, &diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; - if let Some((key, value)) = &step { - audit_setting_write(&db, &authed, key, value.as_ref()).await?; - } - } + instance_config::apply_settings_diff(&db, &settings_diff, &auditor) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; if ai_config_changed { bump_instance_ai_config_revision(); @@ -1455,37 +1368,13 @@ async fn set_instance_config( .collect(); let configs_diff = instance_config::diff_worker_configs(¤t_wc, &desired_wc, ApplyMode::Merge); - let steps = configs_diff - .upserts - .into_iter() - .map(|(g, c)| (g, Some(c))) - .chain(configs_diff.deletes.into_iter().map(|g| (g, None))); - for (group, config) in steps { - let mut diff = instance_config::ConfigsDiff::default(); - let (operation, action_kind) = match config { - Some(config) => { - diff.upserts.insert(group.clone(), config); - ("worker_config.update", ActionKind::Update) - } - None => { - diff.deletes.push(group.clone()); - ("worker_config.delete", ActionKind::Delete) - } - }; - instance_config::apply_configs_diff(&db, &diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; - audit_log( - &db, - &authed, - operation, - action_kind, - "global", - Some(&format!("worker__{group}")), - None, - ) - .await?; - } + instance_config::apply_configs_diff( + &db, + &configs_diff, + &InstanceConfigAuditor { db: &db, author: &authed, source: None }, + ) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; } Ok(()) diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index ad7f12453c..b3aea64f72 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -20,7 +20,7 @@ benchmark = [] embedding = ["windmill-api-embeddings/embedding"] parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256", "dep:tokio-rustls", "windmill-common/openidconnect", "windmill-store/openidconnect"] # The service log search handler reads the columnar store, so tantivy alone is # not enough for this crate to build on its own. tantivy = ["dep:windmill-indexer", "parquet"] @@ -141,6 +141,8 @@ samael = { workspace = true, optional = true } libxml = { workspace = true, optional = true } async-recursion.workspace = true rsa = { workspace = true, optional = true} +p256 = { workspace = true, optional = true } +tokio-rustls = { workspace = true, optional = true } uuid.workspace = true datafusion = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} @@ -189,4 +191,7 @@ flate2.workspace = true strum = { workspace = true, optional = true } dashmap.workspace = true +[dev-dependencies] +rcgen.workspace = true + [build-dependencies] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bab02327ec..ae01930182 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -7436,6 +7436,72 @@ paths: items: $ref: "#/components/schemas/TruncatedToken" + /oidc/signing_status: + get: + summary: what signs OIDC tokens and the keys the JWKS serves (ee only, superadmin) + operationId: getOidcSigningStatus + tags: + - oidc + responses: + "200": + description: OIDC signing status + content: + application/json: + schema: + type: object + required: + - mode + - keys + properties: + mode: + type: string + enum: [database, file, external] + keys: + type: array + items: + type: object + required: + - alg + - state + - source + properties: + kid: + type: string + alg: + type: string + state: + type: string + enum: [pending, active, retired, published] + source: + type: string + enum: [signer, database, file, publish_dir] + signer: + type: object + description: set when an external signer is configured + required: + - breaker + - max_rps + - current_rps + properties: + last_refresh_success: + type: string + format: date-time + last_refresh_error: + type: string + breaker: + type: string + enum: [closed, open, half_open] + max_rps: + type: number + current_rps: + type: number + keys_from_store: + type: boolean + description: the keys served are those stored by the last accepted listing, the signer has not answered this server yet + tls_files_error: + type: string + description: why the mTLS files on disk are not the ones in use + /w/{workspace}/oidc/token/{audience}: post: summary: get OIDC token (ee only) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 430aa801a0..2c2179d830 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -4393,10 +4393,55 @@ lazy_static! { /// change or deletion, which don't bump the version — see /// `invalidate_app_policy_cache` and `process_notify_event`); /// - the entry is within its TTL (backstop for a missed event or a restart). - static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = + static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = cache::Cache::new(1000); } +/// What `execute_component` runs a deployed app against. +struct DeployedPolicy { + policy: Policy, + /// Runnable id -> argument -> context property (`username`, `email`, `groups`, + /// `workspace`, `author`), for every `ctx` field the app's runnables declare. Read + /// from the deployed app rather than taken from the request, which only says what + /// the browser chose to send: a runnable filters rows on these. A deploy that + /// changes them under an unchanged policy evicts nothing, so a caller still naming + /// the replaced version keeps the previous declarations until the entry's TTL. + ctx_inputs: HashMap>, +} + +async fn deployed_ctx_inputs( + db: &DB, + w_id: &str, + path: &str, +) -> Result>> { + let runnables = sqlx::query_scalar::<_, Option>>( + "SELECT app_version.value -> 'runnables' FROM app + JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.path = $1 AND app.workspace_id = $2", + ) + .bind(path) + .bind(w_id) + .fetch_optional(db) + .await? + .flatten(); + let Some(serde_json::Value::Object(runnables)) = runnables.map(|r| r.0) else { + return Ok(HashMap::new()); + }; + Ok(runnables + .into_iter() + .filter_map(|(id, runnable)| { + let ctx_inputs: HashMap = runnable + .get("fields")? + .as_object()? + .iter() + .filter(|(_, f)| f.get("type").and_then(|t| t.as_str()) == Some("ctx")) + .filter_map(|(arg, f)| Some((arg.clone(), f.get("ctx")?.as_str()?.to_string()))) + .collect(); + (!ctx_inputs.is_empty()).then_some((id, ctx_inputs)) + }) + .collect()) +} + /// Backstop time-to-live for an [`APP_POLICY_CACHE`] entry — the maximum a policy /// change can go unreflected if its invalidation event is never consumed. const APP_POLICY_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(300); @@ -4449,7 +4494,7 @@ async fn execute_component( _ => {} }; - let arc_policy: Arc; + let arc_policy: Arc; let policy_triggerables_default = Default::default(); // Preview mode means the request was issued from the editor; the editing // user is already trusted by the policy check, so client-supplied `tag` @@ -4532,7 +4577,7 @@ async fn execute_component( // - The policy and triggerables are fetched from the database. // - In case of inline script, if an entry exists in the `app_script` table, push // an `AppScript` job payload, as in (.1) otherwise. - let (policy, policy_triggerables) = match payload { + let (policy, policy_triggerables, ctx_inputs) = match payload { // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), @@ -4551,6 +4596,7 @@ async fn execute_component( sensitive_inputs: force_viewer_sensitive_inputs.unwrap_or_default(), tag: None, }, + None, ), // 2. "run" mode. _ => { @@ -4580,9 +4626,12 @@ async fn execute_component( .map(|(_, p, _)| p); let policy = if let Some(p) = fresh { arc_policy = p; - &*arc_policy + &arc_policy.policy } else { - arc_policy = Arc::new(policy_fut.await?); + arc_policy = Arc::new(DeployedPolicy { + policy: policy_fut.await?, + ctx_inputs: deployed_ctx_inputs(&db, &w_id, path).await?, + }); APP_POLICY_CACHE.insert( cache_key, ( @@ -4591,7 +4640,7 @@ async fn execute_component( std::time::Instant::now(), ), ); - &*arc_policy + &arc_policy.policy }; // Caller-supplied inline code (`raw_code`), with or without an @@ -4653,7 +4702,11 @@ async fn execute_component( }) .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; - (policy, policy_triggerables) + ( + policy, + policy_triggerables, + arc_policy.ctx_inputs.get(&payload.component), + ) } }; @@ -4731,6 +4784,7 @@ async fn execute_component( let (mut args, job_id) = build_args( policy, policy_triggerables, + ctx_inputs, payload.args, opt_authed.as_ref(), &user_db, @@ -6161,6 +6215,54 @@ fn policy_granted_viewer(authed: &ApiAuthed) -> ApiAuthed { ApiAuthed { scopes: None, ..authed.clone() } } +/// The value of an app context property for this run. Everything about the viewer comes +/// from `authed`, the session the request authenticated with; an anonymous run has none +/// and gets `null`. +async fn ctx_value( + prop: &str, + policy: &Policy, + authed: Option<&ApiAuthed>, + w_id: &str, + db: &DB, +) -> Result> { + let value = match prop { + "username" => authed.map(|a| serde_json::to_value(a.display_username())), + "email" => authed.map(|a| serde_json::to_value(&a.email)), + "workspace" => Some(serde_json::to_value(w_id)), + "groups" => authed.map(|a| serde_json::to_value(&a.groups)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } + _ => { + return Err(Error::BadRequest(format!( + "context variable {} not allowed", + prop + ))) + } + }; + Ok(to_raw_value( + &value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { + Error::internal_err(format!("failed to serialize ctx variable {}: {}", prop, e)) + })?, + )) +} + async fn build_args( policy: &Policy, PolicyTriggerableInputs { @@ -6170,6 +6272,7 @@ async fn build_args( sensitive_inputs, .. }: &PolicyTriggerableInputs, + ctx_inputs: Option<&HashMap>, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -6179,6 +6282,16 @@ async fn build_args( let mut job_id: Option = None; let mut safe_args = HashMap::>::new(); + // Before anything reads `args`: a context input the runnable declares is never taken + // from the request, or a viewer could name someone else as themselves. + for (k, prop) in ctx_inputs.into_iter().flatten() { + args.remove(k); + safe_args.insert( + k.to_string(), + ctx_value(prop, policy, authed, w_id, db).await?, + ); + } + // tracing::error!("{:?}", allow_user_resources); for k in allow_user_resources.iter() { if let Some(arg_val) = args.get(k) { @@ -6270,47 +6383,9 @@ async fn build_args( if arg_str.starts_with("\"$ctx:") { let prop = arg_str.trim_start_matches("\"$ctx:").trim_end_matches("\""); - let value = match prop { - "username" => authed - .as_ref() - .map(|a| serde_json::to_value(a.display_username())), - "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), - "workspace" => Some(serde_json::to_value(&w_id)), - "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. - "author" => { - let author = match ( - policy.on_behalf_of_email.as_deref(), - policy.on_behalf_of.as_deref(), - ) { - (Some(email), _) => Some(email.to_string()), - (None, Some(permissioned_as)) => Some( - windmill_common::users::get_email_from_permissioned_as( - permissioned_as, - w_id, - db, - ) - .await?, - ), - (None, None) => None, - }; - Some(serde_json::to_value(&author)) - } - _ => { - return Err(Error::BadRequest(format!( - "context variable {} not allowed", - prop - ))) - } - }; safe_args.insert( k.to_string(), - to_raw_value(&value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { - Error::internal_err(format!( - "failed to serialize ctx variable for {}: {}", - k, e - )) - })?), + ctx_value(prop, policy, authed, w_id, db).await?, ); } else if !arg_str.contains("\"$var:") && !arg_str.contains("\"$res:") { safe_args.insert(k.to_string(), v); diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index 6663a0f83e..965a9289b5 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -6,7 +6,7 @@ * The build itself is `wmill app bundle`, the same one `wmill app push` runs, so * an app deployed through the API is compiled exactly as the CLI and the editor * compile it — entry point, virtual `wmill` module, `/ui/` shared-UI resolution - * and the Svelte/Vue plugins included. Reimplementing any of that here would be + * and the Svelte plugin included. Reimplementing any of that here would be * a third bundler to keep in step with the other two. */ declare const __wmillAppPolicy: { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 1cd2b4101c..92291bd1ea 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -149,6 +149,8 @@ pub mod oauth2_oss; #[cfg(feature = "private")] pub mod oidc_ee; mod oidc_oss; +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] +pub mod oidc_signer_tools_ee; mod path_autocomplete; mod raw_apps; mod resources; diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index e1931873a9..1e07f374de 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -23,14 +23,12 @@ async fn with_deadline( what: &str, fut: impl std::future::Future>, ) -> Result { - tokio::time::timeout(MCP_DEADLINE, fut) - .await - .map_err(|_| { - Error::ExecutionErr(format!( - "MCP server did not answer within {}s ({what})", - MCP_DEADLINE.as_secs() - )) - })? + tokio::time::timeout(MCP_DEADLINE, fut).await.map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? } /// Connect to the MCP server described by the `mcp` resource at `path`. @@ -65,8 +63,9 @@ async fn connect_mcp_client( let resource_value = not_found_if_none(resource_value_o, "Resource", path)? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - let mcp_resource = serde_json::from_str::(resource_value.0.get()) - .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; + let mut mcp_resource = + serde_json::from_str::(resource_value.0.get()) + .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; #[cfg(feature = "oauth2")] { @@ -126,6 +125,15 @@ async fn connect_mcp_client( None }; + // Header secrets go through the same permissioned path as the token. + for value in mcp_resource.headers.iter_mut().flatten().map(|(_, v)| v) { + if let Some(var_path) = value.strip_prefix("$var:") { + check_scopes(authed, || format!("variables:read:{}", var_path))?; + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + *value = get_value_internal(&db_authed, w_id, var_path, false).await?; + } + } + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) diff --git a/backend/windmill-audit/src/instance_config.rs b/backend/windmill-audit/src/instance_config.rs new file mode 100644 index 0000000000..977e0fe6c6 --- /dev/null +++ b/backend/windmill-audit/src/instance_config.rs @@ -0,0 +1,148 @@ +//! Audit entries for writes to instance settings and worker group configs, whichever way +//! they arrive: an API request, or a declarative sync that no user request stands behind. + +use std::collections::HashMap; + +use windmill_common::{ + audit::AuditAuthor, error::Result, instance_config::ConfigWriteObserver, + users::SUPERADMIN_SYNC_EMAIL, DB, +}; + +use crate::{ + audit_oss::{audit_log, AuditAuthorable}, + ActionKind, +}; + +/// The declarative writer behind an entry, recorded as its `source` parameter. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarativeSource { + /// `windmill sync-config ` + SyncConfig, + /// The Kubernetes operator's ConfigMap sync. + Operator, +} + +impl DeclarativeSource { + pub fn as_str(&self) -> &'static str { + match self { + DeclarativeSource::SyncConfig => "sync-config", + DeclarativeSource::Operator => "operator", + } + } +} + +/// Who a declarative sync's writes are attributed to. It runs on direct database +/// credentials, so there is no user to name. +pub fn declarative_sync_author() -> AuditAuthor { + AuditAuthor { + username: "instance_config_sync".to_string(), + email: SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + } +} + +fn parameters<'a>( + value: Option<&'a str>, + source: Option, +) -> Option> { + let parameters: HashMap<&str, &str> = value + .map(|v| ("value", v)) + .into_iter() + .chain(source.map(|s| ("source", s.as_str()))) + .collect(); + (!parameters.is_empty()).then_some(parameters) +} + +/// A setting is recorded by name, with its value only when that is a boolean or a number. +/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no +/// list of secret-bearing settings stays complete as settings are added. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change instance settings before performing the write this records. +pub async fn audit_setting_write( + db: &DB, + author: &impl AuditAuthorable, + key: &str, + value: Option<&serde_json::Value>, + source: Option, +) -> Result<()> { + let deleted = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::String(s)) => s.trim().is_empty(), + Some(_) => false, + }; + let recorded = match value { + Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { + Some(v.to_string()) + } + _ => None, + }; + audit_log( + db, + author, + if deleted { + "global_settings.delete" + } else { + "global_settings.update" + }, + if deleted { + ActionKind::Delete + } else { + ActionKind::Update + }, + "global", + Some(key), + parameters(recorded.as_deref(), source), + ) + .await +} + +/// A worker group config is recorded by name only: its env vars and init script are free +/// text that can carry a credential. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change worker group configs before performing the write this records. +pub async fn audit_worker_config_write( + db: &DB, + author: &impl AuditAuthorable, + group: &str, + deleted: bool, + source: Option, +) -> Result<()> { + let (operation, action_kind) = if deleted { + ("worker_config.delete", ActionKind::Delete) + } else { + ("worker_config.update", ActionKind::Update) + }; + audit_log( + db, + author, + operation, + action_kind, + "global", + Some(&format!("worker__{group}")), + parameters(None, source), + ) + .await +} + +/// Audits each write of an instance config apply as `author`'s. +/// +/// AUTHORIZATION: as [`audit_setting_write`], for every write it is told of. +pub struct InstanceConfigAuditor<'a, A> { + pub db: &'a DB, + pub author: &'a A, + /// `None` for a write made through the API. + pub source: Option, +} + +impl ConfigWriteObserver for InstanceConfigAuditor<'_, A> { + async fn setting_written(&self, key: &str, value: Option<&serde_json::Value>) -> Result<()> { + audit_setting_write(self.db, self.author, key, value, self.source).await + } + + async fn worker_config_written(&self, group: &str, deleted: bool) -> Result<()> { + audit_worker_config_write(self.db, self.author, group, deleted, self.source).await + } +} diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index f1443fdb18..e9c71f1336 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -4,6 +4,7 @@ use sqlx::FromRow; #[cfg(feature = "private")] pub mod audit_ee; pub mod audit_oss; +pub mod instance_config; #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 4d43206ebc..97dbc7e7bf 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -22,7 +22,7 @@ smtp = ["dep:mail-send", "dep:rustls", "dep:tokio-rustls", "dep:rustls-native-ce scoped_cache = [] cloud = [] dev_override = [] -openidconnect = ["dep:openidconnect", "dep:rsa"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256"] python = ["dep:windmill-parser-py"] [lib] @@ -116,6 +116,7 @@ pep440_rs.workspace = true systemstat.workspace = true size.workspace = true rsa = { workspace = true, optional = true } +p256 = { workspace = true, optional = true } spki = { workspace = true } pkcs1 = { workspace = true } aes-gcm = { workspace = true, optional = true } diff --git a/backend/windmill-common/src/external_instance_pg.rs b/backend/windmill-common/src/external_instance_pg.rs index c0b6203e40..72920a1874 100644 --- a/backend/windmill-common/src/external_instance_pg.rs +++ b/backend/windmill-common/src/external_instance_pg.rs @@ -198,7 +198,8 @@ pub async fn external_instance_database_usages<'c>( /// would be a login nothing can drop any more. Allowed on every edition, so a /// downgraded instance can still clear a setting it no longer uses. async fn ensure_external_instance_pg_removable(conn: &mut sqlx::PgConnection) -> Result<()> { - ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")).await + ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")) + .await } /// Refuse while Windmill has databases or data table roles on the cluster, or a workspace points @@ -394,16 +395,21 @@ pub async fn write_external_instance_pg_setting( } /// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches -/// it, and takes it out of the diff so the generic apply does not write it again. +/// it, and takes it out of the diff so the generic apply does not write it again. `observer` is +/// told of the write here, since the generic apply no longer sees the key. /// /// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config /// sync, which applies what the operator deployed. pub async fn write_external_instance_pg_from_diff( db: &DB, diff: &mut crate::instance_config::SettingsDiff, + observer: &impl crate::instance_config::ConfigWriteObserver, ) -> Result<()> { if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) { write_external_instance_pg_setting(db, Some(&value)).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, Some(&value)) + .await?; } if let Some(i) = diff .deletes @@ -412,6 +418,9 @@ pub async fn write_external_instance_pg_from_diff( { diff.deletes.remove(i); write_external_instance_pg_setting(db, None).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, None) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 3d50558c7f..7c76fab2ee 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1015,6 +1015,43 @@ pub struct ConfigsDiff { pub deletes: Vec, } +/// Told of each write an instance config apply makes, as soon as it has landed: the writes +/// are not transactional, so an apply that fails midway must not leave the ones it did make +/// unrecorded. The audit entry is the implementor's to write, as this crate sits below +/// `windmill-audit`. +pub trait ConfigWriteObserver: Sync { + /// `value` is `None` for a deleted setting. + fn setting_written( + &self, + key: &str, + value: Option<&serde_json::Value>, + ) -> impl std::future::Future> + Send; + + fn worker_config_written( + &self, + group: &str, + deleted: bool, + ) -> impl std::future::Future> + Send; +} + +impl ConfigWriteObserver for () { + async fn setting_written( + &self, + _key: &str, + _value: Option<&serde_json::Value>, + ) -> crate::error::Result<()> { + Ok(()) + } + + async fn worker_config_written( + &self, + _group: &str, + _deleted: bool, + ) -> crate::error::Result<()> { + Ok(()) + } +} + /// Settings that must never be deleted by the operator or bulk API. pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_user_pg_pwd", @@ -1026,6 +1063,8 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "rsa_keys", "jwt_secret", "min_keep_alive_version", + // Deleting it would let a worker sign OIDC tokens with a local key again. + "oidc_external_signer", ]; /// Secrets the server signs with or generated for itself. `jwt_secret` signs API and job @@ -1079,6 +1118,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ "custom_instance_replication_pwd", // Same for the passwords and database registry Windmill keeps for the external cluster. "external_instance_pg_state", + // Written by servers that sign OIDC tokens with an external signer. + "oidc_external_signer", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -1398,10 +1439,14 @@ pub fn diff_worker_configs( /// AUTHORIZATION: replaces instance-wide settings and takes no authed context, so /// callers MUST have established superadmin or equivalent system authority (the CLI /// and the operator both run with direct instance credentials). +/// +/// `observer` is what audits the sync: no user request stands behind these writes, so +/// nothing else records them. pub async fn sync_global_settings_declarative( db: &sqlx::Pool, current: &BTreeMap, desired: &BTreeMap, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for url_key in [ crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, @@ -1465,16 +1510,22 @@ pub async fn sync_global_settings_declarative( .map_err(|e| anyhow::anyhow!("{max_expiration_key}: {e}"))?; let mut diff = diff_global_settings(current, desired, ApplyMode::Replace); - crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?; - apply_settings_diff(db, &diff).await?; + crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff, observer) + .await?; + apply_settings_diff(db, &diff, observer).await?; Ok(()) } -/// Apply a settings diff to the global_settings table. +/// Apply a settings diff to the global_settings table, reporting each write to `observer`. +/// +/// AUTHORIZATION: writes instance-wide settings, credentials included, and checks +/// nothing; `observer` records and authorizes nothing. Callers MUST have established +/// superadmin or equivalent system authority. pub async fn apply_settings_diff( db: &sqlx::Pool, diff: &SettingsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (key, value) in &diff.upserts { sqlx::query( @@ -1497,6 +1548,7 @@ pub async fn apply_settings_diff( format_setting_value(key, value) ); } + observer.setting_written(key, Some(value)).await?; } for key in &diff.deletes { sqlx::query("DELETE FROM global_settings WHERE name = $1") @@ -1511,6 +1563,7 @@ pub async fn apply_settings_diff( } else { tracing::info!("Deleted global setting: {key}"); } + observer.setting_written(key, None).await?; } if diff.unchanged_count > 0 { tracing::info!("{} global setting(s) unchanged", diff.unchanged_count); @@ -1518,10 +1571,16 @@ pub async fn apply_settings_diff( Ok(()) } -/// Apply a configs diff to the config table (worker__ prefix). +/// Apply a configs diff to the config table (worker__ prefix), reporting each write to +/// `observer`. +/// +/// AUTHORIZATION: writes worker group configs, whose init scripts run on the workers, and +/// checks nothing; `observer` records and authorizes nothing. Callers MUST have +/// established superadmin or equivalent system authority. pub async fn apply_configs_diff( db: &sqlx::Pool, diff: &ConfigsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (group_name, config_value) in &diff.upserts { let db_key = format!("worker__{group_name}"); @@ -1534,6 +1593,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Synced worker config: {db_key}"); + observer.worker_config_written(group_name, false).await?; } for group_name in &diff.deletes { let db_key = format!("worker__{group_name}"); @@ -1542,6 +1602,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Deleted worker config: {db_key}"); + observer.worker_config_written(group_name, true).await?; } Ok(()) } @@ -1663,7 +1724,11 @@ impl InstanceConfig { /// /// Reads the current state, computes a diff, then applies upserts and /// deletes for both global settings and worker configs. - pub async fn sync_to_db(&self, db: &sqlx::Pool) -> anyhow::Result<()> { + pub async fn sync_to_db( + &self, + db: &sqlx::Pool, + observer: &impl ConfigWriteObserver, + ) -> anyhow::Result<()> { let desired_settings = self.global_settings.to_settings_map(); // Current global settings @@ -1673,7 +1738,8 @@ impl InstanceConfig { .await?; let current_settings: BTreeMap = rows.into_iter().collect(); - sync_global_settings_declarative(db, ¤t_settings, &desired_settings).await?; + sync_global_settings_declarative(db, ¤t_settings, &desired_settings, observer) + .await?; // Worker configs let desired_configs: BTreeMap = self @@ -1701,7 +1767,7 @@ impl InstanceConfig { let configs_diff = diff_worker_configs(¤t_configs, &desired_configs, ApplyMode::Replace); - apply_configs_diff(db, &configs_diff).await?; + apply_configs_diff(db, &configs_diff, observer).await?; Ok(()) } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 5ccc8beff1..f15b43182f 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -102,6 +102,8 @@ pub mod oauth2; pub mod oidc_ee; #[cfg(all(feature = "enterprise", feature = "openidconnect"))] pub mod oidc_oss; +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] +pub mod oidc_signer_ee; #[cfg(feature = "private")] pub mod otel_ee; pub mod otel_oss; diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index 95e635dd8f..9dac8a07db 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -54,6 +54,19 @@ pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Opt None } +#[cfg(not(feature = "private"))] +pub(crate) fn init_logs_bridge_for_service( + _service: &str, + _hostname: &str, + _env: &str, +) -> Option { + None +} + +/// Export the log records still batched, for a process about to exit. +#[cfg(not(feature = "private"))] +pub async fn flush_otel_logs() {} + #[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) fn init_otlp_tracer( _mode: &Mode, diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 01e75ac489..855e077004 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -220,6 +220,28 @@ pub fn initialize_tracing( (_guard, meter_provider) } +/// Tracing for a subcommand that writes to the instance without running a server +/// (`sync-config`, the operator): stdout, plus the OTel logs bridge when OTel logs are +/// enabled, which is how the audit events of its writes reach a collector. It writes no +/// service log files, as nothing lists such a process to read them back. +/// +/// Only audit events are exported. These processes log each setting they write with its +/// value, masked by a list of known secret fields rather than by shape, and that is fit +/// for the operator's own stdout but not for a collector. +pub fn initialize_subcommand_tracing(service: &str, hostname: &str, environment: &str) { + let logs_bridge = crate::otel_oss::init_logs_bridge_for_service(service, hostname, environment); + tracing_subscriber::registry() + .with(logs_bridge.with_filter(Targets::new().with_target(AUDIT_TARGET, LevelFilter::INFO))) + .with( + tracing_subscriber::fmt::layer().with_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ), + ) + .init(); +} + lazy_static::lazy_static! { pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); } diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index 863d6b0355..5d13a40314 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -70,7 +70,8 @@ impl McpClient { headers.insert(name, value); } _ => { - tracing::warn!("Invalid header: {}={}", key, value); + // The value may be a resolved secret: keep it out of the logs. + tracing::warn!("Invalid header: {}", key); } } } diff --git a/backend/windmill-operator/Cargo.toml b/backend/windmill-operator/Cargo.toml index 30a7c0bb7a..f847e4ab25 100644 --- a/backend/windmill-operator/Cargo.toml +++ b/backend/windmill-operator/Cargo.toml @@ -19,6 +19,7 @@ serde_json.workspace = true sqlx.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true anyhow.workspace = true thiserror.workspace = true kube.workspace = true diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 7437184385..f0a0b5c20f 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -103,9 +103,9 @@ pub fn filter_schema_by_input_transforms( .get_mut("properties") .and_then(|p| p.as_object_mut()) { - for key in &keys_to_remove { - properties.remove(key); - } + // `remove` swaps the last entry into the freed slot, so removing in set order + // would leave the survivors in a different order on every run. + properties.retain(|key, _| !keys_to_remove.contains(key)); } // Also remove from required array @@ -570,7 +570,7 @@ pub async fn load_mcp_tools( // is not allowed to read (resources:read:{path}). Reading through the raw // db pool here would bypass the authorization enforced by the regular MCP // tools API (get_mcp_tools) and act as a confused deputy. - let mcp_resource = client + let mut mcp_resource = client .get_resource_value::(path) .await .map_err(|e| { @@ -615,6 +615,18 @@ pub async fn load_mcp_tools( None }; + // Header secrets go through the same permissioned path as the token. + for (name, value) in mcp_resource.headers.iter_mut().flatten() { + if let Some(var_path) = value.strip_prefix("$var:") { + *value = client.get_variable_value(var_path).await.map_err(|e| { + Error::internal_err(format!( + "Failed to resolve variable {} for header {} of MCP resource {}: {}", + var_path, name, resource_name, e + )) + })?; + } + } + // Create new MCP client for this execution tracing::debug!("Creating fresh MCP client for {}", resource_name); let mcp_conn = McpClient::from_resource(mcp_resource, token) @@ -736,3 +748,32 @@ pub fn any_tool_needs_previous_result(tools: &[Tool]) -> bool { false }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A tool's schema sits in the prompt prefix the provider caches on, so the + /// parameters left for the model must keep their order on every run. + #[test] + fn filtering_preset_inputs_keeps_the_remaining_property_order() { + let keys = ["a", "b", "c", "d", "e", "f", "g", "h"]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let schema = + RawValue::from_string(format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + let preset = ["a", "c", "f"] + .iter() + .map(|k| (k.to_string(), InputTransform::Static { value: to_raw_value(&1) })) + .collect(); + + let filtered = filter_schema_by_input_transforms(schema, &preset).unwrap(); + let value: serde_json::Value = serde_json::from_str(filtered.get()).unwrap(); + let remaining: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(remaining, ["b", "d", "e", "g", "h"]); + } +} diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 24b07eacb9..411dd520b9 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -873,7 +873,8 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value { let mut v = arg_v; if !v.is_string() { - // if not string, convert to string for api request + // The SQL API has no ARRAY/VARIANT binding type: a list or object + // is bound as its JSON text, which the query reads with PARSE_JSON(?). v = json!(v.to_string()); } @@ -935,3 +936,16 @@ fn parse_val(value: &Value, typ: &str) -> Value { )) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn list_arg_binds_as_json_text() { + assert_eq!( + convert_typ_val("varchar[]".to_string(), json!(["sales", "g/ops"])), + json!({ "type": "TEXT", "value": r#"["sales","g/ops"]"# }) + ); + } +} diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0d4ad634c8..470f0f4b11 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -32,6 +32,8 @@ export interface AppFile { guests?: boolean; value: any; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; // Mirrors granular ACLs on the app path. Omitted from app.yaml when no @@ -160,6 +162,100 @@ export function executionModeForPush( } return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; } +/** Mirrors `FRONTEND_SDK_ALLOWED_SCOPES` in the backend's `apps.rs`, which stays + * the authority: this copy only fails a push before it bundles. */ +export const FRONTEND_API_SCOPES = [ + "jobs:run", + "jobs:read", + "users:read", + "resources:read", + "variables:read", + "flow_conversations:read", + "flow_conversations:write", +]; + +export type AppIsolation = { sandbox: boolean; frontendApi: string[] }; + +/** Write `sandbox` and `frontend_api` on a pulled app, only when they are on, so + * an app using neither keeps the file it had. Scopes of an unsandboxed app are + * inert and stay out: a file stating them without `sandbox` does not push. */ +export function markIsolationFromPolicy(app: any, rawApp: boolean) { + if (app?.["policy"]?.["sandbox"] !== true) { + return; + } + app.sandbox = true; + const scopes = app.policy.frontend_sdk_scopes; + if (rawApp && Array.isArray(scopes) && scopes.length > 0) { + app.frontend_api = scopes; + } +} + +/** The isolation the tracked file states, or `undefined` when it has no `sandbox` + * key and the deployed policy carries over. A file that has the key states both + * fields: `sandbox: false` turns isolation off, and a missing `frontend_api` + * means no scopes. */ +export function statedIsolation( + localApp: any, + rawApp: boolean, +): AppIsolation | undefined { + const sandbox = localApp?.["sandbox"]; + const scopes = localApp?.["frontend_api"] ?? []; + if (sandbox !== undefined && typeof sandbox !== "boolean") { + throw new Error("'sandbox' must be true or false"); + } + if ( + !Array.isArray(scopes) || + scopes.some((s: unknown) => typeof s !== "string") + ) { + throw new Error("'frontend_api' must be a list of scopes"); + } + if (scopes.length > 0 && !rawApp) { + throw new Error("'frontend_api' is only available to full-code apps"); + } + const seen = new Set(); + for (const scope of scopes) { + if (!FRONTEND_API_SCOPES.includes(scope)) { + throw new Error( + `Invalid frontend_api scope '${scope}'. Allowed scopes: ${FRONTEND_API_SCOPES.join(", ")}`, + ); + } + if (seen.has(scope)) { + throw new Error(`Duplicate frontend_api scope '${scope}'`); + } + seen.add(scope); + } + if (scopes.length > 0 && sandbox !== true) { + throw new Error( + "'frontend_api' requires 'sandbox: true': an unsandboxed app gets no frontend API token", + ); + } + return sandbox === undefined ? undefined : { sandbox, frontendApi: scopes }; +} + +/** Whether the deployed policy already has the stated isolation. The file's two + * keys are kept out of the superset comparison against the deployed app, which + * could not see a key the file dropped. */ +export function isolationUpToDate( + isolation: AppIsolation | undefined, + deployedPolicy: Policy | undefined, + rawApp: boolean, +): boolean { + if (!isolation) { + return true; + } + if ((deployedPolicy?.sandbox === true) !== isolation.sandbox) { + return false; + } + if (!rawApp) { + return true; + } + const deployed = deployedPolicy?.frontend_sdk_scopes ?? []; + return ( + deployed.length === isolation.frontendApi.length && + deployed.every((s, i) => s === isolation.frontendApi[i]) + ); +} + export async function pushApp( workspace: string, remotePath: string, @@ -198,6 +294,7 @@ export async function pushApp( } const path = localPath + "app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; + const isolation = statedIsolation(localApp, false); replaceInlineScripts(localApp.value, localPath, true); // On create the backend applies folder defaults, so there is nothing to preserve. @@ -210,17 +307,26 @@ export async function pushApp( localApp, remotePath, executionModeForPush(localApp, deployedPolicy), - basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of), + isolation ); // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). - const { extra_perms: localPerms, ...localAppBody } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppBody + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { - if (isSuperset(localAppBody, app)) { + if ( + isSuperset(localAppBody, app) && + isolationUpToDate(isolation, deployedPolicy, false) + ) { log.info(colors.green(`App ${remotePath} is up to date`)); } else { log.info(colors.bold.yellow(`Updating app ${remotePath}...`)); @@ -269,12 +375,13 @@ export async function generatingPolicy( app: any, path: string, executionMode: AppExecutionMode, - base: Policy | undefined + base: Policy | undefined, + isolation?: AppIsolation ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updatePolicy(app.value, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -332,10 +439,21 @@ export function preserveOnBehalfOfFields( * deployed app would keep being able to run runnables this push removed. */ export function finalizeDerivedPolicy( policy: Policy, - executionMode: AppExecutionMode + executionMode: AppExecutionMode, + isolation?: AppIsolation, + rawApp: boolean = false ) { policy.triggerables = undefined; policy.execution_mode = executionMode; + if (isolation) { + // `false` is sent rather than left out: the server reads an omitted value as + // "no opinion" and may sandbox the app anyway, hiding that the file was refused. + policy.sandbox = isolation.sandbox; + if (rawApp) { + policy.frontend_sdk_scopes = + isolation.frontendApi.length > 0 ? isolation.frontendApi : undefined; + } + } } async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 286e878cfb..287d01ebed 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -58,10 +58,10 @@ function conditionsFor(svelte: boolean): string[] { /** * Detects which frontend frameworks are present in package.json */ -export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolean } { +export function detectFrameworks(appDir: string): { svelte: boolean } { const packageJsonPath = path.join(appDir, "package.json"); if (!fs.existsSync(packageJsonPath)) { - return { svelte: false, vue: false }; + return { svelte: false }; } try { @@ -73,10 +73,9 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea return { svelte: "svelte" in allDeps, - vue: "vue" in allDeps, }; } catch { - return { svelte: false, vue: false }; + return { svelte: false }; } } @@ -256,17 +255,6 @@ export async function createFrameworkPlugins(appDir: string): Promise { plugins.push(createSveltePlugin(appDir)); } - if (frameworks.vue) { - log.info(colors.blue("🔧 Vue detected, adding vue plugin...")); - throw new Error("Vue plugin not supported yet"); - // try { - // const esbuildPluginVue = await import("esbuild-plugin-vue3"); - // plugins.push(esbuildPluginVue.default()); - // } catch (error: any) { - // log.warn(colors.yellow(`Failed to load vue plugin: ${error.message}`)); - // } - } - return plugins; } @@ -313,7 +301,7 @@ export async function createBundle( // Use the entryPoint's directory if provided, otherwise fall back to cwd. const appDir = options.entryPoint ? path.dirname(options.entryPoint) : process.cwd(); const frameworks = detectFrameworks(appDir); - const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = options.entryPoint ?? defaultEntry; const outDir = options.outDir ?? "dist"; @@ -331,7 +319,7 @@ export async function createBundle( // Ensure node_modules exists in the app directory await ensureNodeModules(appDir); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); // Ensure output directory exists diff --git a/cli/src/commands/app/bundle_command.ts b/cli/src/commands/app/bundle_command.ts index bef13f4268..c4119aafb0 100644 --- a/cli/src/commands/app/bundle_command.ts +++ b/cli/src/commands/app/bundle_command.ts @@ -12,10 +12,10 @@ interface BundleOptions { } /** Every entry point a raw app may have, most specific first, with the - * framework's preferred one promoted — Svelte and Vue mount from a `.ts`. */ + * framework's preferred one promoted — Svelte mounts from a `.ts`. */ function entryPointFor(appDir: string): string { const frameworks = detectFrameworks(appDir); - const candidates = frameworks.svelte || frameworks.vue + const candidates = frameworks.svelte ? ["index.ts", "index.tsx", "index.js"] : ["index.tsx", "index.ts", "index.js"]; const entry = candidates.find((c) => fs.existsSync(path.join(appDir, c))); diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 2dac8639e2..fdf26fa7da 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -481,7 +481,7 @@ async function dev(opts: DevOptions, appFolder?: string) { // Detect frameworks to determine default entry point const frameworks = detectFrameworks(process.cwd()); - const defaultEntry = frameworks.svelte || frameworks.vue + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = opts.entry ?? defaultEntry; @@ -577,7 +577,7 @@ async function dev(opts: DevOptions, appFolder?: string) { const buildOptions = getDevBuildOptions(entryPoint, frameworks.svelte); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); const wmillPlugin = { @@ -1602,7 +1602,7 @@ const command = new Command() }) .option( "--entry ", - "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)", + "Entry point file (default: index.ts for Svelte, index.tsx otherwise)", ) .option("--no-open", "Don't automatically open the browser") .option( diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 9356b5dd3c..c90a22105d 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -9,6 +9,8 @@ import { GlobalOptions } from "../../types.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; import { loadRunnablesFromBackend } from "./raw_apps.ts"; +import { inferSchema } from "../../utils/metadata.ts"; +import { statedIsolation } from "./app.ts"; import { getFolderSuffix, hasFolderSuffix, @@ -42,12 +44,62 @@ function validateRawAppYaml(appData: any): { errors.push("Field 'summary' must be a string"); } + try { + statedIsolation(appData, true); + } catch (error: any) { + errors.push(error.message); + } + // Note: 'runnables' is no longer required in raw_app.yaml // Runnables can be stored in separate files in the backend folder return { errors, warnings }; } +const SQL_LANGUAGES = new Set([ + "postgresql", + "mysql", + "bigquery", + "snowflake", + "mssql", + "oracledb", + "duckdb", +]); + +// The viewer properties a `ctx` field can supply. The Inputs tab of the app +// editor hints on the same names (RawAppInlineScriptRunnable.svelte). +const VIEWER_IDENTITY_ARGS = ["username", "email", "groups"]; + +/** + * A SQL runnable argument named after a viewer property is most often a + * per-viewer filter. Left as a plain input it takes whatever the browser + * sends, so it has to be a `ctx` field for the server to fill it. + */ +export async function viewerIdentityInputWarnings( + runnables: Record, +): Promise { + const warnings: string[] = []; + for (const [id, runnable] of Object.entries(runnables)) { + const { language, content } = runnable?.inlineScript ?? {}; + if (!SQL_LANGUAGES.has(language) || typeof content !== "string") { + continue; + } + const { schema } = await inferSchema(language, content, {}, id); + for (const arg of VIEWER_IDENTITY_ARGS) { + const field = runnable.fields?.[arg]; + const serverFilled = field?.ctx !== undefined || + field?.value !== undefined; + if (arg in (schema?.properties ?? {}) && !serverFilled) { + warnings.push( + `Runnable '${id}': argument '${arg}' is a frontend input, so a viewer can send any value. ` + + `If it identifies the viewer, set 'fields: { ${arg}: { ctx: ${arg} } }' in backend/${id}.yaml (next to 'type: inline') so the server fills it.`, + ); + } + } + } + return warnings; +} + /** * Validates that runnables exist either in backend/*.yaml files or in raw_app.yaml */ @@ -77,6 +129,7 @@ async function validateRunnables( " - Or a 'runnables' field in raw_app.yaml (legacy format)", ); } else if (hasBackendRunnables) { + warnings.push(...await viewerIdentityInputWarnings(runnablesFromBackend)); log.info( colors.gray( ` Found ${ @@ -114,7 +167,7 @@ async function validateBuild( // Try to create a bundle - this will validate that all dependencies are in place const frameworks = detectFrameworks(appDir); - const entryFile = frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = path.join(appDir, entryFile); await createBundle({ entryPoint, diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts index b011965ae8..40cc2a4416 100644 --- a/cli/src/commands/app/new.ts +++ b/cli/src/commands/app/new.ts @@ -85,21 +85,6 @@ const app = mount(App, { target: document.getElementById("root")! }); export default app; `; -const appVue = ` - -`; - -const indexVue = `import { createApp } from 'vue' -import App from './App.vue' -import "./index.css"; - -createApp(App).mount('#root')`; - const indexCss = `body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; @@ -171,21 +156,6 @@ const templates: Record = { "svelte": "^5.55.5", "windmill-client": "^1" } -}`, - }, - }, - vue: { - name: "Vue 3", - files: { - "/index.ts": indexVue, - "/App.vue": appVue, - "/index.css": indexCss, - "/package.json": `{ - "dependencies": { - "core-js": "3.26.1", - "vue": "3.5.13", - "windmill-client": "^1" - } }`, }, }, @@ -251,7 +221,7 @@ interface DataConfig { schema?: string; } -type FrameworkKey = "react19" | "react18" | "svelte5" | "vue"; +type FrameworkKey = "react19" | "react18" | "svelte5"; interface NewAppOptions extends GlobalOptions { /** App summary (short description). Skips the prompt when provided. */ @@ -384,7 +354,7 @@ async function newApp(opts: NewAppOptions) { } // Ask for framework (skipped if --framework is provided) - const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5", "vue"]; + const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5"]; let framework: string; if (opts.framework !== undefined) { if (!VALID_FRAMEWORKS.includes(opts.framework)) { @@ -403,7 +373,6 @@ async function newApp(opts: NewAppOptions) { { name: "React 19 (Recommended)", value: "react19" }, { name: "React 18", value: "react18" }, { name: "Svelte 5", value: "svelte5" }, - { name: "Vue 3", value: "vue" }, ], }); } @@ -900,7 +869,7 @@ const command = new Command() ) .option( "--framework ", - "Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode." + "Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode." ) .option( "--datatable ", diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 36c8260e25..4c3d8d5215 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -20,13 +20,16 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { type AppExecutionMode, + type AppIsolation, basePolicy, executionModeForPush, finalizeDerivedPolicy, + isolationUpToDate, markAccessFromPolicy, preserveOnBehalfOfFields, replaceInlineScripts, repopulateFields, + statedIsolation, } from "./app.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; @@ -49,6 +52,8 @@ export interface AppFile { runnables?: any; custom_path?: string; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; data?: { @@ -402,6 +407,7 @@ export async function pushRawApp( } const appFilePath = localPath + "raw_app.yaml"; const localApp = (await yamlParseFile(appFilePath)) as AppFile; + const isolation = statedIsolation(localApp, true); // Load runnables from separate YAML files in the backend folder // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) @@ -454,6 +460,7 @@ export async function pushRawApp( deployedPolicy, !!preserveFields.preserve_on_behalf_of, ), + isolation, ); const files = await collectAppFiles(localPath); @@ -461,7 +468,7 @@ export async function pushRawApp( log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); // Detect frameworks to determine entry point const frameworks = detectFrameworks(localPath); - const entryFile = frameworks.svelte || frameworks.vue + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = localPath + entryFile; @@ -485,14 +492,21 @@ export async function pushRawApp( // to `.raw_app.json`, not `.app.json`. The backend granular_acls // handler routes "raw_app" to the `app` table (where v2 raw apps actually // live) while still dispatching DeployedObject::RawApp for git-sync. - const { extra_perms: localPerms, ...localAppNoPerms } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppNoPerms + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { // Check both metadata/runnables AND files for changes // Files need separate comparison because isSuperset only checks if local keys exist in remote - const metadataUpToDate = isSuperset({ ...localAppNoPerms, runnables }, app); + const metadataUpToDate = + isSuperset({ ...localAppNoPerms, runnables }, app) && + isolationUpToDate(isolation, deployedPolicy, true); const filesUpToDate = deepEqual(files, app.value?.files); if (metadataUpToDate && filesUpToDate) { log.info(colors.green(`App ${remotePath} is up to date`)); @@ -560,11 +574,12 @@ export async function generatingPolicy( path: string, executionMode: AppExecutionMode, base: Policy | undefined, + isolation?: AppIsolation, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation, true); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 54887028f0..b9e373abad 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -149,7 +149,7 @@ import { extractCurrentMapping, } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; -import { markAccessFromPolicy } from "../app/app.ts"; +import { markAccessFromPolicy, markIsolationFromPolicy } from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, @@ -1359,6 +1359,7 @@ export function ZipFSElement( } markAccessFromPolicy(app); + markIsolationFromPolicy(app, false); app.policy = undefined; yield { isDirectory: false, @@ -1377,6 +1378,7 @@ export function ZipFSElement( throw error; } markAccessFromPolicy(rawApp); + markIsolationFromPolicy(rawApp, true); // console.log("rawApp", rawApp); rawApp.policy = undefined; // custom_path is derived from the file path, don't store it diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index f9b45b9662..4a92f1e5d2 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1106,6 +1106,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -1954,6 +1957,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2908,6 +2914,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -5091,7 +5100,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -5177,12 +5188,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -5629,6 +5643,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` +## Array parameters + +Declare a list with a \`[]\` suffix on the element type (\`(varchar[])\`, +\`(int[])\`), or \`(array)\` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +\`PARSE_JSON(?)\`, and cast to \`ARRAY\` where a function expects one. + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +\`\`\` + +Or join on the elements with \`FLATTEN\`: + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +\`\`\` + ## Receiving an S3Object as a script parameter Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for @@ -6369,7 +6404,7 @@ You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. \`f/folder/my_app\` or \`u/username/my_app\` -3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\`, \`vue\` +3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\` **summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. @@ -6441,7 +6476,7 @@ my_app{{RAW_APP_SUFFIX}}/ ├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) ├── raw_app.yaml # App configuration (summary, path, data settings) ├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component +├── App.tsx # Main React/Svelte component ├── index.css # Styles ├── package.json # Frontend dependencies ├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) @@ -6599,7 +6634,7 @@ For the rest, tell the user which command fits their intent and let them run it # Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -6615,7 +6650,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -6645,7 +6680,7 @@ Then build what was chosen: the data table and schema become the app's \`data.da ### Entrypoint -The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim. +The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` and keep the entrypoint as the mount shim. React (\`index.tsx\`): @@ -6657,7 +6692,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() \`\`\` -Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. Vue (\`index.ts\`): \`createApp(App).mount('#root')\`. +Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. **Never replace the entrypoint with a bare component** (\`export default function App() { ... }\` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. @@ -6823,6 +6858,18 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. \`public: true\` in \`raw_app.yaml\` deploys the app for anonymous users, and \`guests: true\` for guests. Remove the line and the next push closes the app again. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are \`jobs:run\`, \`jobs:read\`, \`users:read\`, \`resources:read\`, \`variables:read\`, \`flow_conversations:read\` and \`flow_conversations:write\`; grant only the ones the frontend calls. +In \`raw_app.yaml\`: + +\`\`\`yaml +sandbox: true +frontend_api: [jobs:run, users:read] +\`\`\` + +A file with a \`sandbox\` key states both settings: \`sandbox: false\` turns isolation off, and a missing \`frontend_api\` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing \`sandbox: true\` does not unsandbox it. \`frontend_api\` needs \`sandbox: true\`; the push fails otherwise. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. @@ -7455,6 +7502,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with \`@task()\` or \`@task\`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or \`step()\` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. \`sleep()\` there waits in the job, and \`taskScript()\` / \`taskFlow()\` / \`waitForApproval()\` (\`task_script()\` / \`task_flow()\` / \`wait_for_approval()\`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: \`\`\`typescript @@ -7620,8 +7669,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -7665,6 +7715,9 @@ export function workflow(fn: (...args: any[]) => Promise) * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -7763,7 +7816,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -7849,12 +7904,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -8121,7 +8179,7 @@ app related commands - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement - \`--port \` - Port to run the dev server on (will find next available port if occupied) - \`--host \` - Host to bind the dev server to - - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - \`--entry \` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - \`--no-open\` - Don't automatically open the browser - \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability @@ -8132,7 +8190,7 @@ app related commands - \`app new\` - create a new raw app from a template - \`--summary \` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - \`--framework \` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. diff --git a/cli/test/app_isolation_unit.test.ts b/cli/test/app_isolation_unit.test.ts new file mode 100644 index 0000000000..201c4a52a4 --- /dev/null +++ b/cli/test/app_isolation_unit.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { + finalizeDerivedPolicy, + isolationUpToDate, + markIsolationFromPolicy, + statedIsolation, +} from "../src/commands/app/app.ts"; + +function pushed(file: any, deployed: any, rawApp = true) { + const policy = { ...deployed }; + finalizeDerivedPolicy(policy, "publisher", statedIsolation(file, rawApp), rawApp); + return policy; +} + +test("a pull writes isolation only when it is on, and a push deploys what was pulled", () => { + const off: any = { policy: { sandbox: false, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(off, true); + expect("sandbox" in off).toBe(false); + expect("frontend_api" in off).toBe(false); + + const on: any = { policy: { sandbox: true, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(on, true); + expect(on.sandbox).toBe(true); + expect(on.frontend_api).toEqual(["jobs:run"]); + // Into a workspace where the app does not exist yet. + const fresh = pushed(on, undefined); + expect(fresh.sandbox).toBe(true); + expect(fresh.frontend_sdk_scopes).toEqual(["jobs:run"]); +}); + +test("the file wins when it states sandbox, the deployed policy carries over when it does not", () => { + const deployed = { sandbox: true, frontend_sdk_scopes: ["jobs:run"] }; + + const silent = pushed({}, deployed); + expect(silent.sandbox).toBe(true); + expect(silent.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(isolationUpToDate(statedIsolation({}, true), deployed, true)).toBe(true); + + // A file with the key states the scopes too: none here. + const narrowed = pushed({ sandbox: true }, deployed); + expect(narrowed.sandbox).toBe(true); + expect(narrowed.frontend_sdk_scopes).toBeUndefined(); + expect(isolationUpToDate(statedIsolation({ sandbox: true }, true), deployed, true)).toBe(false); + + const opened = pushed({ sandbox: false }, deployed); + expect(opened.sandbox).toBe(false); + expect(opened.frontend_sdk_scopes).toBeUndefined(); +}); + +test("scopes are refused without sandbox, outside the allowlist, or repeated", () => { + expect(() => statedIsolation({ frontend_api: ["jobs:run"] }, true)).toThrow("requires 'sandbox: true'"); + expect(() => statedIsolation({ sandbox: false, frontend_api: ["jobs:run"] }, true)).toThrow( + "requires 'sandbox: true'", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["apps:write"] }, true)).toThrow("Invalid"); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run", "jobs:run"] }, true)).toThrow( + "Duplicate", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run"] }, false)).toThrow("full-code"); +}); diff --git a/cli/test/app_lint_viewer_identity_unit.test.ts b/cli/test/app_lint_viewer_identity_unit.test.ts new file mode 100644 index 0000000000..f328ffe4e2 --- /dev/null +++ b/cli/test/app_lint_viewer_identity_unit.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "bun:test"; +// app.ts first: it is the entry the CLI loads lint.ts through, and loading +// lint.ts ahead of it trips their import cycle. +import "../src/commands/app/app.ts"; +import { viewerIdentityInputWarnings } from "../src/commands/app/lint.ts"; + +const sql = (content: string, fields?: Record) => ({ + type: "inline", + inlineScript: { language: "duckdb", content }, + ...(fields ? { fields } : {}), +}); + +test("app lint warns on a SQL viewer-identity argument the server does not fill", async () => { + const content = + "-- $username (text)\n-- $limit (int)\nSELECT * FROM orders WHERE owner = $username LIMIT $limit"; + + const warnings = await viewerIdentityInputWarnings({ + open: sql(content), + filled: sql(content, { username: { ctx: "username" } }), + }); + + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("Runnable 'open': argument 'username'"); +}); diff --git a/docker/DockerfileExtra b/docker/DockerfileExtra index ace382528b..1d8d0003da 100644 --- a/docker/DockerfileExtra +++ b/docker/DockerfileExtra @@ -71,7 +71,7 @@ ENV XDG_CACHE_HOME=/pyls/.cache RUN uv pip install --system --break-system-packages pipenv tornado python-lsp-jsonrpc ruff==0.16.0 Cython # Install Node-based language servers -RUN npm install -g diagnostic-languageserver pyright +RUN npm install -g pyright # Setup LSP working directory WORKDIR /pyls diff --git a/docker/test_windmill_extra.ts b/docker/test_windmill_extra.ts index 5fd1a74108..c5884c2dd4 100644 --- a/docker/test_windmill_extra.ts +++ b/docker/test_windmill_extra.ts @@ -121,6 +121,60 @@ async function testLspWebSocket(): Promise { }) } +async function testLspShellDiagnostics(): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + ws.close() + reject(new Error('no shellcheck diagnostics within 10s')) + }, 10000) + + const ws = new WebSocket(`ws://${HOST}:${LSP_PORT}/ws/diagnostic`) + const send = (msg: object) => ws.send(JSON.stringify({ jsonrpc: '2.0', ...msg })) + + ws.onopen = () => { + // Editors still send a linter definition; the server's own configuration + // applies, so the diagnostics below have to come from shellcheck. + send({ + id: 1, + method: 'initialize', + params: { + capabilities: {}, + initializationOptions: { + linters: { shellcheck: { command: 'true', sourceName: 'shellcheck' } }, + filetypes: { shell: 'shellcheck' } + } + } + }) + send({ + method: 'textDocument/didOpen', + params: { + textDocument: { + uri: 'file:///tmp/monaco/test.sh', + languageId: 'shell', + version: 1, + text: '#!/bin/bash\necho "$unassigned"\n' + } + } + }) + } + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data.toString()) + if (msg.method !== 'textDocument/publishDiagnostics') return + clearTimeout(timeout) + ws.close() + const codes = msg.params.diagnostics.map((d: { code: string }) => d.code) + if (codes.includes('SC2154')) resolve() + else reject(new Error(`expected SC2154, got ${JSON.stringify(msg.params.diagnostics)}`)) + } + + ws.onerror = () => { + clearTimeout(timeout) + reject(new Error('LSP WebSocket connection failed')) + } + }) +} + async function runLspTests(): Promise { log('\n=== LSP Service Tests (Port ' + LSP_PORT + ') ===') @@ -132,6 +186,7 @@ async function runLspTests(): Promise { let passed = true passed = (await runTest('LSP health check', testLspHealth)) && passed passed = (await runTest('LSP WebSocket connection (pyright)', testLspWebSocket)) && passed + passed = (await runTest('LSP shell diagnostics', testLspShellDiagnostics)) && passed return passed } diff --git a/frontend/scripts/fix-svelte-module-imports.js b/frontend/scripts/fix-svelte-module-imports.js index 9cec6265a0..fe4fde4959 100644 --- a/frontend/scripts/fix-svelte-module-imports.js +++ b/frontend/scripts/fix-svelte-module-imports.js @@ -16,9 +16,22 @@ // tarball. Copy the registry into the package and rewrite those imports // to point at it (mirrors how package-system-prompts.js bundles // `$system_prompts`). +// +// 3. Chat SDK: the `windmill-chat` alias (svelte.config.js) points at +// `chat-sdk/src/index.ts`, which escapes the package root the same way. +// Compile that entry point into the package and rewrite those imports. +// +// 4. `$lib/utils` from inside `utils/`: svelte-package turns it into `./`, +// which is the `utils/` directory (no index) rather than `utils.js`. +// Point a directory specifier with no index at its sibling module. +// +// A relative import that still escapes the package root, or an +// `import.meta.glob` left on an alias, fails the build: either one +// publishes fine and only breaks in the consumer's bundler. +import { execFileSync } from 'child_process' import { copyFileSync, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs' import { fileURLToPath } from 'url' -import { dirname, join, relative, resolve } from 'path' +import { dirname, join, relative, resolve, sep } from 'path' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -29,8 +42,37 @@ const oauthConnectDest = join(packageDir, 'oauth_connect.json') copyFileSync(oauthConnectSrc, oauthConnectDest) -const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?\/[^'"]*\.(?:svelte|json))\2/g +const chatSdkSrc = resolve(__dirname, '..', '..', 'chat-sdk', 'src', 'index.ts') +const chatSdkDest = join(packageDir, 'chat-sdk') + +// File arguments make tsc ignore every tsconfig and emit only what the entry +// point reaches, which leaves out the React and AI SDK adapters. +execFileSync( + 'npx', + [ + 'tsc', + chatSdkSrc, + '--outDir', + chatSdkDest, + '--rootDir', + dirname(chatSdkSrc), + '--declaration', + '--target', + 'ES2021', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck' + ], + { stdio: 'inherit', cwd: resolve(__dirname, '..') } +) + +const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?(?:\/[^'"]*)?)\2/g const OAUTH_CONNECT_RE = /^(\.\.\/)+backend\/oauth_connect\.json$/ +const CHAT_SDK_RE = /^(\.\.\/)+chat-sdk\/src\/index\.ts$/ +const DIRECTORY_RE = /(^|\/)\.\.?$|\/$/ +const ALIASED_GLOB_RE = /import\.meta\.glob(?:<[^>]*>)?\(\s*\[?\s*['"]\$/ function* walk(dir) { for (const entry of readdirSync(dir)) { @@ -50,9 +92,13 @@ function relativeSpecifier(fromFile, toFile) { let rewrittenFiles = 0 let rewrittenImports = 0 +const unresolvable = [] for (const file of walk(packageDir)) { const content = readFileSync(file, 'utf-8') + if (ALIASED_GLOB_RE.test(content)) { + unresolvable.push(`${relative(packageDir, file)}: import.meta.glob on an alias`) + } let changed = false const updated = content.replace(IMPORT_SPECIFIER_RE, (match, prefix, quote, specifier) => { if (OAUTH_CONNECT_RE.test(specifier)) { @@ -60,10 +106,27 @@ for (const file of walk(packageDir)) { rewrittenImports++ return `${prefix}${quote}${relativeSpecifier(file, oauthConnectDest)}${quote}` } + if (CHAT_SDK_RE.test(specifier)) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, join(chatSdkDest, 'index.js'))}${quote}` + } + const target = resolve(dirname(file), specifier) + if (target !== packageDir && !target.startsWith(packageDir + sep)) { + unresolvable.push(`${relative(packageDir, file)}: ${specifier}`) + return match + } + if (DIRECTORY_RE.test(specifier)) { + if (!existsSync(join(target, 'index.js')) && existsSync(target + '.js')) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, target + '.js')}${quote}` + } + return match + } if (!specifier.endsWith('.svelte')) { return match } - const target = resolve(dirname(file), specifier) // A real `.svelte` component file: leave the import alone. if (existsSync(target)) { return match @@ -82,6 +145,14 @@ for (const file of walk(packageDir)) { } } +if (unresolvable.length > 0) { + console.error( + `fix-svelte-module-imports: ${unresolvable.length} import(s) a consumer cannot resolve:\n` + + unresolvable.join('\n') + ) + process.exit(1) +} + console.log( `fix-svelte-module-imports: rewrote ${rewrittenImports} import(s) in ${rewrittenFiles} file(s)` ) diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index aab564a4cf..c624b5a148 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -69,6 +69,8 @@ } from 'lucide-svelte' import { hubPaths } from '$lib/hub' import { isCloudHosted } from '$lib/cloud' + import { untrack } from 'svelte' + import { syncSlackBotArg } from './handlerSlackArg' import SmtpConfigurationStatus from './common/smtp/SmtpConfigurationStatus.svelte' import { SettingService } from '$lib/gen' import { isSmtpSettingsValid } from './instanceSettings/SmtpSettings.svelte' @@ -316,12 +318,13 @@ loadSmtpConfiguration() }) + let builtinArgs: Record | undefined $effect(() => { - if (handlerSelected === 'slack' && isSlackHandler(handlerPath)) { - handlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - handlerExtraArgs['slack'] = undefined - } + const isSlackBuiltin = handlerSelected === 'slack' && isSlackHandler(handlerPath) + const args = handlerExtraArgs + // A replaced args object (Discard, reload) holds the stored config, not what the built-in set. + untrack(() => syncSlackBotArg(args, isSlackBuiltin, args === builtinArgs)) + builtinArgs = isSlackBuiltin ? args : undefined }) let lastHandlerSelected: ErrorHandler | undefined = $state(undefined) diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 1476b9ad87..a1a0f214d3 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -29,6 +29,7 @@ import SettingsPageHeader from './settings/SettingsPageHeader.svelte' import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte' import DbHealth from './instanceSettings/DbHealth.svelte' + import OidcSigningStatus from './instanceSettings/OidcSigningStatus.svelte' interface Props { tab?: string @@ -1186,6 +1187,15 @@ description="On-demand database diagnostics. Analyze table sizes, job retention, connection pool health, vacuum status, and more." /> + {:else if category == 'OIDC signing'} + + {#if $enterpriseLicense} + + {/if} {:else if category == 'Auth/OAuth/SAML'} +{#snippet tableLoading()} +
+ Loading table... +
+{/snippet} + {#if workspaceSettingsInitialized === false}
{#if fromWorkspaceSettings} @@ -1522,17 +1527,20 @@ {#if showTable}
{#if currentTable?.ops} - {#key currentTable.ops} - - {/key} + + {#await import('./DBTable.svelte')} + {@render tableLoading()} + {:then DBTable} + {#key currentTable.ops} + + {/key} + {/await} {:else if tableOps.error}
Could not read this file as a table: {tableOps.error.message}
{:else} -
- Loading table... -
+ {@render tableLoading()} {/if}
{:else} diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 357c9dc6ae..eb5559a2aa 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -17,7 +17,7 @@ import { truncate } from '$lib/utils' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { ArrowRightLeft, Ban, @@ -46,6 +46,8 @@ tabToAuthSubTab, categoryToTabMap, buildSearchableSettingItems, + listedNavigationGroups, + usesExternalOidcSigner, type SearchableSettingItem } from './instanceSettings' import TextInput from './text_input/TextInput.svelte' @@ -227,7 +229,14 @@ return instanceSettings?.buildFullDiff() ?? { original: '', modified: '' } } // --- Settings search --- - const searchableItems = buildSearchableSettingItems() + let externalOidcSigner = $state(false) + $effect(() => { + if ($enterpriseLicense) usesExternalOidcSigner().then((v) => (externalOidcSigner = v)) + }) + const navigationGroups = $derived( + listedNavigationGroups(instanceSettingsNavigationGroups, externalOidcSigner) + ) + const searchableItems = $derived(buildSearchableSettingItems(navigationGroups)) let scrollTimeout: ReturnType | undefined let highlightTimeout: ReturnType | undefined @@ -300,11 +309,7 @@
- + {#if $workspaceStore !== 'admins'}
/\` for bare names; only use \`f//\` when the folder is known to exist. The first write tool snapshots the workspace app onto the draft, and subsequent writes accumulate. -- To create a new app, use \`init_app\` with a path, optional summary, a framework (\`react19\` / \`react18\` / \`svelte5\` / \`vue\`) and, for an app that stores data, its \`data\` setup. "Starting a new app" below decides the framework and when the data setup is asked. Ask each of its questions with \`askUserQuestion\`, one question per call with the suggested answer first, before \`init_app\` and before any SQL. \`init_app\` only records \`data\`: a new schema is yours to create with the datatable SQL tool. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing. +- To create a new app, use \`init_app\` with a path, optional summary, a framework (\`react19\` / \`react18\` / \`svelte5\`) and, for an app that stores data, its \`data\` setup. "Starting a new app" below decides the framework and when the data setup is asked. Ask each of its questions with \`askUserQuestion\`, one question per call with the suggested answer first, before \`init_app\` and before any SQL. \`init_app\` only records \`data\`: a new schema is yours to create with the datatable SQL tool. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing. - \`init_app\` seeds a starter inline runnable named \`a\` (bun, \`main(x: string) => string\`) so the React/Svelte demo button works on first render. Replace or remove it once you start building real backend runnables. - Frontend file paths start with \`/\` (e.g. \`/index.tsx\`, \`/App.tsx\`, \`/styles.css\`). Use \`write_app_file\` / \`patch_app_file\` / \`delete_app_file\`. - Backend inline runnables are addressed as \`backend//main.{ts|py}\` from the file tools, but you create or update them via \`write_app_runnable\` / \`delete_app_runnable\` (which take the runnable shape directly: \`{ name, type, inlineScript?, path?, staticInputs? }\`). diff --git a/frontend/src/lib/components/handlerSlackArg.test.ts b/frontend/src/lib/components/handlerSlackArg.test.ts new file mode 100644 index 0000000000..b087a55f26 --- /dev/null +++ b/frontend/src/lib/components/handlerSlackArg.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { syncSlackBotArg } from './handlerSlackArg' + +const AUTO = '$res:f/slack_bot/bot_token' + +describe('syncSlackBotArg', () => { + it("leaves a custom handler's slack value alone", () => { + const args = { slack: '$res:u/me/my_slack' } + syncSlackBotArg(args, false, false) + syncSlackBotArg(args, false, true) + expect(args).toEqual({ slack: '$res:u/me/my_slack' }) + }) + + it('sets the bot token for the built-in and drops it when leaving', () => { + const args: Record = {} + syncSlackBotArg(args, true, false) + expect(args.slack).toBe(AUTO) + syncSlackBotArg(args, false, true) + expect(args).toStrictEqual({}) + }) +}) diff --git a/frontend/src/lib/components/handlerSlackArg.ts b/frontend/src/lib/components/handlerSlackArg.ts new file mode 100644 index 0000000000..6b8e4d6f5c --- /dev/null +++ b/frontend/src/lib/components/handlerSlackArg.ts @@ -0,0 +1,17 @@ +import { WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts' + +const AUTO_SLACK_ARG = '$res:' + WORKSPACE_SLACK_BOT_TOKEN_PATH + +// `slack` belongs to the Slack built-in only while it holds the value the picker set; any other +// value is a custom handler's own argument and must survive handler switches and reloads. +export function syncSlackBotArg( + args: Record, + isSlackBuiltin: boolean, + wasSlackBuiltin: boolean +) { + if (isSlackBuiltin) { + args.slack = AUTO_SLACK_ARG + } else if (wasSlackBuiltin && args.slack === AUTO_SLACK_ARG) { + delete args.slack + } +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 83fba2b8ac..27906dddee 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -5,6 +5,7 @@ import { instanceBannerFormError } from './instanceBanner' import { ACCENT_COLOR_SETTING, parseAccentColor } from '$lib/accentColor' import { parseMaxTokenExpirationDays } from '$lib/tokenExpiration' import { writable } from 'svelte/store' +import { OidcService } from '$lib/gen' /** * Bumped after instance settings are successfully saved. Settings whose display @@ -830,6 +831,7 @@ export const settings: Record = { } ], 'DB Health': [], + 'OIDC signing': [], Registries: [ { label: 'Instance Python Version', @@ -1344,6 +1346,13 @@ export const instanceSettingsNavigationGroups = [ aiId: 'instance-settings-scim-saml', aiDescription: 'Instance SCIM/SAML settings', isEE: true + }, + { + id: 'oidc_signing', + label: 'OIDC signing', + aiId: 'instance-settings-oidc-signing', + aiDescription: 'Keys that sign and verify the OIDC tokens Windmill issues', + isEE: true } ] }, @@ -1501,6 +1510,7 @@ export const tabToCategoryMap: Record = { github_enterprise_app: 'GitHub App', websocket: 'WebSocket', db_health: 'DB Health', + oidc_signing: 'OIDC signing', lsp: 'LSP' } @@ -1510,6 +1520,30 @@ export const tabToAuthSubTab: Record = { scim_saml: 'scim' } +/** + * "OIDC signing" is listed only on instances that sign with an external signer: its page + * says nothing actionable elsewhere. + */ +export function listedNavigationGroups( + groups: G[], + externalOidcSigner: boolean +): G[] { + if (externalOidcSigner) return groups + return groups.map((group) => ({ + ...group, + items: group.items.filter((item) => item.id !== 'oidc_signing') + })) +} + +/** The status endpoint is EE only: call it once the license is known. */ +export async function usesExternalOidcSigner(): Promise { + try { + return (await OidcService.getOidcSigningStatus()).mode === 'external' + } catch { + return false + } +} + // Navigation groups for the initial setup flow (no Users tab) export const setupNavigationGroups = instanceSettingsNavigationGroups .map((group) => ({ @@ -1538,6 +1572,7 @@ export const categoryToTabMap: Record = { 'GitHub App': 'github_enterprise_app', WebSocket: 'websocket', 'DB Health': 'db_health', + 'OIDC signing': 'oidc_signing', LSP: 'lsp' } diff --git a/frontend/src/lib/components/instanceSettings/OidcSigningStatus.svelte b/frontend/src/lib/components/instanceSettings/OidcSigningStatus.svelte new file mode 100644 index 0000000000..e223b7452c --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/OidcSigningStatus.svelte @@ -0,0 +1,100 @@ + + +{#if loadError} + {loadError} +{:else if status} +
+
+ Tokens are signed by: + {modeLabels[status.mode]} +
+ {#if status.signer} +
+ Last key listing: + {status.signer.last_refresh_success + ? displayDate(status.signer.last_refresh_success) + : 'never'} + {#if status.signer.breaker !== 'closed'} + Signer unreachable + {:else if !status.signer.last_refresh_error && status.signer.last_refresh_success} + Signer reachable + {/if} +
+ {#if status.signer.last_refresh_error} + + {status.signer.last_refresh_error} + + {/if} + {#if status.signer.keys_from_store} + + The keys below are those stored by the last accepted listing. Tokens cannot be signed + until the signer answers. + + {/if} + {#if status.signer.tls_files_error} + + {status.signer.tls_files_error}. The previous certificate and key are kept. + + {/if} + {/if} + + + + Key ID (kid) + Algorithm + State + Source + + + + {#each status.keys as key, i (`${key.source}:${key.kid ?? i}`)} + + + + {key.kid} + {#if key.kid}{/if} + + + {key.alg} + + {key.state} + + {key.source} + + {/each} + + + {#if status.keys.length === 0} +
The JWKS serves no key.
+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/mcp/providerIcon.ts b/frontend/src/lib/components/mcp/providerIcon.ts index ed8d059a57..a666b2b1c5 100644 --- a/frontend/src/lib/components/mcp/providerIcon.ts +++ b/frontend/src/lib/components/mcp/providerIcon.ts @@ -12,8 +12,11 @@ import { findMcpEntry, findMcpEntryByUrl } from './registry' * the chat does not otherwise reach that barrel. `import.meta.glob` gives the * filenames at build time and the module only when a match is actually used, so * a workspace with two connections downloads two icons. + * + * The pattern is relative because svelte-package rewrites `$lib` only in import + * specifiers: an aliased glob ships as-is and no consumer's bundler expands it. */ -const iconModules = import.meta.glob('$lib/components/icons/*.svelte') as Record< +const iconModules = import.meta.glob('../icons/*.svelte') as Record< string, () => Promise<{ default: Component }> > diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index f708b97efe..ca23505ff1 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -21,6 +21,7 @@ genWmillTs, normalizeRawAppRuntimeLogs, createRawAppBuildTracker, + RAW_APP_BUILD_WAIT_MS, type Runnable, type RawAppRuntimeLogEntry, type RawAppRuntimeLogRequester, @@ -488,6 +489,17 @@ addEventListener('message', function (e) { // Latest UI Builder error; cleared on next successful build. let buildError = $state(undefined) const buildTracker = createRawAppBuildTracker() + // The preview shell can be ready before the UI Builder has the files, so a fresh + // editor counts as building: reads made then would see an empty page. + buildTracker.start() + const BUILD_PENDING_TEXT = `The app is still building (or its build never started, e.g. no index entrypoint file): no build result arrived within ${RAW_APP_BUILD_WAIT_MS / 1000} seconds, so the preview shows an older build or nothing yet. Call get_app_runtime_logs to check the build, then retry.` + // A failed build never reaches the preview, which keeps the last good one, if any. + const buildFailedText = (error: string) => + `The latest build failed, so the preview does not include the latest edits (it shows the last build that succeeded, or nothing if none has). Fix the build first. Build error:\n${error}` + const SDK_CONSENT_PENDING_TEXT = + 'The preview is waiting for the user to approve the permissions the app declares in policy.frontend_sdk_scopes, so the app code is not running and the page is empty. This is not a bug in the app: do not change the code. Ask the user to approve the permissions in the preview, then retry.' + const SDK_STARTING_TEXT = + 'The preview is still loading the permissions the app declares in policy.frontend_sdk_scopes, so the app code has not started yet. Retry in a moment.' const PREVIEW_SETTLE_MS = 1000 let editorDestroyed = false // Latest uncaught runtime error thrown by the rendered app; cleared on next build. @@ -496,7 +508,7 @@ addEventListener('message', function (e) { // entrypoint defines a component without ever mounting it. Cleared on next build. let emptyRender = $state(false) // The repair hint above has to match the app's framework: only React apps have - // an `index.tsx` and `createRoot`; Svelte and Vue mount from `index.ts`. Keyed + // an `index.tsx` and `createRoot`; Svelte mounts from `index.ts`. Keyed // off file extensions rather than exact template filenames, which users rename. let mountHint = $derived.by(() => { const paths = Object.keys(files ?? {}).map((p) => p.replace(/^\//, '')) @@ -507,9 +519,6 @@ addEventListener('message', function (e) { call: "mount(App, { target: document.getElementById('root')! })" } } - if (paths.some((p) => p.endsWith('.vue'))) { - return { entrypoint: entrypoint ?? 'index.ts', call: "createApp(App).mount('#root')" } - } return { entrypoint: entrypoint ?? 'index.tsx', call: "createRoot(document.getElementById('root')!).render()" @@ -1979,6 +1988,18 @@ addEventListener('message', function (e) { // Live DOM inspection for the session chat. The element is re-read on every call, // so the model always sees the current render. const requestDomQuery: RawAppDomRequester = async (query: RawAppDomQuery) => { + await buildTracker.wait() + if (editorDestroyed) return undefined + if (buildTracker.pending) return { text: BUILD_PENDING_TEXT } + if (sdkPrompt) return { text: SDK_CONSENT_PENDING_TEXT } + if (previewSdkPending) return { text: SDK_STARTING_TEXT } + const res = await readPreviewDom(query) + return res && buildError !== undefined + ? { text: `${buildFailedText(buildError)}\n\n${res.text}` } + : res + } + + const readPreviewDom: RawAppDomRequester = async (query: RawAppDomQuery) => { const selector = query.selector?.trim() const res = await requestFromPreview('domRead', { selector }) if (!res) return undefined @@ -2145,6 +2166,12 @@ addEventListener('message', function (e) { // reconstructs the DOM into an SVG foreignObject, and a WebGL canvas is only // captured when its context was created with preserveDrawingBuffer. const captureScreenshot: RawAppScreenshotRequester = async () => { + await buildTracker.wait() + if (editorDestroyed) throw new Error('The app preview was closed') + if (buildTracker.pending) throw new Error(BUILD_PENDING_TEXT) + if (sdkPrompt) throw new Error(SDK_CONSENT_PENDING_TEXT) + if (previewSdkPending) throw new Error(SDK_STARTING_TEXT) + if (buildError !== undefined) throw new Error(buildFailedText(buildError)) const body = await requestFromPreview('elementRect', { selector: 'body' }) if (!body?.rect) throw new Error('App preview is not ready') const { width, height } = body.rect diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte index 200e76d85b..ddbdae5bf3 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte @@ -15,6 +15,7 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import Tabs from '../common/tabs/Tabs.svelte' import { Tab } from '../common' + import { SQL_LANGUAGES } from '$lib/infer' import RawAppInputsSpecEditor from './RawAppInputsSpecEditor.svelte' import SplitPanesWrapper from '../splitPanes/SplitPanesWrapper.svelte' import SchemaForm from '../SchemaForm.svelte' @@ -80,6 +81,13 @@ } } + // A SQL argument named after a viewer property is most often a per-viewer filter, + // which only a ctx field protects. `wmill app lint` warns on the same names. + const VIEWER_IDENTITY_ARGS = ['username', 'email', 'groups'] + let isSql = $derived( + isRunnableByName(runnable) && SQL_LANGUAGES.includes(runnable.inlineScript?.language ?? '') + ) + let selectedTab = $state('test') let args = $state({}) let hubFlowPreview: OpenFlow | undefined = $state(undefined) @@ -266,6 +274,7 @@ documentationLink={meta?.['documentationLink']} allowTypeChange={meta?.['allowTypeChange']} displayType + suggestCtx={isSql && VIEWER_IDENTITY_ARGS.includes(k)} /> {/each}
diff --git a/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte index 67ead38501..859a693ca9 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte @@ -13,6 +13,7 @@ import { fieldTypeToTsType } from '../apps/utils' import type { InputType } from '../apps/inputType' import Select from '$lib/components/select/Select.svelte' + import { Alert } from '$lib/components/common' import { userStore } from '$lib/stores' import { useOperatingUser, @@ -67,6 +68,8 @@ allowTypeChange?: boolean loading?: boolean documentationLink?: string | undefined + /** Warn that this input, left as a frontend input, is open to any value a viewer sends. */ + suggestCtx?: boolean } let { @@ -85,7 +88,8 @@ displayType = false, allowTypeChange = true, loading = false, - documentationLink = undefined + documentationLink = undefined, + suggestCtx = false }: Props = $props() run(() => { @@ -193,6 +197,11 @@
{:else if componentInput?.type === 'user' || componentInput?.type == undefined} Field's value is a frontend input + {#if suggestCtx} + + If this identifies the viewer, switch it to context so the server fills it. + + {/if} {/if} {#if componentInput?.type === 'user' && ((fieldType == 'object' && format?.startsWith('resource-') && format !== 'resource-s3_object') || fieldType == 'resource')}
diff --git a/frontend/src/lib/components/raw_apps/templates.ts b/frontend/src/lib/components/raw_apps/templates.ts index 50eae99e97..00d7594386 100644 --- a/frontend/src/lib/components/raw_apps/templates.ts +++ b/frontend/src/lib/components/raw_apps/templates.ts @@ -1,5 +1,5 @@ // The starter app is a tour of what a raw app can do: each card calls one of the -// STARTER_RUNNABLES below through a different `./wmill` export. The three framework +// STARTER_RUNNABLES below through a different `./wmill` export. The framework // variants share `indexCss` and must stay in step with each other and with the runnables. const reactIndex = ` @@ -13,7 +13,7 @@ root.render(); ` const STATS_SAMPLE_TEXT = - 'Windmill turns scripts into apps. Write the frontend in React, Svelte or Vue, and the backend in TypeScript, Python or any other supported language. Scripts run on workers, and the app calls them like functions.' + 'Windmill turns scripts into apps. Write the frontend in React or Svelte, and the backend in TypeScript, Python or any other supported language. Scripts run on workers, and the app calls them like functions.' const appTsx = `import React, { useState } from 'react' import { backend, backendAsync, getJob, streamJob } from './wmill' @@ -516,230 +516,6 @@ const app = mount(App, { target: document.getElementById("root")! }); export default app; ` -const appVue = ` - -` - -const indexVue = `import { createApp } from 'vue' -import App from './App.vue' -import "./index.css"; - -createApp(App).mount('#root')` - const indexCss = `/* The app renders inside an iframe whose host page has its own theme the iframe cannot read, so paint an opaque surface here: without one the host's background shows through and dark text lands on it unreadable. */ @@ -1158,23 +934,10 @@ export const svelte5Template = { }` } -export const vueTemplate = { - '/index.ts': indexVue, - '/App.vue': appVue, - '/index.css': indexCss, - '/package.json': `{ - "dependencies": { - "core-js": "3.26.1", - "vue": "3.5.13" - } -}` -} - export const FRAMEWORK_TEMPLATES = { react19: react19Template, react18: react18Template, - svelte5: svelte5Template, - vue: vueTemplate + svelte5: svelte5Template } export type FrameworkKey = keyof typeof FRAMEWORK_TEMPLATES @@ -1551,171 +1314,6 @@ export async function main() { } } -export const appVueRouter = ` - - - - -` - export const appSvelteRouter = `