From 2a40f25bda81d41b9f578338c85656744f59ebfc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 21 Sep 2026 23:36:24 +0200 Subject: [PATCH 01/34] fix: add missing credential fields to settings redaction lists (#11272) * fix: redact Azure and AWS credential fields in instance settings Co-Authored-By: Claude Opus 5 (1M context) * fix: redact the GitHub App private key in the settings change log Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../windmill-common/src/instance_config.rs | 23 ++++++++++++++++++- .../lib/components/InstanceSettings.svelte | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 14ef869b37..f4884d736e 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -1029,9 +1029,10 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ ), ( "object_store_cache_config", - &["secret_key", "serviceAccountKey"], + &["secret_key", "serviceAccountKey", "accessKey"], ), ("custom_instance_pg_databases", &["user_pwd"]), + ("github_enterprise_app", &["private_key"]), ]; fn redact_json_value(value: &serde_json::Value) -> serde_json::Value { @@ -2647,6 +2648,26 @@ mod tests { assert!(formatted.contains("****")); } + #[test] + fn format_setting_value_redacts_nested_credentials() { + let val = serde_json::json!({ + "type": "Azure", + "accountName": "acct", + "containerName": "c", + "accessKey": "azure-storage-account-key-12345" + }); + let formatted = format_setting_value("object_store_cache_config", &val); + assert!(!formatted.contains("azure-storage-account-key-12345")); + assert!(formatted.contains("acct")); + + let val = serde_json::json!({ + "app_id": 1, + "private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEsecretbody\n-----END RSA PRIVATE KEY-----" + }); + let formatted = format_setting_value("github_enterprise_app", &val); + assert!(!formatted.contains("MIIEsecretbody")); + } + #[test] #[should_panic(expected = "literal_value() called on unresolved secret ref")] fn string_or_secret_ref_literal_value_panics_on_ref() { diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index d87de125a9..3c511fc002 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -727,8 +727,8 @@ // Each entry maps a top-level key to its sensitive sub-field names. const nestedSensitiveFields: Record = { smtp_settings: ['smtp_password'], - secret_backend: ['token'], - object_store_cache_config: ['secret_key', 'serviceAccountKey'], + secret_backend: ['token', 'client_secret', 'secret_access_key'], + object_store_cache_config: ['secret_key', 'serviceAccountKey', 'accessKey'], custom_instance_pg_databases: ['user_pwd'], rsa_keys: ['private_key'], github_enterprise_app: ['private_key'] From 2699d5065ea3dfe985636733fb6022ca23187291 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 00:54:34 +0200 Subject: [PATCH 02/34] fix: check workflow-as-code task tags against CUSTOM_TAGS (#11273) * fix: check workflow-as-code task tags against CUSTOM_TAGS Co-Authored-By: Claude Opus 5 (1M context) * fix: resolve and check every WAC child before the parent parks Co-Authored-By: Claude Opus 5 (1M context) * fix: build parent-code WAC children at push time, not in the pre-pass Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/bun_jobs.rs | 56 ++++ backend/windmill-worker/src/bun_executor.rs | 269 +++++++++++--------- 2 files changed, 209 insertions(+), 116 deletions(-) diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 63d5341ba3..14f8c422a4 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1303,6 +1303,62 @@ async fn test_bun_wac_inline_task_cache_is_per_task(db: Pool) -> anyho Ok(()) } +/// A task that picks its own worker tag is held to CUSTOM_TAGS like a flow step: with none +/// set, only a superadmin may send it to `gpu`, so the workflow fails instead of queueing it. +#[sqlx::test(fixtures("base"))] +async fn test_bun_wac_task_tag_outside_custom_tags(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +import { workflow, task } from "windmill-client"; + +const onGpu = task(async (n: number) => { + return n * 2; +}, { tag: "gpu" }); + +export const main = workflow(async (n: number) => { + return await onGpu(n); +}); +"# + .to_owned(); + + let job = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Bun, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + })) + .arg("n", serde_json::json!(5)) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, port) + .await; + + assert!(!job.success, "the workflow must fail"); + let result = job.json_result().unwrap(); + let message = result["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("task 'onGpu' cannot run on tag 'gpu'"), + "unexpected error: {result}" + ); + let children: i64 = sqlx::query_scalar("SELECT count(*) FROM v2_job WHERE parent_job = $1") + .bind(job.id) + .fetch_one(&db) + .await?; + assert_eq!(children, 0, "no child is queued on the refused tag"); + Ok(()) +} + // ============================================================================ // Environment Variable Tests // ============================================================================ diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 117fbda086..7ed7b8d5da 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2689,13 +2689,13 @@ try {{ /// Resolve a module file from the parent script's modules map. /// For Script jobs, fetches from the `script` table by hash. /// For Preview jobs, fetches from `v2_job.raw_code` (modules stored inline). -fn resolve_parent_module( - modules: &Option>, +fn resolve_parent_module<'a>( + modules: &'a Option>, module_key: &str, -) -> error::Result { +) -> error::Result<&'a windmill_common::scripts::ScriptModule> { if let Some(modules) = modules { if let Some(module) = modules.get(module_key) { - return Ok(module.clone()); + return Ok(module); } } Err(error::Error::ExecutionErr(format!( @@ -2720,7 +2720,10 @@ pub async fn handle_wac_v2_output( }; use serde_json::Value; use windmill_common::get_latest_flow_version_info_for_path; - use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode}; + use windmill_common::jobs::{ + check_tag_available_for_workspace_internal, script_path_to_payload, JobKind, JobPayload, + RawCode, + }; use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, }; @@ -2950,6 +2953,115 @@ pub async fn handle_wac_v2_output( ))), }?; + // Resolve what every task runs, and check any tag it picks, before the parent parks: + // a task that cannot be pushed then fails the dispatch with no sibling queued. The + // parent's code and input are built per child at push time, so a fan-out never holds + // a copy of them for each child at once. + enum ChildRunnable<'a> { + // Re-runs the parent with `_executing_key`. + Parent, + // A `./` module of the parent script. + Module(&'a windmill_common::scripts::ScriptModule), + // A deployed script or flow. + Deployed(JobPayload), + } + struct ResolvedChild<'a> { + runnable: ChildRunnable<'a>, + email: String, + permissioned_as: String, + } + let mut children: Vec = Vec::with_capacity(num_steps); + for step in &steps { + let (runnable, on_behalf_of) = match step.dispatch_type.as_str() { + "script" if step.script.starts_with("./") => { + // Module-relative path: resolve from parent script's modules + let module_key = step.script.strip_prefix("./").unwrap(); + let module = resolve_parent_module(modules, module_key)?; + // Inline module code, not a separate runnable: it has no + // identity of its own and runs as the parent. + (ChildRunnable::Module(module), None) + } + "script" => { + // Resolve script path to job payload (handles hash, lang, etc.) + let (payload, _, _, _, _, on_behalf_of) = script_path_to_payload( + &step.script, + None, // no authed db for background workers + db.clone(), + &job.workspace_id, + Some(true), // skip preprocessor + ) + .await?; + (ChildRunnable::Deployed(payload), on_behalf_of) + } + "flow" => { + let flow_info = get_latest_flow_version_info_for_path( + None, + db, + &job.workspace_id, + &step.script, + true, + ) + .await?; + let payload = JobPayload::Flow { + path: step.script.clone(), + dedicated_worker: flow_info.dedicated_worker, + apply_preprocessor: false, + version: flow_info.version, + labels: flow_info.labels.clone(), + }; + let on_behalf_of = flow_info.on_behalf_of(&job.workspace_id, db).await?; + (ChildRunnable::Deployed(payload), on_behalf_of) + } + // "inline" — re-run parent with _executing_key + _ => (ChildRunnable::Parent, None), + }; + + // A target runnable that opts into on-behalf-of runs under its own + // identity, never the caller's, so a step that reaches it through a + // workflow cannot widen or narrow its permissions. `created_by` still + // credits the caller, matching how the run API pushes these jobs. + let (email, permissioned_as) = match on_behalf_of { + Some(on_behalf_of) => (on_behalf_of.email, on_behalf_of.permissioned_as), + None => ( + job.permissioned_as_email.clone(), + job.permissioned_as.clone(), + ), + }; + + // A task inheriting the parent's tag skips the check: that tag was + // checked when the parent was pushed, and a dedicated worker's tag is + // never in CUSTOM_TAGS. + if let Some(tag) = step + .tag + .as_deref() + .filter(|t| !t.is_empty() && *t != job.tag.as_str()) + { + let is_super_admin = + windmill_common::auth::is_super_admin_email(db, &email).await?; + check_tag_available_for_workspace_internal( + db, + &job.workspace_id, + tag, + is_super_admin, + None, + ) + .warn_after_seconds_with_sql( + 1, + "check_tag_available_for_workspace_internal".to_string(), + ) + .await + .map_err(|e| match e { + error::Error::BadRequest(msg) => error::Error::BadRequest(format!( + "task '{}' cannot run on tag '{tag}': {msg}", + step.name + )), + e => e, + })?; + } + + children.push(ResolvedChild { runnable, email, permissioned_as }); + } + // Step 1: Save checkpoint, suspend parent, and seed child checkpoints // in a single transaction — all BEFORE children become visible. let segment_ms; @@ -3034,106 +3146,46 @@ pub async fn handle_wac_v2_output( // partial failure (e.g. pushing child 3 of 5 fails). let mut pushed_ids: Vec = Vec::with_capacity(num_steps); let push_result: error::Result<()> = async { - for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + for ((step, (_, child_uuid)), child) in + steps.iter().zip(job_ids.iter()).zip(children) + { // A task with a runnable of its own (a deployed script or flow) queues // at that runnable's priority; any other task is the parent's code and // queues at the parent's. let own_runnable = matches!(step.dispatch_type.as_str(), "script" | "flow") && !step.script.starts_with("./"); - // Resolve job payload based on dispatch_type - let (job_payload, child_args, is_external, on_behalf_of) = - match step.dispatch_type.as_str() { - "script" if step.script.starts_with("./") => { - // Module-relative path: resolve from parent script's modules - let module_key = step.script.strip_prefix("./").unwrap(); - let module = resolve_parent_module(modules, module_key)?; - let payload = JobPayload::Code(RawCode { - content: module.content, - path: job.runnable_path.clone(), - hash: None, - language: module.language, - lock: module.lock, - cache_ttl: None, - cache_ignore_s3_path: None, - dedicated_worker: None, - concurrency_settings: ConcurrencySettingsWithCustom::default(), - debouncing_settings: DebouncingSettings::default(), - modules: None, - tag: None, - }); - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - // Inline module code, not a separate runnable: it has no - // identity of its own and runs as the parent. - (payload, step_args, true, None) - } - "script" => { - // Resolve script path to job payload (handles hash, lang, etc.) - let (payload, _, _, _, _, on_behalf_of) = script_path_to_payload( - &step.script, - None, // no authed db for background workers - db.clone(), - &job.workspace_id, - Some(true), // skip preprocessor - ) - .await?; - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - (payload, step_args, true, on_behalf_of) - } - "flow" => { - let flow_info = get_latest_flow_version_info_for_path( - None, - db, - &job.workspace_id, - &step.script, - true, - ) - .await?; - let payload = JobPayload::Flow { - path: step.script.clone(), - dedicated_worker: flow_info.dedicated_worker, - apply_preprocessor: false, - version: flow_info.version, - labels: flow_info.labels.clone(), - }; - let on_behalf_of = - flow_info.on_behalf_of(&job.workspace_id, db).await?; - let step_args: HashMap> = step - .args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect(); - (payload, step_args, true, on_behalf_of) - } - _ => { - // "inline" — re-run parent with _executing_key - ( - job_payload_template.clone(), - parent_args.clone(), - false, - None, - ) - } - }; - - let push_args = PushArgs { args: &child_args, extra: None }; + let is_external = !matches!(child.runnable, ChildRunnable::Parent); + let job_payload = match child.runnable { + ChildRunnable::Parent => job_payload_template.clone(), + ChildRunnable::Module(module) => JobPayload::Code(RawCode { + content: module.content.clone(), + path: job.runnable_path.clone(), + hash: None, + language: module.language, + lock: module.lock.clone(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + modules: None, + tag: None, + }), + ChildRunnable::Deployed(payload) => payload, + }; + let step_args: Option>> = + is_external.then(|| { + step.args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect() + }); + let push_args = + PushArgs { args: step_args.as_ref().unwrap_or(&parent_args), extra: None }; // Apply step-level overrides to payload (cache, concurrency) let mut job_payload = job_payload; @@ -3181,21 +3233,6 @@ pub async fn handle_wac_v2_output( } } - // A target runnable that opts into on-behalf-of runs under its own - // identity, never the caller's, so a step that reaches it through a - // workflow cannot widen or narrow its permissions. `created_by` still - // credits the caller, matching how the run API pushes these jobs. - let (child_email, child_permissioned_as) = match on_behalf_of.as_ref() { - Some(on_behalf_of) => ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - ), - None => ( - job.permissioned_as_email.as_str(), - job.permissioned_as.clone(), - ), - }; - let (_, mut tx) = push( db, PushIsolationLevel::IsolatedRoot(db.clone()), @@ -3203,8 +3240,8 @@ pub async fn handle_wac_v2_output( job_payload, push_args, &job.created_by, - child_email, - child_permissioned_as, + &child.email, + child.permissioned_as, None, None, None, From 4d12ea461454f89367bd91daa7e8c39a89035507 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 01:40:53 +0200 Subject: [PATCH 03/34] feat: deploy from the UI to a workspace on another instance (#11245) * feat: deploy from the UI to a workspace on another instance Co-Authored-By: Claude Opus 5 (1M context) * fix: keep the remote deploy proxy from being spent by a link Co-Authored-By: Claude Opus 5 (1M context) * fix: key remote deploy proxy URLs instead of a global client header Co-Authored-By: Claude Opus 5 (1M context) * fix: keep the remote deploy proxy key out of logs and restricted hands Co-Authored-By: Claude Opus 5 (1M context) * fix: serialize remote deploy connect with account and key changes Co-Authored-By: Claude Opus 5 (1M context) * feat: key remote deploy tokens to the account and connect by signing in Co-Authored-By: Claude Opus 5 (1M context) * fix: bind remote deploy connect to its target and order its locks Co-Authored-By: Claude Opus 5 (1M context) * fix: serialize remote deploy connect with target changes Co-Authored-By: Claude Opus 5 (1M context) * docs: list every lock remote deploy connect takes in auth-surface Co-Authored-By: Claude Opus 5 (1M context) * fix: never wait on the membership lock in remote deploy connect Co-Authored-By: Claude Opus 5 (1M context) * fix: keep a superseded target response out of the settings form Co-Authored-By: Claude Opus 5 (1M context) * fix: void stale remote deploy tokens on read instead of locking in connect Co-Authored-By: Claude Opus 5 (1M context) * fix: void remote deploy tokens older than the last target change Co-Authored-By: Claude Opus 5 (1M context) * fix: order remote deploy connections by when their connect started Co-Authored-By: Claude Opus 5 (1M context) * fix: bind stored remote deploy tokens to the membership and target they were connected under Co-Authored-By: Claude Opus 5 (1M context) * fix: answer remote deploy connect without settings as no target Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...e6641817d09528f9d1b7210abe9881b817710.json | 32 + ...f73daf9e09cac11685988bc861cc348a0c518.json | 22 + ...bb04695b1c50e17755dad3bd162254f67458f.json | 16 + ...00ca014154e76eb322646e7e4e753b6420acd.json | 12 + ...72e4a01f50d48b76b90746eef72919c408e44.json | 28 + ...c621b9b03b92f6c20ab1bdd1637232f26952d.json | 41 + ...2750560e459acc7c83c48d47d7bdad29051be.json | 12 + ...e848344974bc2271459ba4ae5952274c11429.json | 12 + ...31b3f107fc6d0312ac3da9247a922bcb05b67.json | 43 + ...2017c41b9a0e3985a25372e5f64723124e473.json | 15 + ...32f1bca7b235af9db581147d838454a4c9083.json | 16 + ...3786891eb99e9ba86b11cfb17d28856babd2e.json | 14 + ...530f057d91c0e0986fdc0de7d9a3ef0c143e2.json | 15 + ...7bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json | 12 + ...3d29ae8b924200c46da457a9346379c60fa9f.json | 15 + ...0f43e27a6eea453415495b66c50045defacff.json | 12 + ...1c363a4e5719b5ceb17e8194c2eb038f59c68.json | 12 + ...6dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json | 12 + ...e975ff2b1baee63651e0ce79156b331389923.json | 15 + ...f124a055eb44b3c5892fd18ff7baf6010ae9.json} | 4 +- ...f67554643d0e730bec2e64840ad3d1862e540.json | 12 + ...03c2b12a3cf12ee81e67b0e9e520b3315e7d6.json | 14 + ...45cb080009a7a6d9ef33111755b470ff5abe2.json | 15 + ...3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json | 12 + ...f006687bd9de4611203b5dd0c28eacee02842.json | 15 + ...52fb2b4cf7983049d2c490940df360c7e2b30.json | 15 - ...0a0c0827dea6115daf9d006f8ccda0d041d76.json | 20 + backend/Cargo.lock | 3 + .../20260919213810_remote_deploy.down.sql | 4 + .../20260919213810_remote_deploy.up.sql | 35 + backend/summarized_schema.txt | 5 +- .../tests/remote_deploy.rs | 297 +++++++ backend/windmill-api-users/src/users.rs | 25 +- backend/windmill-api-workspaces/Cargo.toml | 3 + backend/windmill-api-workspaces/src/lib.rs | 1 + .../src/remote_deploy.rs | 794 ++++++++++++++++++ .../windmill-api-workspaces/src/workspaces.rs | 23 +- .../src/workspaces_extra.rs | 10 +- backend/windmill-api/openapi.yaml | 132 +++ backend/windmill-api/src/lib.rs | 6 + backend/windmill-api/src/tracing_init.rs | 5 +- backend/windmill-common/src/auth.rs | 13 +- docs/auth-surface.md | 46 + .../src/lib/components/DeployWorkspace.svelte | 206 ++++- .../ParentWorkspaceProtectionAlert.svelte | 7 +- .../lib/components/RemoteDeployConnect.svelte | 181 ++++ .../RemoteDeployTargetSetting.svelte | 111 +++ .../components/settings/TokensTable.svelte | 3 +- frontend/src/lib/remoteDeploy.test.ts | 44 + frontend/src/lib/remoteDeploy.ts | 101 +++ frontend/src/lib/utils_deployable.ts | 14 +- frontend/src/lib/utils_workspace_deploy.ts | 11 +- .../remote_deploy/callback/+page.svelte | 74 ++ .../user/remote_deploy_authorize/+page.svelte | 132 +++ .../(logged)/workspace_settings/+page.svelte | 7 + 55 files changed, 2678 insertions(+), 78 deletions(-) create mode 100644 backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json create mode 100644 backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json create mode 100644 backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json create mode 100644 backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json create mode 100644 backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json create mode 100644 backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json create mode 100644 backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json create mode 100644 backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json create mode 100644 backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json create mode 100644 backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json create mode 100644 backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json create mode 100644 backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json create mode 100644 backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json create mode 100644 backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json create mode 100644 backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json create mode 100644 backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json create mode 100644 backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json create mode 100644 backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json create mode 100644 backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json rename backend/.sqlx/{query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json => query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json} (79%) create mode 100644 backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json create mode 100644 backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json create mode 100644 backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json create mode 100644 backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json create mode 100644 backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json delete mode 100644 backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json create mode 100644 backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json create mode 100644 backend/migrations/20260919213810_remote_deploy.down.sql create mode 100644 backend/migrations/20260919213810_remote_deploy.up.sql create mode 100644 backend/windmill-api-integration-tests/tests/remote_deploy.rs create mode 100644 backend/windmill-api-workspaces/src/remote_deploy.rs create mode 100644 frontend/src/lib/components/RemoteDeployConnect.svelte create mode 100644 frontend/src/lib/components/RemoteDeployTargetSetting.svelte create mode 100644 frontend/src/lib/remoteDeploy.test.ts create mode 100644 frontend/src/lib/remoteDeploy.ts create mode 100644 frontend/src/routes/(root)/(logged)/remote_deploy/callback/+page.svelte create mode 100644 frontend/src/routes/(root)/(logged)/user/remote_deploy_authorize/+page.svelte diff --git a/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json b/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json new file mode 100644 index 0000000000..15d1703e5d --- /dev/null +++ b/backend/.sqlx/query-132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO remote_deploy_token\n (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key,\n connected_at, member_since, target_changed_at)\n SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10\n WHERE $9::timestamptz IS NOT NULL\n OR EXISTS (SELECT 1 FROM token WHERE token_hash = $11)\n ON CONFLICT (workspace_id, email) DO UPDATE SET\n base_url = EXCLUDED.base_url, remote_workspace_id = EXCLUDED.remote_workspace_id,\n token = EXCLUDED.token, remote_email = EXCLUDED.remote_email,\n proxy_key = EXCLUDED.proxy_key, connected_at = EXCLUDED.connected_at,\n member_since = EXCLUDED.member_since, target_changed_at = EXCLUDED.target_changed_at\n WHERE remote_deploy_token.connected_at < EXCLUDED.connected_at\n RETURNING connected_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "connected_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Varchar", + "Varchar", + "Timestamptz", + "Timestamptz", + "Timestamptz", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "132bab72036874c33805ff194fce6641817d09528f9d1b7210abe9881b817710" +} diff --git a/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json b/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json new file mode 100644 index 0000000000..45fbabdae9 --- /dev/null +++ b/backend/.sqlx/query-14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT remote_deploy_target FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "remote_deploy_target", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "14cef81d324504d9cc90a9ffeb9f73daf9e09cac11685988bc861cc348a0c518" +} diff --git a/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json b/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json new file mode 100644 index 0000000000..e698dbecb2 --- /dev/null +++ b/backend/.sqlx/query-207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = $1\n WHERE workspace_id = $2 AND email = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "207db6c65949b85fda6d6289921bb04695b1c50e17755dad3bd162254f67458f" +} diff --git a/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json b/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json new file mode 100644 index 0000000000..b3b683eca1 --- /dev/null +++ b/backend/.sqlx/query-3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE token_prefix = 'GONE_TOKEN'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3a0ac41e645225bbe54b83f246500ca014154e76eb322646e7e4e753b6420acd" +} diff --git a/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json b/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json new file mode 100644 index 0000000000..9901e61b6a --- /dev/null +++ b/backend/.sqlx/query-3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, token FROM remote_deploy_token\n WHERE workspace_id = $1 AND token <> '' FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "token", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "3b6f1e9765de11ef2da0ec5388c72e4a01f50d48b76b90746eef72919c408e44" +} diff --git a/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json b/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json new file mode 100644 index 0000000000..b6babc98b0 --- /dev/null +++ b/backend/.sqlx/query-51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT clock_timestamp() AS \"started_at!\", s.remote_deploy_target,\n s.remote_deploy_target_changed_at,\n (SELECT u.created_at FROM usr u\n WHERE u.workspace_id = s.workspace_id AND u.email = $2) AS member_since\n FROM workspace_settings s WHERE s.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "remote_deploy_target", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "remote_deploy_target_changed_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 3, + "name": "member_since", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + true, + true, + null + ] + }, + "hash": "51e1741eb9e959945302fd6f7e8c621b9b03b92f6c20ab1bdd1637232f26952d" +} diff --git a/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json b/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json new file mode 100644 index 0000000000..65266cbae0 --- /dev/null +++ b/backend/.sqlx/query-530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, read_only)\n VALUES (encode(sha256('READ_ONLY_TOKEN'::bytea), 'hex'), 'READ_ONLY_', 'READ_ONLY_TOKEN',\n 'test@windmill.dev', 'read only', false, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "530fe81babd82e7b4a9142962c62750560e459acc7c83c48d47d7bdad29051be" +} diff --git a/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json b/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json new file mode 100644 index 0000000000..c29a42c56b --- /dev/null +++ b/backend/.sqlx/query-5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET remote_deploy_target_changed_at = '2000-01-01'\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5517b714165f959a1df1a0ac9cbe848344974bc2271459ba4ae5952274c11429" +} diff --git a/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json b/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json new file mode 100644 index 0000000000..7633c95854 --- /dev/null +++ b/backend/.sqlx/query-5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT t.token, t.remote_email, t.proxy_key, t.connected_at FROM remote_deploy_token t\n JOIN workspace_settings s ON s.workspace_id = t.workspace_id\n WHERE t.workspace_id = $1 AND t.email = $2 AND t.base_url = $3\n AND t.remote_workspace_id = $4 AND t.token <> ''\n AND s.remote_deploy_target_changed_at IS NOT DISTINCT FROM t.target_changed_at\n AND (EXISTS (SELECT 1 FROM usr u WHERE u.workspace_id = t.workspace_id\n AND u.email = t.email AND u.created_at = t.member_since)\n OR EXISTS (SELECT 1 FROM password p WHERE p.email = t.email AND p.super_admin))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "remote_email", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "proxy_key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "connected_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "5b292c49ce6b64f4de61f74e18831b3f107fc6d0312ac3da9247a922bcb05b67" +} diff --git a/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json b/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json new file mode 100644 index 0000000000..20b5bd1a94 --- /dev/null +++ b/backend/.sqlx/query-6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO remote_deploy_token\n (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key,\n connected_at)\n VALUES ($1, $2, '', '', '', '', '', clock_timestamp())\n ON CONFLICT (workspace_id, email) DO UPDATE SET\n token = '', proxy_key = '', connected_at = clock_timestamp()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "6b58f2df413a867a44311fd779a2017c41b9a0e3985a25372e5f64723124e473" +} diff --git a/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json b/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json new file mode 100644 index 0000000000..40b6e22db6 --- /dev/null +++ b/backend/.sqlx/query-81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE workspace_id = $1\n AND ($2::text IS NULL OR base_url <> $2 OR remote_workspace_id <> $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "81d5cceb2e405e1d17730dfa93a32f1bca7b235af9db581147d838454a4c9083" +} diff --git a/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json b/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json new file mode 100644 index 0000000000..c732c4549d --- /dev/null +++ b/backend/.sqlx/query-8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = $1 WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "8ce4461e7dd087782a239972c173786891eb99e9ba86b11cfb17d28856babd2e" +} diff --git a/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json b/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json new file mode 100644 index 0000000000..7d819b9b84 --- /dev/null +++ b/backend/.sqlx/query-960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "960d6eca64de008f821b1e96345530f057d91c0e0986fdc0de7d9a3ef0c143e2" +} diff --git a/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json b/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json new file mode 100644 index 0000000000..a7fe120abb --- /dev/null +++ b/backend/.sqlx/query-965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET connected_at = clock_timestamp() + interval '1 hour'\n WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "965f9fb9d73f21695cf9c4018107bf8dec0a9b1c38a6776f8b3db7f2f8a883b7" +} diff --git a/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json b/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json new file mode 100644 index 0000000000..112ca1603f --- /dev/null +++ b/backend/.sqlx/query-9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET remote_deploy_target = $1::jsonb,\n remote_deploy_target_changed_at = CASE\n WHEN remote_deploy_target IS DISTINCT FROM $1::jsonb THEN clock_timestamp()\n ELSE remote_deploy_target_changed_at END\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9b61979cc78e8b7b63ffa6a59cf3d29ae8b924200c46da457a9346379c60fa9f" +} diff --git a/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json b/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json new file mode 100644 index 0000000000..18951e2a87 --- /dev/null +++ b/backend/.sqlx/query-a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM usr WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a866da25370c8b829fd8038c55c0f43e27a6eea453415495b66c50045defacff" +} diff --git a/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json b/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json new file mode 100644 index 0000000000..6529400823 --- /dev/null +++ b/backend/.sqlx/query-a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE remote_deploy_token SET token = 'not-under-this-key'\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a9ae6c222727b7078fd25ed91f51c363a4e5719b5ceb17e8194c2eb038f59c68" +} diff --git a/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json b/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json new file mode 100644 index 0000000000..4ec7511c7c --- /dev/null +++ b/backend/.sqlx/query-af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin)\n VALUES (encode(sha256('GONE_TOKEN'::bytea), 'hex'), 'GONE_TOKEN', 'GONE_TOKEN',\n 'test@windmill.dev', 'gone', true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "af1bc312f13d7590a138e2de4996dab8e5ac9c774cf81df9da85ac3c40e8b3c7" +} diff --git a/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json b/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json new file mode 100644 index 0000000000..77cd542d6b --- /dev/null +++ b/backend/.sqlx/query-b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b2014ff45992b34c08f27419c26e975ff2b1baee63651e0ce79156b331389923" +} diff --git a/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json b/backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json similarity index 79% rename from backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json rename to backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json index 8a780ccda7..c15e0edb33 100644 --- a/backend/.sqlx/query-383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99.json +++ b/backend/.sqlx/query-b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n AND NOT starts_with(label, 'cli-login:')\n ))\n RETURNING token_prefix", + "query": "UPDATE token SET label = $1\n WHERE email = $2 AND token_prefix = $3\n AND (label IS NULL OR (\n label <> 'session'\n AND label <> 'guest_session'\n AND lower(label) NOT LIKE 'ephemeral%'\n AND label <> 'debugger-token'\n AND label NOT LIKE 'mcp-oauth-%'\n AND NOT starts_with(label, 'embed_app:')\n AND NOT starts_with(label, 'sdk_app:')\n AND NOT starts_with(label, 'impersonation:')\n AND NOT starts_with(label, 'cli-login:')\n AND NOT starts_with(label, 'remote-deploy:')\n ))\n RETURNING token_prefix", "describe": { "columns": [ { @@ -20,5 +20,5 @@ false ] }, - "hash": "383c80239525d9c4ee90e2f5cb6e46c8a0983c52ad15db083f84a9e344dc4c99" + "hash": "b30fb28004897cd1d201a416cdabf124a055eb44b3c5892fd18ff7baf6010ae9" } diff --git a/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json b/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json new file mode 100644 index 0000000000..7422dd2333 --- /dev/null +++ b/backend/.sqlx/query-c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE email = 'test@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "c96486a83ba0f4bf8daf3c525d6f67554643d0e730bec2e64840ad3d1862e540" +} diff --git a/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json b/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json new file mode 100644 index 0000000000..85c999e3ba --- /dev/null +++ b/backend/.sqlx/query-cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET remote_deploy_target = $1, remote_deploy_target_changed_at = now()\n WHERE workspace_id = 'test-workspace'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "cad45b99cd110589665e864f73403c2b12a3cf12ee81e67b0e9e520b3315e7d6" +} diff --git a/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json b/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json new file mode 100644 index 0000000000..cde3e698d9 --- /dev/null +++ b/backend/.sqlx/query-d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM remote_deploy_token WHERE workspace_id = $1 AND email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d21d9d7e37045529ae4b73febc145cb080009a7a6d9ef33111755b470ff5abe2" +} diff --git a/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json b/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json new file mode 100644 index 0000000000..4031e19231 --- /dev/null +++ b/backend/.sqlx/query-d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET created_at = created_at - interval '1 hour'\n WHERE workspace_id = 'test-workspace' AND email = 'test2@windmill.dev'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d305aed6172e25707492677b6cd3b8f3f6ea8f2c8f5bdc1a97736c8e52378ee9" +} diff --git a/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json b/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json new file mode 100644 index 0000000000..a1d873778f --- /dev/null +++ b/backend/.sqlx/query-ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at FROM workspace_settings WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ee3ed82783b5b7d2f41f16a1d94f006687bd9de4611203b5dd0c28eacee02842" +} diff --git a/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json b/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json deleted file mode 100644 index ec641bf32d..0000000000 --- a/backend/.sqlx/query-eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "eefa0588a6a927fd9b3f65e1df652fb2b4cf7983049d2c490940df360c7e2b30" -} diff --git a/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json b/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json new file mode 100644 index 0000000000..c3e7b0168d --- /dev/null +++ b/backend/.sqlx/query-f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM remote_deploy_token WHERE email = 'test2@windmill.dev'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "f9692bba979c2b5a22e65106a330a0c0827dea6115daf9d006f8ccda0d041d76" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 9a25928de4..ebd6dce019 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15526,6 +15526,7 @@ version = "1.816.0" dependencies = [ "axum 0.8.9", "chrono", + "constant_time_eq 0.3.1", "futures", "hex", "http 1.5.0", @@ -15533,6 +15534,7 @@ dependencies = [ "lazy_static", "magic-crypt", "regex", + "reqwest 0.13.5", "serde", "serde_json", "sha2 0.10.9", @@ -15541,6 +15543,7 @@ dependencies = [ "tokio", "tokio-postgres", "tracing", + "url", "uuid", "windmill-api-auth", "windmill-api-jobs", diff --git a/backend/migrations/20260919213810_remote_deploy.down.sql b/backend/migrations/20260919213810_remote_deploy.down.sql new file mode 100644 index 0000000000..ae26684ac6 --- /dev/null +++ b/backend/migrations/20260919213810_remote_deploy.down.sql @@ -0,0 +1,4 @@ +DROP TABLE IF EXISTS remote_deploy_token; +ALTER TABLE workspace_settings + DROP COLUMN IF EXISTS remote_deploy_target, + DROP COLUMN IF EXISTS remote_deploy_target_changed_at; diff --git a/backend/migrations/20260919213810_remote_deploy.up.sql b/backend/migrations/20260919213810_remote_deploy.up.sql new file mode 100644 index 0000000000..c32f81fd88 --- /dev/null +++ b/backend/migrations/20260919213810_remote_deploy.up.sql @@ -0,0 +1,35 @@ +-- The workspace on another Windmill instance this workspace deploys into from the UI: +-- `{"base_url": ..., "workspace_id": ...}`, and when it last changed. A token connected under an +-- earlier change counts for nothing, even once the setting points back at the target it was for. +ALTER TABLE workspace_settings + ADD COLUMN remote_deploy_target JSONB, + ADD COLUMN remote_deploy_target_changed_at TIMESTAMPTZ; + +-- One user's token for the remote deploy target, encrypted with the workspace key. +-- `base_url`/`remote_workspace_id` name the target it was granted for: a token is only +-- ever sent to that target, so re-pointing the workspace setting cannot redirect it. +-- Keyed by the account rather than by membership, since a superadmin deploys from workspaces +-- it is not a member of. The account key cascades: whatever deletes or renames an account +-- takes its tokens along, so a later account with the same address cannot inherit them. +CREATE TABLE remote_deploy_token ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + email VARCHAR(255) NOT NULL REFERENCES password(email) ON DELETE CASCADE ON UPDATE CASCADE, + base_url VARCHAR(1000) NOT NULL, + remote_workspace_id VARCHAR(50) NOT NULL, + token TEXT NOT NULL, + remote_email VARCHAR(255) NOT NULL, + -- Part of every proxy URL, and readable only by its owner through the API: a link from + -- elsewhere, which rides the session cookie, cannot know it and so cannot spend the token. + proxy_key VARCHAR(64) NOT NULL, + -- When the connect that wrote the row started, or when a disconnect emptied it. + connected_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- What the connect ran under: the `created_at` of the owner's membership (NULL for a + -- superadmin with none) and the target's `remote_deploy_target_changed_at`. The row counts + -- only while both are still the same, so a re-add or a target change voids it. + member_since TIMESTAMPTZ, + target_changed_at TIMESTAMPTZ, + PRIMARY KEY (workspace_id, email) +); + +GRANT ALL ON remote_deploy_token TO windmill_user; +GRANT ALL ON remote_deploy_token TO windmill_admin; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index ea756fc23c..169eb81681 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -171,6 +171,9 @@ postgres_trigger: path(char), script_path(char), is_flow(bool), workspace_id(cha FK: (workspace_id) -> workspace(id) raw_app: path(char), version(int), workspace_id(char), summary(char), edited_at(ts), data(text), extra_perms(jsonb), labels(text[]) FK: (workspace_id) -> workspace(id) +remote_deploy_token: workspace_id(char), email(char), base_url(char), remote_workspace_id(char), token(text), remote_email(char), proxy_key(char), connected_at(timestamptz), member_since(timestamptz), target_changed_at(timestamptz) + FK: (email) -> password(email) + FK: (workspace_id) -> workspace(id) resource: workspace_id(char), path(char), value(jsonb), description(text), resource_type(char), extra_perms(jsonb), edited_at(ts), created_by(char), labels(text[]) FK: (workspace_id) -> workspace(id) resource_type: workspace_id(char), name(char), schema(jsonb), description(text), edited_at(ts), created_by(char), format_extension(char), is_fileset(bool), display_name(char) @@ -234,7 +237,7 @@ workspace_protection_rule: workspace_id(char), name(char), rules(int), bypass_gr FK: (workspace_id) -> workspace(id) workspace_runnable_dependencies: flow_path(char), runnable_path(char), script_hash(bigint), runnable_is_flow(bool), workspace_id(char), app_path(char), id(bigint), runnable_is_agent(bool) FK: (app_path, workspace_id) -> app(path, workspace_id) | (flow_path, workspace_id) -> flow(path, workspace_id) -workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int), add_admins_and_developers_to_forks(bool) +workspace_settings: workspace_id(char), slack_team_id(char), slack_name(char), slack_command_script(char), slack_email(char), customer_id(char), plan(char), webhook(text), ai_config(jsonb), large_file_storage(jsonb), git_sync(jsonb), default_app(char), default_scripts(jsonb), deploy_ui(jsonb), mute_critical_alerts(bool), color(char), operator_settings(jsonb), teams_command_script(text), teams_team_id(text), teams_team_name(text), git_app_installations(jsonb), ducklake(jsonb), slack_oauth_client_id(char), slack_oauth_client_secret(char), datatable(jsonb), teams_team_guid(text), auto_invite(jsonb), error_handler(jsonb), success_handler(jsonb), public_app_execution_limit_per_minute(int), dbt_warehouses(jsonb), guest_access_enabled(bool), guest_jwt_public_key(text), guest_jwt_jwks_url(text), ai_sessions_backup_generation(int), add_admins_and_developers_to_forks(bool), remote_deploy_target(jsonb), remote_deploy_target_changed_at(ts) FK: (workspace_id) -> workspace(id) zombie_job_counter: job_id(uuid), counter(int) FK: (job_id) -> v2_job(id) diff --git a/backend/windmill-api-integration-tests/tests/remote_deploy.rs b/backend/windmill-api-integration-tests/tests/remote_deploy.rs new file mode 100644 index 0000000000..27447c3095 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/remote_deploy.rs @@ -0,0 +1,297 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; + +use windmill_common::variables::{build_crypt, encrypt}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +/// The deploy target is this very server, which the proxy has no way to tell from another +/// instance: it only ever knows the configured URL, the caller's stored token, and the path. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_remote_deploy_proxy(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/remote_deploy"); + + // Planted rather than set through the route, which is enterprise-gated. Unlike the route, it + // deletes no token for another target, which is what a connect landing just after the + // route's cleanup leaves behind. + let set_target = |base_url: String, workspace_id: &'static str| { + let db = db.clone(); + async move { + sqlx::query!( + "UPDATE workspace_settings + SET remote_deploy_target = $1, remote_deploy_target_changed_at = now() + WHERE workspace_id = 'test-workspace'", + json!({ "base_url": base_url, "workspace_id": workspace_id }) + ) + .execute(&db) + .await + .unwrap(); + } + }; + set_target(format!("http://localhost:{port}"), "test-workspace").await; + + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + let status = resp.json::().await?; + assert_eq!(status["target"]["workspace_id"], "test-workspace"); + assert!(status["connection"].is_null()); + + // Deploying before connecting names the step that is missing, rather than reaching the target + // with the caller's credentials for this instance. + let resp = authed(client().get(format!("{base}/proxy/none/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("Connect to")); + + let target = + json!({ "base_url": format!("http://localhost:{port}"), "workspace_id": "test-workspace" }); + + // A token obtained for another target is refused before it is sent anywhere. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": { "base_url": format!("http://localhost:{port}"), "workspace_id": "elsewhere" }})) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // A token the target refuses is not stored. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "not-a-token", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let connection = resp.json::().await?; + assert_eq!(connection["remote_email"], "test@windmill.dev"); + let key = connection["proxy_key"].as_str().unwrap().to_string(); + + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + // Whatever the remote returns is served from this origin, so it must never render as a page. + let csp = resp.headers()["content-security-policy"] + .to_str()? + .to_string(); + assert!(csp.starts_with("sandbox"), "{csp}"); + assert_eq!( + resp.json::().await?["email"], + "test@windmill.dev" + ); + + // A link from elsewhere rides the session cookie but cannot know the key, so it cannot spend + // the stored token. + // Nor can a credential that may not use the proxy read the key, to build such a link itself. + sqlx::query!( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, read_only) + VALUES (encode(sha256('READ_ONLY_TOKEN'::bytea), 'hex'), 'READ_ONLY_', 'READ_ONLY_TOKEN', + 'test@windmill.dev', 'read only', false, true)" + ) + .execute(&db) + .await?; + let resp = client() + .get(format!("{base}/target")) + .header("Authorization", "Bearer READ_ONLY_TOKEN") + .send() + .await?; + assert_eq!(resp.status(), 200); + assert!(resp.json::().await?["connection"].is_null()); + + let guessed = "x".repeat(key.len()); + let resp = authed(client().get(format!("{base}/proxy/{guessed}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // A connect locks nothing against a key rotation, so its row can land under the old key: that + // is no connection, which the drawer offers to replace, rather than an error on every deploy. + sqlx::query!( + "UPDATE remote_deploy_token SET token = 'not-under-this-key' + WHERE workspace_id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A target that refuses the stored token must not answer 401: the browser reads an + // unhandled 401 as its own session having expired and logs the user out of this instance. + let mc = build_crypt(&db, "test-workspace").await?; + sqlx::query!( + "UPDATE remote_deploy_token SET token = $1 WHERE workspace_id = 'test-workspace'", + encrypt(&mc, "revoked-token") + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 502); + + // Nor does a connect lock anything against a removal from the workspace, so its row can land + // after the removal cleared the table: a row from an earlier membership must not come back + // with a re-add, even one whose `created_at` reads earlier than the connect (it is the start + // of the re-adding transaction). + let as_test2 = + |builder: reqwest::RequestBuilder| builder.header("Authorization", "Bearer SECRET_TOKEN_2"); + let resp = as_test2(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN_2", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let key2 = resp.json::().await?["proxy_key"] + .as_str() + .unwrap() + .to_string(); + let resp = as_test2(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!( + resp.json::().await?["connection"]["proxy_key"], + key2.as_str() + ); + sqlx::query!( + "UPDATE usr SET created_at = created_at - interval '1 hour' + WHERE workspace_id = 'test-workspace' AND email = 'test2@windmill.dev'" + ) + .execute(&db) + .await?; + let resp = as_test2(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + let resp = as_test2(client().get(format!("{base}/proxy/{key2}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("Connect to")); + + // Deleting the account must take its token with it, or the next account created with that + // address would act on the remote as this one. + let resp = authed(client().delete(format!( + "http://localhost:{port}/api/users/delete/test2@windmill.dev" + ))) + .send() + .await?; + assert_eq!(resp.status(), 200); + let left = sqlx::query_scalar!( + "SELECT count(*) FROM remote_deploy_token WHERE email = 'test2@windmill.dev'" + ) + .fetch_one(&db) + .await?; + assert_eq!(left, Some(0)); + + // The token was granted for one target and is never sent to another, so re-pointing the + // workspace leaves the caller unconnected instead of handing its token to the new target. + set_target(format!("http://localhost:{port}"), "other-workspace").await; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + let resp = authed(client().get(format!("{base}/proxy/{key}/users/whoami"))) + .send() + .await?; + assert_eq!(resp.status(), 400); + + // The row for the old target outlived the change, as one from a connect in flight across it + // would: pointing the setting back must not revive it, even with a stamp that reads earlier + // than the connect. + set_target(format!("http://localhost:{port}"), "test-workspace").await; + sqlx::query!( + "UPDATE workspace_settings SET remote_deploy_target_changed_at = '2000-01-01' + WHERE workspace_id = 'test-workspace'" + ) + .execute(&db) + .await?; + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A connect in flight across a disconnect must not undo it when it lands. The disconnect is + // stamped an hour ahead, as if every connect starting now had started before it. + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + let resp = authed(client().post(format!("{base}/disconnect"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + sqlx::query!( + "UPDATE remote_deploy_token SET connected_at = clock_timestamp() + interval '1 hour' + WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'" + ) + .execute(&db) + .await?; + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + let resp = authed(client().get(format!("{base}/target"))) + .send() + .await?; + assert!(resp.json::().await?["connection"].is_null()); + + // A superadmin outside the workspace has no membership to bind to, so the row is bound to the + // account by the credential making the request: an account deleted during the remote call, + // and its address taken by another, must not inherit it. A deleted credential that auth still + // holds in its cache stands in for one whose account went away mid-call. + sqlx::query!( + "DELETE FROM usr WHERE workspace_id = 'test-workspace' AND email = 'test@windmill.dev'" + ) + .execute(&db) + .await?; + sqlx::query!("DELETE FROM remote_deploy_token WHERE email = 'test@windmill.dev'") + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin) + VALUES (encode(sha256('GONE_TOKEN'::bytea), 'hex'), 'GONE_TOKEN', 'GONE_TOKEN', + 'test@windmill.dev', 'gone', true)" + ) + .execute(&db) + .await?; + let as_gone = + |builder: reqwest::RequestBuilder| builder.header("Authorization", "Bearer GONE_TOKEN"); + let resp = as_gone(client().get(format!("{base}/target"))) + .send() + .await?; + assert_eq!(resp.status(), 200); + sqlx::query!("DELETE FROM token WHERE token_prefix = 'GONE_TOKEN'") + .execute(&db) + .await?; + let resp = as_gone(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 400); + let resp = authed(client().post(format!("{base}/connect"))) + .json(&json!({"token": "SECRET_TOKEN", "target": target})) + .send() + .await?; + assert_eq!(resp.status(), 200); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 33768f0e6b..a1b1054a25 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -2563,6 +2563,14 @@ pub async fn delete_workspace_user_internal( .execute(&mut **tx) .await?; + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + email_to_delete, + w_id + ) + .execute(&mut **tx) + .await?; + sqlx::query!( "DELETE FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", username_to_delete, @@ -4015,6 +4023,7 @@ async fn update_token_label( AND NOT starts_with(label, 'sdk_app:') AND NOT starts_with(label, 'impersonation:') AND NOT starts_with(label, 'cli-login:') + AND NOT starts_with(label, 'remote-deploy:') )) RETURNING token_prefix", req.label.as_deref(), @@ -4061,13 +4070,25 @@ async fn leave_workspace( &format!("u/{}", authed.username), ) .await?; - sqlx::query!( + let left = sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND username = $2", &w_id, authed.username ) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + // A superadmin deploys from workspaces it is no member of; leaving none is no reason to drop + // its connection. + if left > 0 { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + &authed.email, + &w_id + ) + .execute(&mut *tx) + .await?; + } audit_log( &mut *tx, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 52a58aa1a3..7ca1f692ad 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -35,6 +35,7 @@ windmill-store.workspace = true axum.workspace = true chrono.workspace = true +constant_time_eq.workspace = true futures = { workspace = true, optional = true } hex.workspace = true magic-crypt.workspace = true @@ -42,6 +43,7 @@ http.workspace = true hyper.workspace = true lazy_static.workspace = true regex.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true sha2.workspace = true @@ -49,5 +51,6 @@ sqlx.workspace = true tokio.workspace = true tokio-postgres.workspace = true tracing.workspace = true +url.workspace = true uuid.workspace = true strum.workspace = true diff --git a/backend/windmill-api-workspaces/src/lib.rs b/backend/windmill-api-workspaces/src/lib.rs index e7da2b8496..c437d7e2b4 100644 --- a/backend/windmill-api-workspaces/src/lib.rs +++ b/backend/windmill-api-workspaces/src/lib.rs @@ -9,6 +9,7 @@ pub mod datatable_permissions; pub mod datatable_permissions_oss; pub mod datatable_replay_oss; pub mod deployment_requests; +pub mod remote_deploy; pub mod workspaces; pub mod workspaces_extra; pub mod workspaces_oss; diff --git a/backend/windmill-api-workspaces/src/remote_deploy.rs b/backend/windmill-api-workspaces/src/remote_deploy.rs new file mode 100644 index 0000000000..3e701cbf14 --- /dev/null +++ b/backend/windmill-api-workspaces/src/remote_deploy.rs @@ -0,0 +1,794 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Deploying from the UI into a workspace of another Windmill instance. +//! +//! The browser runs the same deploy it runs between two workspaces of this instance, and sends +//! every call aimed at the target through [`proxy`], which forwards it to the remote workspace +//! with the caller's own token for that instance. The remote authorizes and audits the deploy as +//! that person, so nothing here needs to know what a deploy is made of. + +use std::time::Duration; + +use axum::{ + body::{Body, Bytes}, + extract::{DefaultBodyLimit, Extension, OriginalUri, Path}, + http::{header, HeaderMap, Method, StatusCode}, + response::Response, + routing::{any, get, post}, + Json, Router, +}; +use chrono::{DateTime, Utc}; +use lazy_static::lazy_static; +use magic_crypt::MagicCrypt256; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use windmill_api_auth::{is_effectively_unscoped, ApiAuthed, Tokened}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + auth::hash_token, + error::{error_source_chain, Error, Result}, + ssrf::validate_url_for_ssrf, + utils::{configure_client, rd_string, require_admin}, + variables::{build_crypt, decrypt, encrypt}, + worker::CLOUD_HOSTED, + DB, +}; + +lazy_static! { + static ref REMOTE_WORKSPACE_ID: Regex = Regex::new("^[a-zA-Z0-9_-]{1,50}$").unwrap(); + /// Shared so that a deploy's calls reuse connections. A cloud instance instead builds a client + /// per request, pinned to the addresses it just validated for that target. + static ref REMOTE_CLIENT: reqwest::Client = remote_client_builder().build().unwrap(); +} + +const PROXY_PREFIX: &str = "/remote_deploy/proxy/"; + +pub fn workspaced_service(proxy_body_limit: usize) -> Router { + Router::new() + .route("/target", get(get_target).post(set_target)) + .route("/connect", post(connect)) + .route("/disconnect", post(disconnect)) + // `{key}` is the caller's `proxy_key`. The session cookie is `SameSite=Lax`, so it rides a + // top-level navigation from any site: without a value such a link cannot know, it could + // spend the stored token, to run something on the remote or to render its content on this + // origin. `Sec-Fetch-Site` is no substitute, since plain-http instances never receive it. + .route( + "/proxy/{key}/{*rest}", + any(proxy).layer(DefaultBodyLimit::max(proxy_body_limit)), + ) +} + +/// A request URI as logs should record it: with the proxy key masked, since that key is what keeps +/// a link from spending a stored remote token. +pub struct RedactedUri<'a>(pub &'a axum::http::Uri); + +impl std::fmt::Display for RedactedUri<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Some((head, keyed)) = self.0.path().split_once(PROXY_PREFIX) else { + return self.0.fmt(f); + }; + let rest = keyed.find('/').map_or("", |i| &keyed[i..]); + write!(f, "{head}{PROXY_PREFIX}***{rest}")?; + match self.0.query() { + Some(query) => write!(f, "?{query}"), + None => Ok(()), + } + } +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct RemoteDeployTarget { + /// Root URL of the remote instance, without `/api`. + pub base_url: String, + pub workspace_id: String, +} + +impl RemoteDeployTarget { + fn normalized(self) -> Result { + let base_url = self.base_url.trim().trim_end_matches('/').to_string(); + let parsed = url::Url::parse(&base_url) + .map_err(|e| Error::BadRequest(format!("Invalid remote instance URL: {e}")))?; + if !matches!(parsed.scheme(), "http" | "https") || parsed.host_str().is_none() { + return Err(Error::BadRequest( + "The remote instance URL must be an http(s) URL with a host".to_string(), + )); + } + if !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.query().is_some() + || parsed.fragment().is_some() + { + return Err(Error::BadRequest( + "The remote instance URL must not carry credentials, a query or a fragment" + .to_string(), + )); + } + let workspace_id = self.workspace_id.trim().to_string(); + if !REMOTE_WORKSPACE_ID.is_match(&workspace_id) { + return Err(Error::BadRequest(format!( + "Invalid remote workspace id: {workspace_id}" + ))); + } + Ok(Self { base_url, workspace_id }) + } + + fn api_url(&self, suffix: &str) -> String { + format!("{}/api/w/{}/{suffix}", self.base_url, self.workspace_id) + } +} + +#[derive(Serialize)] +struct RemoteDeployConnection { + remote_email: String, + /// Goes in every proxy URL, see [`workspaced_service`]. + proxy_key: String, + connected_at: DateTime, +} + +#[derive(Serialize)] +struct RemoteDeployStatus { + target: Option, + /// The caller's own connection to `target`. + connection: Option, +} + +/// A stored token acts on another instance as its owner, with none of the restrictions of the +/// credential that reaches it. Only the owner acting directly with full rights may use or replace +/// it: a job token runs as whoever a `wm_deployers` member pointed `on_behalf_of` at, and a +/// scoped or read-only token was never granted this. +fn require_own_credentials(authed: &ApiAuthed) -> Result<()> { + if authed.job_id.is_some() + || authed.read_only + || !is_effectively_unscoped(authed.scopes.as_deref()) + { + return Err(Error::PermissionDenied( + "Deploying to a remote instance requires your own session or an unscoped token, \ + not a job, scoped or read-only token" + .to_string(), + )); + } + Ok(()) +} + +fn parse_target(target: Option) -> Result> { + target + .map(serde_json::from_value) + .transpose() + .map_err(|e| Error::internal_err(format!("reading the remote deploy target: {e}"))) +} + +fn no_target(w_id: &str) -> Error { + Error::BadRequest(format!( + "Workspace {w_id} has no remote deploy target. An admin sets it in the workspace \ + settings, under Dev workspace" + )) +} + +async fn load_target(db: &DB, w_id: &str) -> Result> { + let target = sqlx::query_scalar!( + "SELECT remote_deploy_target FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten(); + parse_target(target) +} + +async fn require_target(db: &DB, w_id: &str) -> Result { + load_target(db, w_id).await?.ok_or_else(|| no_target(w_id)) +} + +fn remote_client_builder() -> reqwest::ClientBuilder { + configure_client(reqwest::ClientBuilder::new()) + .user_agent("windmill/remote-deploy") + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(10)) + .timeout(Duration::from_secs(120)) +} + +async fn remote_client(target: &RemoteDeployTarget) -> Result { + // A workspace admin chose the URL. A self-hosted instance may deploy into its own private + // network, as it may reach a private git remote; a cloud workspace must not. + if !*CLOUD_HOSTED { + return Ok(REMOTE_CLIENT.clone()); + } + validate_url_for_ssrf(&target.base_url) + .await? + .apply_dns_pinning(remote_client_builder()) + .build() + .map_err(|e| Error::internal_err(format!("building the remote deploy client: {e}"))) +} + +fn unreachable(target: &RemoteDeployTarget, e: reqwest::Error) -> Error { + Error::BadGateway(format!( + "Could not reach the remote instance {}: {}", + target.base_url, + error_source_chain(&e) + )) +} + +/// Responses carrying a `proxy_key` must not be kept by any cache between here and the browser. +const NO_STORE: [(header::HeaderName, &str); 1] = [(header::CACHE_CONTROL, "no-store")]; + +struct StoredConnection { + token: String, + remote_email: String, + proxy_key: String, + connected_at: DateTime, +} + +/// The caller's connection to `target`, if they may still use it. +/// +/// `connect` takes no lock against what clears these rows (a removal from the workspace, a target +/// change, a key rotation): writers take those rows in every order, so any lock it held could close +/// a deadlock. A row can therefore land after one of them ran, however long its remote call took, +/// and is voided here instead, by identity rather than by comparing times: it counts only while +/// the target setting and the owner's membership are the very ones it was connected under (see +/// `member_since`, `target_changed_at`) or the owner is a superadmin, and while it decrypts. An +/// emptied row is a disconnect's. +async fn load_connection( + db: &DB, + w_id: &str, + email: &str, + target: &RemoteDeployTarget, +) -> Result> { + let Some(row) = sqlx::query_as!( + StoredConnection, + "SELECT t.token, t.remote_email, t.proxy_key, t.connected_at FROM remote_deploy_token t + JOIN workspace_settings s ON s.workspace_id = t.workspace_id + WHERE t.workspace_id = $1 AND t.email = $2 AND t.base_url = $3 + AND t.remote_workspace_id = $4 AND t.token <> '' + AND s.remote_deploy_target_changed_at IS NOT DISTINCT FROM t.target_changed_at + AND (EXISTS (SELECT 1 FROM usr u WHERE u.workspace_id = t.workspace_id + AND u.email = t.email AND u.created_at = t.member_since) + OR EXISTS (SELECT 1 FROM password p WHERE p.email = t.email AND p.super_admin))", + w_id, + email, + &target.base_url, + &target.workspace_id + ) + .fetch_optional(db) + .await? + else { + return Ok(None); + }; + match decrypt(&build_crypt(db, w_id).await?, row.token) { + Ok(token) => Ok(Some(StoredConnection { token, ..row })), + Err(e) => { + tracing::warn!("remote deploy token of {email} in {w_id} does not decrypt: {e}"); + Ok(None) + } + } +} + +async fn get_target( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result<( + [(header::HeaderName, &'static str); 1], + Json, +)> { + let target = load_target(&db, &w_id).await?; + // The connection carries the proxy key. A credential that may not use the proxy (see + // `require_own_credentials`) must not read it either: it could hand its owner a working link. + let connection = match &target { + Some(target) if require_own_credentials(&authed).is_ok() => { + load_connection(&db, &w_id, &authed.email, target) + .await? + .map(|c| RemoteDeployConnection { + remote_email: c.remote_email, + proxy_key: c.proxy_key, + connected_at: c.connected_at, + }) + } + _ => None, + }; + Ok((NO_STORE, Json(RemoteDeployStatus { target, connection }))) +} + +#[derive(Deserialize)] +struct SetRemoteDeployTarget { + target: Option, +} + +async fn set_target( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> Result { + if !cfg!(feature = "enterprise") { + return Err(Error::BadRequest( + "Deploying to another instance is only available on Windmill Enterprise Edition" + .to_string(), + )); + } + // Everyone who connects afterwards hands their token to this URL. + require_own_credentials(&authed)?; + require_admin(authed.is_admin, &authed.username)?; + let target = request + .target + .map(RemoteDeployTarget::normalized) + .transpose()?; + + let mut tx = db.begin().await?; + let (base_url, remote_workspace_id) = match &target { + Some(t) => (Some(t.base_url.as_str()), Some(t.workspace_id.as_str())), + None => (None, None), + }; + audit_log( + &mut *tx, + &authed, + "workspaces.edit_remote_deploy_target", + ActionKind::Update, + &w_id, + None, + Some( + [ + ("base_url", base_url.unwrap_or("")), + ("remote_workspace_id", remote_workspace_id.unwrap_or("")), + ] + .into(), + ), + ) + .await?; + let target_json = target + .as_ref() + .map(serde_json::to_value) + .transpose() + .map_err(|e| Error::internal_err(e.to_string()))?; + sqlx::query!( + "UPDATE workspace_settings SET remote_deploy_target = $1::jsonb, + remote_deploy_target_changed_at = CASE + WHEN remote_deploy_target IS DISTINCT FROM $1::jsonb THEN clock_timestamp() + ELSE remote_deploy_target_changed_at END + WHERE workspace_id = $2", + target_json, + &w_id + ) + .execute(&mut *tx) + .await?; + // A token is only ever sent to the target it was granted for, so the ones for any other + // target are dead credentials. + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE workspace_id = $1 + AND ($2::text IS NULL OR base_url <> $2 OR remote_workspace_id <> $3)", + &w_id, + base_url, + remote_workspace_id + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + + Ok(match target { + Some(t) => format!( + "Workspace {w_id} deploys to {} on {}", + t.workspace_id, t.base_url + ), + None => format!("Removed the remote deploy target of workspace {w_id}"), + }) +} + +#[derive(Deserialize)] +struct ConnectRequest { + token: String, + /// The target the caller got the token for, as the drawer showed it. + target: RemoteDeployTarget, +} + +#[derive(Deserialize)] +struct RemoteWhoami { + email: String, +} + +async fn connect( + authed: ApiAuthed, + Tokened { token: credential }: Tokened, + Extension(db): Extension, + Path(w_id): Path, + Json(request): Json, +) -> Result<( + [(header::HeaderName, &'static str); 1], + Json, +)> { + require_own_credentials(&authed)?; + // Read before the remote call, which can take long enough for any of it to change: the row is + // bound to the target version and the membership seen here (see `load_connection`), and + // stamped with when this connect started, so that a disconnect or a newer connect made while + // it runs keeps it from landing. + let start = sqlx::query!( + r#"SELECT clock_timestamp() AS "started_at!", s.remote_deploy_target, + s.remote_deploy_target_changed_at, + (SELECT u.created_at FROM usr u + WHERE u.workspace_id = s.workspace_id AND u.email = $2) AS member_since + FROM workspace_settings s WHERE s.workspace_id = $1"#, + &w_id, + &authed.email + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| no_target(&w_id))?; + let target = parse_target(start.remote_deploy_target)?.ok_or_else(|| no_target(&w_id))?; + // A token is only ever sent to the instance it was meant for. Without this, re-pointing the + // target between the user getting a token and posting it here would hand it to the new one; + // after this check the token still goes to `target` only, never to what the setting became. + let requested = request.target.normalized()?; + if requested.base_url != target.base_url || requested.workspace_id != target.workspace_id { + return Err(Error::BadRequest( + "The remote deploy target changed since you started connecting; reload and connect \ + again" + .to_string(), + )); + } + let token = request.token.trim(); + if token.is_empty() { + return Err(Error::BadRequest("The token is empty".to_string())); + } + + let response = remote_client(&target) + .await? + .get(target.api_url("users/whoami")) + .bearer_auth(token) + .send() + .await + .map_err(|e| unreachable(&target, e))?; + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(Error::BadRequest(format!( + "{} did not accept this token for workspace {} ({status}): {body}", + target.base_url, target.workspace_id + ))); + } + let whoami: RemoteWhoami = response.json().await.map_err(|e| { + Error::BadGateway(format!( + "{} did not answer like a Windmill instance: {}", + target.base_url, + error_source_chain(&e) + )) + })?; + + let proxy_key = rd_string(32); + let mc = build_crypt(&db, &w_id).await?; + // No lock against removals, target changes or key rotations: `load_connection` voids a row + // that lands after one of them. A disconnect or a newer connect stamps the row itself, which + // this one then leaves alone. Without a membership to bind to, the row is bound to the account + // through the credential making this request, which deleting the account removes (except + // through `leave_instance`, which leaves its tokens): the address could otherwise be deleted + // and taken by a new account while the remote call runs, and the foreign key would accept it. + let mut tx = db.begin().await?; + let connected_at = sqlx::query_scalar!( + "INSERT INTO remote_deploy_token + (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key, + connected_at, member_since, target_changed_at) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 + WHERE $9::timestamptz IS NOT NULL + OR EXISTS (SELECT 1 FROM token WHERE token_hash = $11) + ON CONFLICT (workspace_id, email) DO UPDATE SET + base_url = EXCLUDED.base_url, remote_workspace_id = EXCLUDED.remote_workspace_id, + token = EXCLUDED.token, remote_email = EXCLUDED.remote_email, + proxy_key = EXCLUDED.proxy_key, connected_at = EXCLUDED.connected_at, + member_since = EXCLUDED.member_since, target_changed_at = EXCLUDED.target_changed_at + WHERE remote_deploy_token.connected_at < EXCLUDED.connected_at + RETURNING connected_at", + &w_id, + &authed.email, + &target.base_url, + &target.workspace_id, + encrypt(&mc, token), + &whoami.email, + &proxy_key, + start.started_at, + start.member_since, + start.remote_deploy_target_changed_at, + hash_token(&credential) + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + Error::BadRequest(match start.member_since { + Some(_) => "This connect was not saved: you disconnected or connected again while it \ + was running" + .to_string(), + None => format!( + "This connect was not saved: you disconnected, connected again or signed out \ + while it was running. Connecting workspace {w_id} without being one of its \ + members needs a session or an API token of this instance" + ), + }) + })?; + audit_log( + &mut *tx, + &authed, + "workspaces.remote_deploy_connect", + ActionKind::Create, + &w_id, + Some(&whoami.email), + Some([("base_url", target.base_url.as_str())].into()), + ) + .await?; + tx.commit().await?; + + Ok(( + NO_STORE, + Json(RemoteDeployConnection { remote_email: whoami.email, proxy_key, connected_at }), + )) +} + +async fn disconnect( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result { + require_own_credentials(&authed)?; + let mut tx = db.begin().await?; + // Emptied rather than deleted, and stamped, even with no connection yet: a connect that started + // before this must not bring one back when it lands (see `connect`). + sqlx::query!( + "INSERT INTO remote_deploy_token + (workspace_id, email, base_url, remote_workspace_id, token, remote_email, proxy_key, + connected_at) + VALUES ($1, $2, '', '', '', '', '', clock_timestamp()) + ON CONFLICT (workspace_id, email) DO UPDATE SET + token = '', proxy_key = '', connected_at = clock_timestamp()", + &w_id, + &authed.email + ) + .execute(&mut *tx) + .await?; + audit_log( + &mut *tx, + &authed, + "workspaces.remote_deploy_disconnect", + ActionKind::Delete, + &w_id, + None, + None, + ) + .await?; + tx.commit().await?; + Ok(format!( + "Disconnected from the remote deploy target of {w_id}" + )) +} + +/// The remote URL for the request path after the proxy prefix and key, as the client sent it. +/// +/// `url` rewrites a path while parsing it: it resolves dot segments (`%2e` spellings included) +/// and turns backslashes into slashes. Either would reach a route other than the one this +/// instance authorized — outside the remote workspace, or a route its read-only check does not +/// recognize — so a path the parser would change is refused rather than forwarded. +fn forwarded_url( + target: &RemoteDeployTarget, + original_path: &str, + query: Option<&str>, +) -> Result { + let suffix = original_path + .split_once(PROXY_PREFIX) + .and_then(|(_, keyed)| keyed.split_once('/')) + .map(|(_, suffix)| suffix) + .unwrap_or_default(); + let invalid = || Error::BadRequest(format!("Invalid remote deploy path: {suffix}")); + let base_path = url::Url::parse(&target.base_url) + .map_err(|_| invalid())? + .path() + .trim_end_matches('/') + .to_string(); + let mut url = url::Url::parse(&target.api_url(suffix)).map_err(|_| invalid())?; + let expected_path = format!("{base_path}/api/w/{}/{suffix}", target.workspace_id); + if suffix.is_empty() || url.path() != expected_path { + return Err(invalid()); + } + url.set_query(query); + Ok(url) +} + +async fn proxy( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, key, _rest)): Path<(String, String, String)>, + OriginalUri(uri): OriginalUri, + method: Method, + headers: HeaderMap, + body: Bytes, +) -> Result { + require_own_credentials(&authed)?; + let target = require_target(&db, &w_id).await?; + let url = forwarded_url(&target, uri.path(), uri.query())?; + let stored = load_connection(&db, &w_id, &authed.email, &target) + .await? + .ok_or_else(|| { + Error::BadRequest(format!( + "Connect to {} with your own token before deploying there", + target.base_url + )) + })?; + if !constant_time_eq::constant_time_eq(key.as_bytes(), stored.proxy_key.as_bytes()) { + return Err(Error::BadRequest( + "This remote deploy link is not yours or is out of date; reload the page".to_string(), + )); + } + + // Only what describes the payload: the caller's cookie and token belong to this instance. + let mut request = remote_client(&target) + .await? + .request(method, url) + .bearer_auth(stored.token) + .body(body); + for name in [header::CONTENT_TYPE, header::ACCEPT] { + if let Some(value) = headers.get(&name) { + request = request.header(name, value.clone()); + } + } + let response = request.send().await.map_err(|e| unreachable(&target, e))?; + + let status = response.status(); + // Passed through, a 401 would read as this instance's session having expired and log the + // user out of it. + if status == StatusCode::UNAUTHORIZED { + let body = response.text().await.unwrap_or_default(); + return Err(Error::BadGateway(format!( + "{} rejected your stored token: {body}", + target.base_url + ))); + } + if status.is_redirection() { + let location = response + .headers() + .get(header::LOCATION) + .and_then(|l| l.to_str().ok()) + .unwrap_or_default() + .to_string(); + return Err(Error::BadGateway(format!( + "{} redirected to {location}. Set the remote deploy target to the URL it redirects to", + target.base_url + ))); + } + // The body is the remote's and may be anything its users stored (an uploaded file, a job + // result typed `text/html`); served from this origin it must never render as a page here. + let mut builder = Response::builder() + .status(status) + .header(header::X_CONTENT_TYPE_OPTIONS, "nosniff") + .header( + header::CONTENT_SECURITY_POLICY, + "sandbox; default-src 'none'", + ); + if let Some(content_type) = response.headers().get(header::CONTENT_TYPE) { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + builder + .body(Body::from_stream(response.bytes_stream())) + .map_err(|e| Error::internal_err(format!("building the proxied response: {e}"))) +} + +/// Move the stored remote tokens to a new workspace key. A token that no longer decrypts is +/// dropped: its owner connects again. A disconnect's emptied row is kept, as the stamp that keeps +/// an older connect from landing. +pub(crate) async fn reencrypt_tokens( + conn: &mut sqlx::PgConnection, + w_id: &str, + old: &MagicCrypt256, + new: &MagicCrypt256, +) -> Result<()> { + let rows = sqlx::query!( + "SELECT email, token FROM remote_deploy_token + WHERE workspace_id = $1 AND token <> '' FOR UPDATE", + w_id + ) + .fetch_all(&mut *conn) + .await?; + for row in rows { + match decrypt(old, row.token) { + Ok(plain) => { + sqlx::query!( + "UPDATE remote_deploy_token SET token = $1 + WHERE workspace_id = $2 AND email = $3", + encrypt(new, &plain), + w_id, + row.email + ) + .execute(&mut *conn) + .await?; + } + Err(_) => { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE workspace_id = $1 AND email = $2", + w_id, + row.email + ) + .execute(&mut *conn) + .await?; + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn forwarded_url_refuses_paths_the_parser_rewrites() { + let target = RemoteDeployTarget { + base_url: "https://prod.example.com/windmill".to_string(), + workspace_id: "prod".to_string(), + }; + let prefix = "/api/w/dev/remote_deploy/proxy/someKey"; + let url = forwarded_url( + &target, + &format!("{prefix}/scripts/get/p/f/team/my%20script"), + Some("with_starred_info=true"), + ) + .unwrap(); + assert_eq!( + url.as_str(), + "https://prod.example.com/windmill/api/w/prod/scripts/get/p/f/team/my%20script?with_starred_info=true" + ); + for path in [ + format!("{prefix}/../../users/list"), + format!("{prefix}/%2e%2e/%2e%2e/users/list"), + format!("{prefix}/jobs\\run_wait_result\\p/f/team/action"), + prefix.to_string(), + ] { + assert!( + forwarded_url(&target, &path, None).is_err(), + "{path} should be refused" + ); + } + } + + #[test] + fn redacted_uri_masks_the_proxy_key() { + let proxied: axum::http::Uri = "/api/w/dev/remote_deploy/proxy/secretKey/flows/get/f/a?x=1" + .parse() + .unwrap(); + assert_eq!( + RedactedUri(&proxied).to_string(), + "/api/w/dev/remote_deploy/proxy/***/flows/get/f/a?x=1" + ); + let other: axum::http::Uri = "/api/w/dev/flows/list?x=1".parse().unwrap(); + assert_eq!(RedactedUri(&other).to_string(), "/api/w/dev/flows/list?x=1"); + } + + #[test] + fn target_normalization() { + let target = RemoteDeployTarget { + base_url: " https://prod.example.com/ ".to_string(), + workspace_id: " prod ".to_string(), + } + .normalized() + .unwrap(); + assert_eq!(target.base_url, "https://prod.example.com"); + assert_eq!( + target.api_url("flows/create"), + "https://prod.example.com/api/w/prod/flows/create" + ); + + for (base_url, workspace_id) in [ + ("ftp://prod.example.com", "prod"), + ("https://user:pass@prod.example.com", "prod"), + ("https://prod.example.com?token=x", "prod"), + ("https://prod.example.com", "prod/../admins"), + ] { + assert!( + RemoteDeployTarget { + base_url: base_url.to_string(), + workspace_id: workspace_id.to_string(), + } + .normalized() + .is_err(), + "{base_url} {workspace_id} should be refused" + ); + } + } +} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 4822c2f3b9..029f5d3945 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -5960,6 +5960,13 @@ async fn set_encryption_key( &new_encryption_key, ) .await?; + crate::remote_deploy::reencrypt_tokens( + &mut tx, + &w_id, + &previous_encryption_key, + &new_encryption_key, + ) + .await?; tx.commit().await?; @@ -10143,13 +10150,25 @@ async fn leave_workspace( &format!("u/{}", authed.username), ) .await?; - sqlx::query!( + let left = sqlx::query!( "DELETE FROM usr WHERE workspace_id = $1 AND email = $2", &w_id, &authed.email ) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + // A superadmin deploys from workspaces it is no member of; leaving none is no reason to drop + // its connection. + if left > 0 { + sqlx::query!( + "DELETE FROM remote_deploy_token WHERE email = $1 AND workspace_id = $2", + &authed.email, + &w_id + ) + .execute(&mut *tx) + .await?; + } audit_log( &mut *tx, diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 82423480fa..9039bda2ef 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -113,7 +113,7 @@ pub(crate) async fn change_workspace_id( // Duplicate workspace settings (keep copy in old workspace for reference) info!("Duplicating workspace_settings table"); sqlx::query!( - "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks FROM workspace_settings WHERE workspace_id = $2", + "INSERT INTO workspace_settings (workspace_id, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at) SELECT $1, slack_team_id, slack_name, slack_command_script, slack_email, customer_id, plan, webhook, ai_config, large_file_storage, git_sync, default_app, default_scripts, deploy_ui, mute_critical_alerts, color, operator_settings, teams_command_script, teams_team_id, teams_team_name, git_app_installations, git_credentials, ducklake, dbt_warehouses, slack_oauth_client_id, slack_oauth_client_secret, datatable, teams_team_guid, auto_invite, error_handler, success_handler, public_app_execution_limit_per_minute, error_handler_fallback_to_instance_alerts, guest_access_enabled, guest_jwt_public_key, guest_jwt_jwks_url, add_admins_and_developers_to_forks, remote_deploy_target, remote_deploy_target_changed_at FROM workspace_settings WHERE workspace_id = $2", &rw.new_id, &old_id ) @@ -841,6 +841,14 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + sqlx::query!( + "UPDATE remote_deploy_token SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + info!("Updating variable table"); sqlx::query!( "UPDATE variable SET workspace_id = $1 WHERE workspace_id = $2", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a114f74267..46c498de4c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4187,6 +4187,100 @@ paths: deploy_to: type: string + /w/{workspace}/remote_deploy/target: + get: + summary: get the workspace's deploy target on another instance, and the caller's connection to it + description: | + Once connected, deploy calls aimed at the target go through + `/w/{workspace}/remote_deploy/proxy/{proxy_key}/{route}`, which forwards any method to + `{base_url}/api/w/{target workspace}/{route}` with the caller's stored token. + operationId: getRemoteDeployTarget + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: remote deploy target and the caller's connection to it + content: + application/json: + schema: + $ref: "#/components/schemas/RemoteDeployStatus" + post: + summary: set or clear the workspace's deploy target on another instance + description: Changing the target drops every token stored for the previous one. + operationId: setRemoteDeployTarget + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + target: + $ref: "#/components/schemas/RemoteDeployTarget" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/remote_deploy/connect: + post: + summary: store the caller's own token for the remote deploy target + description: | + The token is checked against the target's whoami before it is stored. `target` names the + target the token was obtained for, and the request is refused if the workspace now points + elsewhere, so a token is never sent to an instance it was not meant for. + operationId: connectRemoteDeploy + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + token: + type: string + target: + $ref: "#/components/schemas/RemoteDeployTarget" + required: + - token + - target + responses: + "200": + description: the stored connection + content: + application/json: + schema: + $ref: "#/components/schemas/RemoteDeployConnection" + + /w/{workspace}/remote_deploy/disconnect: + post: + summary: forget the caller's token for the remote deploy target + operationId: disconnectRemoteDeploy + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/is_premium: get: summary: get if workspace is premium @@ -36532,6 +36626,44 @@ components: items: $ref: "#/components/schemas/GitSyncObjectType" + RemoteDeployTarget: + type: object + properties: + base_url: + type: string + description: root URL of the remote Windmill instance, without /api + workspace_id: + type: string + description: workspace on the remote instance that deploys land in + required: + - base_url + - workspace_id + + RemoteDeployConnection: + type: object + properties: + remote_email: + type: string + description: identity the stored token has on the remote instance + proxy_key: + type: string + description: goes in every proxy URL, `/w/{workspace}/remote_deploy/proxy/{proxy_key}/{route}` + connected_at: + type: string + format: date-time + required: + - remote_email + - proxy_key + - connected_at + + RemoteDeployStatus: + type: object + properties: + target: + $ref: "#/components/schemas/RemoteDeployTarget" + connection: + $ref: "#/components/schemas/RemoteDeployConnection" + WorkspaceDefaultScripts: type: object properties: diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 0fd5ac31b5..d6b29a8501 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -731,6 +731,12 @@ pub async fn run_server( path_autocomplete::workspaced_service(), ) .nest("/raw_apps", raw_apps::workspaced_service()) + .nest( + "/remote_deploy", + windmill_api_workspaces::remote_deploy::workspaced_service( + request_size_limit * 5, + ), + ) // CORS so the opaque-origin app iframe can read // resources/list, resources/type/* with a scoped token. .nest( diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 56adeee184..d99a7e5907 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -14,6 +14,7 @@ use axum::response::Response as AxumResponse; use hyper::Response; use tower_http::trace::{MakeSpan, OnFailure, OnResponse}; use uuid::Uuid; +use windmill_api_workspaces::remote_deploy::RedactedUri; use windmill_common::log_context::{with_log_context, LogContext}; lazy_static::lazy_static! { @@ -86,7 +87,7 @@ impl MakeSpan for MyMakeSpan { tracing::error_span!( "request", method = %request.method(), - uri = %request.uri(), + uri = %RedactedUri(request.uri()), username = field::Empty, workspace_id = field::Empty, traceId = tracing_id, @@ -113,7 +114,7 @@ pub async fn log_context_middleware(request: Request, next: Next) -> AxumRespons let ctx = LogContext { method: Some(request.method().to_string()), - uri: Some(request.uri().to_string()), + uri: Some(RedactedUri(request.uri()).to_string()), trace_id, ..Default::default() }; diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index c12d62f206..86b7f173ac 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -20,7 +20,8 @@ use crate::{ /// Whether `label` denotes a user-created token rather than a system token /// (`session`, `guest_session`, `ephemeral*`, `debugger-token`, `mcp-oauth-*`, -/// `embed_app:*`, `sdk_app:*`, `impersonation:*`, `cli-login:*`). System-token labels are load-bearing — +/// `embed_app:*`, `sdk_app:*`, `impersonation:*`, `cli-login:*`, `remote-deploy:*`). +/// System-token labels are load-bearing — /// session cleanup, super_admin propagation, expiry notifications and username overrides /// all key off them — so they must not be user-editable. `None` (no label) is treated as /// a user token. @@ -48,6 +49,7 @@ pub fn is_user_token(label: Option<&str>) -> bool { && !l.starts_with(RAW_APP_SDK_TOKEN_LABEL_PREFIX) && !l.starts_with("impersonation:") && !l.starts_with(CLI_LOGIN_TOKEN_LABEL_PREFIX) + && !l.starts_with(REMOTE_DEPLOY_TOKEN_LABEL_PREFIX) } } } @@ -73,6 +75,14 @@ pub const RAW_APP_SDK_TOKEN_LABEL_PREFIX: &str = "sdk_app:"; /// `/users/tokens/create`. pub const CLI_LOGIN_TOKEN_LABEL_PREFIX: &str = "cli-login:"; +/// Label prefix, followed by the host of the instance deploying with it, of the token the +/// `/user/remote_deploy_authorize` page mints for another Windmill instance. Reserved in +/// [`is_user_token`], whose SQL and frontend mirrors spell it out: it is renewed by connecting +/// again from the deploying instance, which reports this one rejecting it, so an expiry email or +/// alert here points at the wrong place. Not in [`is_server_minted_label`], since the page mints it +/// through `/users/tokens/create`. +pub const REMOTE_DEPLOY_TOKEN_LABEL_PREFIX: &str = "remote-deploy:"; + /// Whether `label` belongs to a namespace only the server mints, and which therefore must be /// rejected by `create_token`. Narrower than [`is_user_token`], which also drives label /// editability and expiry notifications and can afford to reserve more: `Ephemeral lsp token`, @@ -997,6 +1007,7 @@ mod tests { assert!(!is_user_token(Some("sdk_app:u/admin/raw app"))); assert!(!is_user_token(Some("impersonation:admin@windmill.dev"))); assert!(!is_user_token(Some("cli-login:admin"))); + assert!(!is_user_token(Some("remote-deploy:windmill.example.com"))); } #[test] diff --git a/docs/auth-surface.md b/docs/auth-surface.md index 21d42dd39b..a383b9267d 100644 --- a/docs/auth-surface.md +++ b/docs/auth-surface.md @@ -46,6 +46,52 @@ Symbols, not line numbers, are cited: they drift less. (`windmill-api-auth/src/lib.rs`) errors on `authed.job_id.is_some()`. A script that needs `users/create`, `tokens/impersonate`, `set_login_type`, … must use a dedicated superadmin user token stored as a secret, never `$WM_TOKEN`. Token scopes cannot narrow superadmin routes. +- **A remote deploy token is a credential for another instance**, held per account and workspace + (`remote_deploy_token`, encrypted under the workspace key, re-keyed by `set_encryption_key`). + Its `email` references `password(email)` with `ON DELETE/UPDATE CASCADE`, so whatever deletes or + renames an account takes the token along and a recycled address cannot inherit it; removal from + a workspace deletes it explicitly (`delete_workspace_user_internal`, both `leave_workspace`). The + row records the target it was granted for, and + `remote_deploy::proxy` only sends it to a target still matching the workspace setting, so + re-pointing the setting cannot redirect anyone's token to a URL of the admin's choosing. + `require_own_credentials` refuses job, scoped and read-only tokens (on `set_target` too): the + stored token carries none of their restrictions. The proxy turns the local session into a remote + bearer credential, so every ambient-cookie vector becomes one on the remote: its URL carries the + row's random `proxy_key` (a link riding the `SameSite=Lax` cookie cannot know it; a header would + do, but the frontend's only per-call hook is the global `OpenAPI.HEADERS`, whose mere presence + switches every download to in-memory blobs). The key is served `no-store`, withheld from the + credentials `require_own_credentials` refuses, and masked in this instance's own request logs + (`RedactedUri`, used by the request span and the log context); a reverse proxy in front still + writes the full path to its access log. `connect` names the target the token was obtained for + and is refused, before the token is sent anywhere, if the workspace now points elsewhere; the + token only ever goes to that target. It takes no lock against what clears these rows (a removal + from the workspace, a target change, a key rotation): their writers take the account, membership, + key and settings rows in every order, so any lock held there could close a deadlock. A row can + therefore land after one of them ran, however long its remote call took, and every read + (`load_connection`) voids it instead, by identity rather than by comparing times: the connect + records, before its remote call, the target setting's version (`remote_deploy_target_changed_at`) + and the `created_at` of the owner's membership, and the row counts only while both are still + the same (or the owner is a superadmin) and it decrypts. So a target set A → B → A, or a + removal and re-add, voids it whatever the clocks say. A superadmin with no membership is bound + through the credential making the request instead: the insert requires its `token` row to + still exist, and `delete_user`, offboarding and SCIM removal delete an account's tokens, so an + address deleted that way and re-created during the call does not inherit the connection (a JWT, + having no row, cannot connect a workspace its owner is not a member of). `leave_instance` + deletes only the `password` row, leaving the tokens and the memberships, which is not covered. + Neither is a member's connect across an account re-creation: an address is not expected to pass + to another person. A disconnect empties and stamps the row + (inserting one if needed) rather than deleting it, `connected_at` is when a connect started, + and the connect's upsert leaves a row stamped after that alone, so an older connect cannot undo + a disconnect or a newer connect. Connecting by redirect: the remote's + `/user/remote_deploy_authorize` page + mints a token bound to the one remote workspace (`remote-deploy:`, a label + reserved in `is_user_token` so its expiry emails nobody) only on an + explicit Authorize, only for a callback whose path is `/remote_deploy/callback`, and refuses to + render inside a frame; the token travels in the fragment, and the callback checks a single-use + `state` the drawer stored, so no other page can plant a token as the user's. The proxy also refuses a path the URL parser would rewrite, + serves every response under `CSP: sandbox` + `nosniff`, forwards only the method, query, body, + content-type and accept — never this instance's cookie or token — and turns the target's 401 + into a 502, because the browser logs the user out of *this* instance on an unhandled 401. - **`login_type`** (`password` table) is a free-form `VARCHAR(50)`. Password login and password reset require `login_type = 'password'`; `set_password` also accepts `pending_oauth` and turns the account into a `password` one in the same statement (an account created ahead of its owner diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index 1f938dcfdf..a01e845246 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -13,8 +13,14 @@ ResourceService, ScheduleService, UserService, - WorkspaceService + WorkspaceService, + type RemoteDeployStatus } from '$lib/gen' + import { remoteDeployLabel, remoteDeployWorkspace } from '$lib/remoteDeploy' + import RemoteDeployConnect from './RemoteDeployConnect.svelte' + import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import TextInput from './text_input/TextInput.svelte' import { getAllModules } from './flows/flowExplorer' import Button from './common/button/Button.svelte' import Tooltip from './Tooltip.svelte' @@ -67,11 +73,22 @@ let canSeeTarget: 'yes' | 'cant-deploy-to-workspace' | 'cant-see-all-deps' | undefined = $state(undefined) + /** What the target answered when it refused, for a destination whose credential can go stale. */ + let targetError: string | undefined = $state(undefined) + + // The two destinations a workspace can have: its parent on this instance, and a workspace on + // another instance. Both are set by an admin, so a workspace may have either, both or neither. + type Destination = 'parent' | 'remote' + let parentWorkspace: string | undefined = $state(undefined) + let remoteStatus = $state(undefined) + let destination: Destination | undefined = $state(undefined) + let remoteTarget = $derived(remoteStatus?.target) + let isRemote = $derived(destination === 'remote') type Dependency = { kind: Kind; path: string; include: boolean } let dependencies: Dependency[] | undefined = $state(undefined) - const allAlreadyExists: { [key: string]: boolean } = $state({}) + let allAlreadyExists: { [key: string]: boolean } = $state({}) let diffDrawer: DiffDrawer | undefined = $state(undefined) let notSet: boolean | undefined = $state(undefined) @@ -120,17 +137,37 @@ } async function reload(path: string) { + const target = workspaceToDeployTo + // Everything below is about this one target; switching destination starts its own pass, and + // the one it replaced must not write its answers into the new one's state. + const stillCurrent = () => target === workspaceToDeployTo + canSeeTarget = undefined + targetError = undefined + dependencies = undefined + allAlreadyExists = {} + deploymentStatus = {} + targetOnBehalfOfInfo = {} + onBehalfOfChoice = {} + customOnBehalfOf = {} try { - if (!$superadmin) { - const targetUser = await UserService.whoami({ workspace: workspaceToDeployTo! }) - canPreserveOnBehalfOf = - targetUser.is_admin || targetUser.groups?.includes('wm_deployers') || false - } else { - canPreserveOnBehalfOf = true + // Being superadmin here says nothing about the other instance, so a remote target is + // always asked. + let canPreserve = true + if (!$superadmin || isRemote) { + const targetUser = await UserService.whoami({ workspace: target! }) + canPreserve = + targetUser.is_admin || + targetUser.is_super_admin || + targetUser.groups?.includes('wm_deployers') || + false } + if (!stillCurrent()) return + canPreserveOnBehalfOf = canPreserve canSeeTarget = 'yes' - } catch { + } catch (e: any) { + if (!stillCurrent()) return canSeeTarget = 'cant-deploy-to-workspace' + targetError = e?.body ?? e?.message canPreserveOnBehalfOf = false return } @@ -139,6 +176,7 @@ try { allDeps = await getDependencies(kind, path) } catch { + if (!stillCurrent()) return canSeeTarget = 'cant-see-all-deps' return } @@ -150,10 +188,9 @@ } }) for (const dep of sortedSet) { - allAlreadyExists[computeStatusPath(dep.kind, dep.path)] = await checkAlreadyExists( - dep.kind, - dep.path - ) + const exists = await checkAlreadyExists(dep.kind, dep.path) + if (!stillCurrent()) return + allAlreadyExists[computeStatusPath(dep.kind, dep.path)] = exists } dependencies = sortedSet.map((x) => ({ ...x, @@ -169,26 +206,17 @@ ['flow', 'script', 'app', 'trigger'].includes(d.kind) )) { const key = computeStatusPath(dep.kind, dep.path) + let source: string | undefined + let targetValue: string | undefined try { - sourceOnBehalfOfInfo[key] = await getOnBehalfOf( - dep.kind, - dep.path, - $workspaceStore!, - additionalInformation - ) - } catch { - sourceOnBehalfOfInfo[key] = undefined - } + source = await getOnBehalfOf(dep.kind, dep.path, $workspaceStore!, additionalInformation) + } catch {} try { - targetOnBehalfOfInfo[key] = await getOnBehalfOf( - dep.kind, - dep.path, - workspaceToDeployTo!, - additionalInformation - ) - } catch { - targetOnBehalfOfInfo[key] = undefined - } + targetValue = await getOnBehalfOf(dep.kind, dep.path, target!, additionalInformation) + } catch {} + if (!stillCurrent()) return + sourceOnBehalfOfInfo[key] = source + targetOnBehalfOfInfo[key] = targetValue } } @@ -322,25 +350,29 @@ return checkItemExists(kind, path, workspaceToDeployTo!, additionalInformation) } - const deploymentStatus: Record< + let deploymentStatus: Record< string, { status: 'loading' | 'deployed' | 'failed'; error?: string } > = $state({}) async function deploy(kind: Kind, path: string) { const statusPath = computeStatusPath(kind, path) + const target = workspaceToDeployTo deploymentStatus[statusPath] = { status: 'loading' } const result = await deployItem({ kind, path, workspaceFrom: $workspaceStore!, - workspaceTo: workspaceToDeployTo!, + workspaceTo: target!, additionalInformation, onBehalfOf: getOnBehalfOfForDeploy(statusPath, kind), - onBehalfOfPrincipal: getOnBehalfOfPermissionedAsForDeploy(statusPath, kind) + onBehalfOfPrincipal: getOnBehalfOfPermissionedAsForDeploy(statusPath, kind), + targetBaseUrl: isRemote ? remoteTarget?.base_url : undefined }) + // The statuses on screen are the current destination's; this deploy went to `target`. + if (target !== workspaceToDeployTo) return if (result.success) { allAlreadyExists[statusPath] = true deploymentStatus[statusPath] = { status: 'deployed' } @@ -376,13 +408,45 @@ }) } + async function loadDestinations() { + const workspace = $workspaceStore! + const [deployTo, remote] = await Promise.all([ + WorkspaceService.getDeployTo({ workspace }), + WorkspaceService.getRemoteDeployTarget({ workspace }).catch(() => undefined) + ]) + if (workspace !== $workspaceStore) return + parentWorkspace = deployTo.deploy_to + remoteStatus = remote + // Keep the user's pick across a reload, but only while this workspace still has it: the + // drawer stays mounted when the workspace is switched. + const available = { + parent: parentWorkspace != undefined, + remote: remote?.target != undefined + } + if (!destination || !available[destination]) { + destination = available.parent ? 'parent' : available.remote ? 'remote' : undefined + } + notSet = destination == undefined + } + $effect(() => { - WorkspaceService.getDeployTo({ workspace: $workspaceStore! }).then((x) => { - workspaceToDeployTo = x.deploy_to - if (x.deploy_to == undefined) { - notSet = true - } - }) + $workspaceStore && untrack(() => loadDestinations()) + }) + + // Until the caller has a token for the remote instance there is nothing to deploy against, so + // the target stays unset and the drawer's Deploy buttons stay disabled. + let destinationWorkspace = $derived( + destination === 'parent' + ? parentWorkspace + : isRemote && remoteStatus?.connection + ? remoteDeployWorkspace($workspaceStore!, remoteStatus.connection) + : undefined + ) + let destinationLabel = $derived( + isRemote && remoteTarget ? remoteDeployLabel(remoteTarget) : (parentWorkspace ?? '') + ) + $effect(() => { + workspaceToDeployTo = destinationWorkspace }) $effect(() => { @@ -453,8 +517,9 @@ >Deploy to staging/prod from the web UI is only available with an enterprise license {:else if notSet == true} - As an admin, go to Settings {'->'} Workspace {'->'} Dev workspaceAs an admin, go to Settings {'->'} Workspace {'->'} Dev workspace, to pair this workspace with a + prod workspace on this instance or point it at a workspace on another instance {:else}

Destination Workspace  Workspace to deploy to is set in the workspace settingsDestination  Where this workspace deploys is set in the workspace settings

- + {#if parentWorkspace && remoteTarget} + (destination = v as Destination)} + noWFull + > + {#snippet children({ item })} + + + {/snippet} + + {:else} + + {/if} + + {#if isRemote && remoteTarget} +
+ +
+ {/if} {#if workspaceToDeployTo} { canDeployToWorkspace = canDeploy }} /> {/if} - {#if canSeeTarget == undefined} + {#if workspaceToDeployTo == undefined} + + {:else if canSeeTarget == undefined}
{:else if canSeeTarget == 'yes'} @@ -604,6 +709,15 @@ user to deploy this item using the shareable link or get the proper permissions on the dependencies + {:else if isRemote} +
+

{targetError ?? 'The remote instance did not accept the request'}

+

Disconnect above and connect again with a valid token for that instance, or ask someone + permissioned there to deploy this item using the shareable link

 {:else}
{} }: { parentWorkspaceId: string + /** How to name the target in the message, when its id is not what a reader would recognize + * — a remote deploy target is addressed through a proxy path. */ + displayName?: string onUpdateCanDeploy?: (value: boolean) => void } = $props() @@ -73,7 +77,8 @@

- The workspace {parentWorkspaceId} has a protection rule{activeDeployRulesets.length > 1 + The workspace {displayName ?? parentWorkspaceId} has a protection rule{activeDeployRulesets.length > + 1 ? 's' : ''} {activeDeployRulesets.map((r) => r.name).join(', ')} diff --git a/frontend/src/lib/components/RemoteDeployConnect.svelte b/frontend/src/lib/components/RemoteDeployConnect.svelte new file mode 100644 index 0000000000..d5824a4f2e --- /dev/null +++ b/frontend/src/lib/components/RemoteDeployConnect.svelte @@ -0,0 +1,181 @@ + + +{#if connection} +

+ + Deploying as {connection.remote_email} on + {target.base_url} + + +
+{:else} +
+

+ Deploys to {target.workspace_id} on {target.base_url} run with your own account on that instance, + so its permissions, protection rules and audit logs apply. Sign in there to connect; the token + it issues is stored encrypted and only ever sent to that instance. +

+
+ + {#if waitingForRemote} + Waiting for {host}… + {/if} + +
+ {#if showPaste} +
+ + Create a token on {host} + + +
+
+ e.key == 'Enter' && connect()} + /> +
+ +
+
+ {/if} + {#if error} +

{error}

+ {/if} +
+{/if} diff --git a/frontend/src/lib/components/RemoteDeployTargetSetting.svelte b/frontend/src/lib/components/RemoteDeployTargetSetting.svelte new file mode 100644 index 0000000000..91f876ec17 --- /dev/null +++ b/frontend/src/lib/components/RemoteDeployTargetSetting.svelte @@ -0,0 +1,111 @@ + + + + {#if !$enterpriseLicense} + + Deploying to another instance from the web UI is only available with an enterprise license + + {:else} +
+ + +
+ + {#if saved} + + {/if} +
+ {#if saved} +

+ Changing the instance URL or workspace drops every token stored for the current target, so + everyone connects again. +

+ {/if} +
+ {/if} + diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index dc0244da9d..1024aefa72 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -63,7 +63,8 @@ !label.startsWith('embed_app:') && !label.startsWith('sdk_app:') && !label.startsWith('impersonation:') && - !label.startsWith('cli-login:') + !label.startsWith('cli-login:') && + !label.startsWith('remote-deploy:') ) } diff --git a/frontend/src/lib/remoteDeploy.test.ts b/frontend/src/lib/remoteDeploy.test.ts new file mode 100644 index 0000000000..7064dc83b6 --- /dev/null +++ b/frontend/src/lib/remoteDeploy.test.ts @@ -0,0 +1,44 @@ +import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest' +import { PENDING_TTL_MS, remoteDeployAuthorizeUrl, takePendingConnect } from './remoteDeploy' + +const target = { base_url: 'https://prod.example.com', workspace_id: 'prod' } + +function startConnect(workspace = 'dev'): string { + const url = new URL(remoteDeployAuthorizeUrl(target, workspace, '/deploy/flow/f/a')) + return url.searchParams.get('state')! +} + +describe('remote deploy connect state', () => { + beforeEach(() => { + localStorage.clear() + vi.stubGlobal('window', { location: { origin: 'https://dev.example.com' } }) + }) + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + // The state is what keeps any other page from planting a token as the user's. + it('matches only the state it issued, and only once', () => { + const state = startConnect() + expect(takePendingConnect('forged')).toBeUndefined() + expect(takePendingConnect(state)).toMatchObject({ workspace: 'dev', target }) + expect(takePendingConnect(state)).toBeUndefined() + }) + + it('keeps concurrent attempts apart', () => { + const first = startConnect('dev') + const second = startConnect('staging') + expect(takePendingConnect(first)?.workspace).toBe('dev') + expect(takePendingConnect(second)?.workspace).toBe('staging') + }) + + it('expires an attempt left unfinished, and drops it from storage', () => { + vi.useFakeTimers() + const abandoned = startConnect() + vi.advanceTimersByTime(PENDING_TTL_MS + 60_000) + startConnect() + expect(localStorage.length).toBe(1) + expect(takePendingConnect(abandoned)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/remoteDeploy.ts b/frontend/src/lib/remoteDeploy.ts new file mode 100644 index 0000000000..ad887504fb --- /dev/null +++ b/frontend/src/lib/remoteDeploy.ts @@ -0,0 +1,101 @@ +import { base } from '$lib/base' +import type { RemoteDeployConnection, RemoteDeployTarget } from '$lib/gen' +import { randomSecret } from '$lib/utils/uuid' + +/** + * The workspace argument that aims a generated-client call at `workspace`'s deploy target on + * another instance. The client fills `{workspace}` with `encodeURI`, which keeps the slashes, so + * `/w/{workspace}/flows/create` reaches `/w//remote_deploy/proxy//flows/create`, + * which the backend forwards to the remote workspace with the caller's token for it. + */ +export function remoteDeployWorkspace( + workspace: string, + connection: RemoteDeployConnection +): string { + return `${workspace}/remote_deploy/proxy/${connection.proxy_key}` +} + +export function remoteDeployLabel(target: RemoteDeployTarget): string { + return `${target.workspace_id} on ${remoteHost(target.base_url)}` +} + +export function remoteHost(url: string): string { + try { + return new URL(url).host + } catch { + return url + } +} + +// Connecting by signing in on the remote: the drawer opens the remote's authorize page, which mints +// a token and sends it back in the fragment of `CALLBACK_PATH`; that page stores it through +// `connect` and tells the drawer on `CHANNEL`. + +/** The path the remote's authorize page only ever sends a token to. */ +export const CALLBACK_PATH = '/remote_deploy/callback' +export const CHANNEL = 'windmill-remote-deploy' +const PENDING_PREFIX = 'remote-deploy-connect:' +export const PENDING_TTL_MS = 15 * 60_000 + +type PendingConnect = { + workspace: string + /** Sent back with the token, so it only reaches the instance it came from. */ + target: RemoteDeployTarget + returnTo: string + at: number +} + +export type RemoteDeployEvent = + | { type: 'connected'; workspace: string } + | { type: 'failed'; workspace: string; error: string } + +/** + * The remote authorize URL for connecting `workspace`. Records a `state` for the callback to + * check: without it, any page could send this instance a token of its choosing to store as the + * user's, and their deploys would land on the remote as someone else. `localStorage` rather than + * `sessionStorage`, because the callback runs in the window the remote sends back to, which is + * not always this one; one entry per attempt, so two tabs connecting at once do not cancel out. + */ +export function remoteDeployAuthorizeUrl( + target: RemoteDeployTarget, + workspace: string, + returnTo: string +): string { + dropExpiredConnects() + const state = randomSecret(16) + const pending: PendingConnect = { workspace, target, returnTo, at: Date.now() } + localStorage.setItem(PENDING_PREFIX + state, JSON.stringify(pending)) + const params = new URLSearchParams({ + workspace: target.workspace_id, + callback: `${window.location.origin}${base}${CALLBACK_PATH}`, + state + }) + return `${target.base_url}/user/remote_deploy_authorize?${params}` +} + +/** An attempt abandoned before its callback would otherwise stay in storage for good. */ +function dropExpiredConnects() { + for (let i = localStorage.length - 1; i >= 0; i--) { + const key = localStorage.key(i) + if (!key?.startsWith(PENDING_PREFIX)) continue + let at = 0 + try { + at = JSON.parse(localStorage.getItem(key) ?? '{}').at ?? 0 + } catch {} + if (Date.now() - at > PENDING_TTL_MS) localStorage.removeItem(key) + } +} + +/** The attempt `state` names, used once: a replayed callback finds nothing to match. */ +export function takePendingConnect(state: string): PendingConnect | undefined { + const key = PENDING_PREFIX + state + let pending: PendingConnect | undefined + try { + pending = JSON.parse(localStorage.getItem(key) ?? 'null') ?? undefined + } catch {} + localStorage.removeItem(key) + if (!pending || Date.now() - pending.at > PENDING_TTL_MS) { + return undefined + } + return pending +} diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 7d517c3bb4..63ea88adcb 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -151,7 +151,10 @@ export async function getTriggersDeployData( kind: TriggerKind, path: string, workspace: string, - onBehalfOf?: string + onBehalfOf?: string, + /** Root URL of the instance the trigger is deployed to, when that is not this one. A GCP or + * Azure push subscription posts to the instance serving the trigger, so it has to name that one. */ + targetBaseUrl?: string ) { const preservePermissionedAs = onBehalfOf !== undefined @@ -247,7 +250,9 @@ export async function getTriggersDeployData( ...gcpTrigger, delivery_config: gcpTrigger.delivery_config ?? undefined, base_endpoint: - gcpTrigger.delivery_type === 'push' ? `${window.location.origin}${base}` : undefined, + gcpTrigger.delivery_type === 'push' + ? (targetBaseUrl ?? `${window.location.origin}${base}`) + : undefined, permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs } @@ -311,6 +316,11 @@ export async function getTriggersDeployData( return { data: { ...azureTrigger, + // Not stored on the trigger, but a push subscription is created from it. + base_endpoint: + azureTrigger.azure_mode === 'namespace_pull' + ? undefined + : (targetBaseUrl ?? `${window.location.origin}${base}`), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 228e32fc06..46dc39b737 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -362,6 +362,11 @@ export interface DeployItemParams { * between the two probes. The result then carries `conflict`. */ createOnly?: boolean + /** + * Root URL of the instance `workspaceTo` lives on, when the deploy crosses instances. Only what + * a payload must state about where it landed needs it — a GCP or Azure push trigger's endpoint. + */ + targetBaseUrl?: string } /** @@ -380,7 +385,8 @@ export async function deployItem( additionalInformation, onBehalfOf, onBehalfOfPrincipal, - createOnly + createOnly, + targetBaseUrl } = params if (kind === 'trigger') { @@ -399,7 +405,8 @@ export async function deployItem( additionalInformation.triggers.kind, path, workspaceFrom, - onBehalfOf + onBehalfOf, + targetBaseUrl ) if (alreadyExists) { // Strip operational state so the update doesn't flip the target's diff --git a/frontend/src/routes/(root)/(logged)/remote_deploy/callback/+page.svelte b/frontend/src/routes/(root)/(logged)/remote_deploy/callback/+page.svelte new file mode 100644 index 0000000000..c702f66471 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/remote_deploy/callback/+page.svelte @@ -0,0 +1,74 @@ + + + + {#if status === 'connecting'} +
+ {:else if status === 'connected'} +

Connected. You can close this tab.

+ {:else} + {error} + {#if returnTo} +
+ +
+ {/if} + {/if} + diff --git a/frontend/src/routes/(root)/(logged)/user/remote_deploy_authorize/+page.svelte b/frontend/src/routes/(root)/(logged)/user/remote_deploy_authorize/+page.svelte new file mode 100644 index 0000000000..2362b7f116 --- /dev/null +++ b/frontend/src/routes/(root)/(logged)/user/remote_deploy_authorize/+page.svelte @@ -0,0 +1,132 @@ + + + + {#if framed} + + This page cannot be used inside another page. Open it in its own tab. + + {:else if invalid} + + This authorization link is malformed. Start again from the deploy drawer of the other + instance. + + {:else} +
+

+ {callback!.origin} asks to deploy into workspace + {workspace} on this instance as + {$usersWorkspaceStore?.email ?? 'you'}. +

+

+ It will receive a token limited to that workspace, with your permissions in it, valid for up + to 90 days. Only authorize if you just started this from {callback!.host}. You can revoke + the token at any time in your account settings, under tokens. +

+ {#if unencrypted} + + {callback!.origin} is served over plain http, so the token will cross the network unencrypted + on its way there. + + {/if} + {#if error} + {error} + {/if} +
+
+ + +
+ {/if} + {#if framed || invalid} +
+ +
+ {/if} + diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 424aa0267e..053ec2f2a1 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -8,6 +8,7 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import DeployToSetting from '$lib/components/DeployToSetting.svelte' + import RemoteDeployTargetSetting from '$lib/components/RemoteDeployTargetSetting.svelte' import DevWorkspaceSetting from '$lib/components/DevWorkspaceSetting.svelte' import ErrorOrRecoveryHandler from '$lib/components/ErrorOrRecoveryHandler.svelte' import PageHeader from '$lib/components/PageHeader.svelte' @@ -1552,6 +1553,12 @@
+ +
+ Deploy to another instance + +
{:else if tab == 'rulesets'} Date: Tue, 22 Sep 2026 02:57:05 -0400 Subject: [PATCH 04/34] fix: clear the impersonation token from session storage on logout (#11274) Impersonating a service account stashes the impersonator's own auth token under `pre_impersonation_token` in sessionStorage so the banner can restore it. Ending impersonation removed it, but logging out did not: `clearUser()` only reset the in-memory stores, leaving a usable token readable for the lifetime of the tab. Remove both impersonation keys as part of `clearUser()`, in their own try/catch so a storage failure (the opaque-origin iframe shim path) cannot skip the server-side logout. Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/logout.test.ts | 28 ++++++++++++++++++++++++++++ frontend/src/lib/logout.ts | 13 +++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 frontend/src/lib/logout.test.ts diff --git a/frontend/src/lib/logout.test.ts b/frontend/src/lib/logout.test.ts new file mode 100644 index 0000000000..160bce9934 --- /dev/null +++ b/frontend/src/lib/logout.test.ts @@ -0,0 +1,28 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' + +vi.mock('./onboardingProfile', () => ({ noteSessionEmail: vi.fn() })) +vi.mock('./components/sidebar/accountSetup.svelte', () => ({ + accountSetup: { reset: vi.fn() } +})) +vi.mock('./storeUtils', () => ({ clearStores: vi.fn() })) +vi.mock('$lib/gen', () => ({ UserService: { logout: vi.fn(async () => {}) } })) + +import { clearUser } from './logout' + +describe('clearUser', () => { + beforeEach(() => { + sessionStorage.clear() + vi.clearAllMocks() + }) + + // Logging out while impersonating must not leave the impersonator's own token behind. + it('drops the impersonation token backup', async () => { + sessionStorage.setItem('pre_impersonation_token', 'a-token') + sessionStorage.setItem('pre_impersonation_email', 'admin@windmill.dev') + + await clearUser() + + expect(sessionStorage.getItem('pre_impersonation_token')).toBeNull() + expect(sessionStorage.getItem('pre_impersonation_email')).toBeNull() + }) +}) diff --git a/frontend/src/lib/logout.ts b/frontend/src/lib/logout.ts index 1c0915dfa9..58fce16fe3 100644 --- a/frontend/src/lib/logout.ts +++ b/frontend/src/lib/logout.ts @@ -6,11 +6,24 @@ import { clearStores } from './storeUtils' // Note: logout and logoutWithRedirect have been moved to logoutKit.ts // as they depend on SvelteKit navigation +// The impersonation backup holds the impersonator's own auth token, so it must go on +// logout too, not only when impersonation is ended from the banner — otherwise a usable +// token stays readable for the lifetime of the tab. +function clearImpersonationFromStorage(): void { + try { + sessionStorage.removeItem('pre_impersonation_token') + sessionStorage.removeItem('pre_impersonation_email') + } catch (e) { + console.error('error interacting with session storage', e) + } +} + export async function clearUser() { try { noteSessionEmail(undefined) accountSetup.reset() clearStores() + clearImpersonationFromStorage() await UserService.logout() } catch (error) {} } From 9e58f9a20f40545d10c3885a66b4ee9efd9c10c2 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 22 Sep 2026 02:57:21 -0400 Subject: [PATCH 05/34] chore(security): resolve Dependabot alerts in backend/Cargo.lock and windmill-parser-wasm/Cargo.lock (#11182) Targeted dependency bumps only; no behavior change intended. Co-authored-by: Claude Fable 5.1 --- backend/Cargo.lock | 92 ++++++------ backend/Cargo.toml | 10 +- .../parsers/windmill-parser-wasm/Cargo.lock | 133 ++++++++++++------ 3 files changed, 142 insertions(+), 93 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ebd6dce019..82c625df67 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -588,15 +588,15 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.5.6" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" dependencies = [ - "filetime", "futures-core", "libc", "portable-atomic", "rustc-hash 2.1.3", + "rustix 1.1.5", "tokio", "tokio-stream", "xattr", @@ -1070,15 +1070,16 @@ dependencies = [ [[package]] name = "aws-sdk-config" -version = "1.68.0" +version = "1.101.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7de8bcff88ff2948bf2154204b7eba0ea30698c1c23770c4a917505fb9fcbd5" +checksum = "0360cf8f8ab0e328f89b5c60bdcc6203ebce83b8a018c05ed177c4f32f819aff" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1086,7 +1087,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", - "once_cell", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1144,15 +1145,16 @@ dependencies = [ [[package]] name = "aws-sdk-sqs" -version = "1.77.0" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a6dcd63dffae1f7872df42ce9f1f0f785f8c9a3d3e880ef6c171d0415439df" +checksum = "0894909314920ab2c907110e7f91b9f7a6c826bdc830ccf36079c69af2b82335" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1160,6 +1162,7 @@ dependencies = [ "bytes", "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] @@ -1214,15 +1217,16 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.79.0" +version = "1.98.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317377afba3498fca4948c5d32b399ef9a5ad35561a1e8a6f2ac7273dabf802d" +checksum = "89c4f19655ab0856375e169865c91264de965bd74c407c7f1e403184b1049409" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", + "aws-smithy-observability", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -1231,6 +1235,7 @@ dependencies = [ "aws-types", "fastrand 2.5.0", "http 0.2.12", + "http 1.5.0", "regex-lite", "tracing", ] @@ -2221,7 +2226,7 @@ dependencies = [ "memmap2", "num-traits", "num_cpus", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", "rayon", "safetensors", @@ -2257,7 +2262,7 @@ dependencies = [ "candle-nn", "fancy-regex 0.17.0", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rayon", "serde", "serde_json", @@ -5101,7 +5106,7 @@ checksum = "719a903cc23e4a89e87962c2a80fdb45cdaad0983a89bd150bb57b4c8571a7d5" dependencies = [ "half", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", ] @@ -5833,7 +5838,7 @@ dependencies = [ "cfg-if", "crunchy", "num-traits", - "rand 0.9.0", + "rand 0.9.5", "rand_distr", "zerocopy", ] @@ -5975,7 +5980,7 @@ dependencies = [ "log", "native-tls", "num_cpus", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.12.28", "serde", "serde_json", @@ -6002,7 +6007,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.0", + "rand 0.9.5", "serde", "thiserror 2.0.20", "tinyvec", @@ -6024,7 +6029,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.0", + "rand 0.9.5", "resolv-conf", "serde", "smallvec", @@ -8444,7 +8449,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.12.28", "ring 0.17.14", "rustls-pemfile 2.2.0", @@ -8813,7 +8818,7 @@ dependencies = [ "futures-util", "opentelemetry 0.30.0", "percent-encoding", - "rand 0.9.0", + "rand 0.9.5", "serde_json", "thiserror 2.0.20", "tokio", @@ -8915,7 +8920,7 @@ dependencies = [ "hmac", "pkcs12", "pkcs5", - "rand 0.9.0", + "rand 0.9.5", "rc2", "sha1", "sha2 0.10.9", @@ -9499,7 +9504,7 @@ dependencies = [ "hmac", "md-5 0.10.6", "memchr", - "rand 0.9.0", + "rand 0.9.5", "sha2 0.10.9", "stringprep", ] @@ -9954,13 +9959,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", - "zerocopy", ] [[package]] @@ -10044,7 +10048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" dependencies = [ "num-traits", - "rand 0.9.0", + "rand 0.9.5", ] [[package]] @@ -11109,7 +11113,7 @@ dependencies = [ "openssl-sys", "pkg-config", "quick-xml", - "rand 0.9.0", + "rand 0.9.5", "serde", "thiserror 2.0.20", "url", @@ -13392,7 +13396,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.5", "socket2 0.6.5", "tokio", "tokio-util", @@ -14084,7 +14088,7 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "470dbf6591da1b39d43c14523b2b469c86879a53e8b758c8e090a470fe7b1fbe" dependencies = [ - "rand 0.9.0", + "rand 0.9.5", "uuid", "web-time", ] @@ -14811,7 +14815,7 @@ dependencies = [ "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", "prometheus", - "rand 0.9.0", + "rand 0.9.5", "rdkafka", "reqwest 0.13.5", "rumqttc", @@ -14971,7 +14975,7 @@ dependencies = [ "prometheus", "quick-xml", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "rsa", @@ -15287,7 +15291,7 @@ dependencies = [ "futures", "hex", "hmac", - "rand 0.9.0", + "rand 0.9.5", "rdkafka", "reqwest 0.13.5", "rmcp", @@ -15653,7 +15657,7 @@ dependencies = [ "postgres-native-tls 0.5.3", "prometheus", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "reqwest-middleware", @@ -16214,7 +16218,7 @@ dependencies = [ "once_cell", "prometheus", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest 0.13.5", "serde", @@ -16354,7 +16358,7 @@ dependencies = [ "hyper 1.11.1", "itertools 0.14.0", "lazy_static", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -16418,7 +16422,7 @@ dependencies = [ "itertools 0.14.0", "lazy_static", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -16613,7 +16617,7 @@ dependencies = [ "pg_escape", "postgres-native-tls 0.5.2", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "rust_decimal", "serde", "serde_json", @@ -16693,7 +16697,7 @@ dependencies = [ "chrono", "hex", "itertools 0.14.0", - "rand 0.9.0", + "rand 0.9.5", "serde", "serde_json", "sqlx", @@ -16758,7 +16762,7 @@ dependencies = [ "prometheus", "prost", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "rcgen", "regex", "reqwest 0.13.5", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b25f331b88..a911d57b81 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -493,7 +493,7 @@ sql-builder = "^3" # behind a feature and changed `hash_password`), so a float breaks the build. argon2 = "0.6" quick_cache = "^0" -rand = "=0.9.0" +rand = "^0.9.3" rand_core = { version = "^0", features = ["std"] } arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } @@ -514,7 +514,7 @@ reqwest = { version = "^0.13", features = ["json", "stream", "gzip", "multipart" eventsource-stream = "0.2.3" time = "^0" serde_urlencoded = "^0" -astral-tokio-tar = "^0.5.6" +astral-tokio-tar = "^0.6.2" tempfile = "^3" x509-parser = "^0.16" tokio-util = { version = "=0.7.17", features = ["io"] } @@ -523,7 +523,7 @@ itertools = "^0.14.0" regex = "^1" semver = "^1" aws-sigv4 = "^1.3.4" -aws-sdk-config = "=1.68.0" +aws-sdk-config = "^1.94.0" aws-sdk-rds = "^1" async-trait = "0.1.88" @@ -685,8 +685,8 @@ aws-sdk-bedrockruntime = "=1.122.0" aws-credential-types = "^1" aws-smithy-types = "^1" aws-sdk-secretsmanager = "^1" -aws-sdk-sqs = "=1.77.0" -aws-sdk-sts = "=1.79.0" +aws-sdk-sqs = "^1.89.0" +aws-sdk-sts = "^1.91.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" rustls = "=0.23.35" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 83f0414e86..dcd846c0d8 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -775,9 +775,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" [[package]] name = "colorchoice" @@ -1530,7 +1530,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -2580,7 +2580,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", - "wasi", + "wasi 0.11.1+wasi-snapshot-preview1", "windows-sys 0.61.2", ] @@ -2816,7 +2816,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -2902,15 +2902,14 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.78" +version = "0.10.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" dependencies = [ "bitflags", "cfg-if", "foreign-types", "libc", - "once_cell", "openssl-macros", "openssl-sys", ] @@ -2934,9 +2933,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-sys" -version = "0.9.114" +version = "0.9.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", @@ -3047,6 +3046,16 @@ dependencies = [ "phf_shared 0.12.1", ] +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" +dependencies = [ + "phf_shared 0.13.1", + "serde", +] + [[package]] name = "phf_codegen" version = "0.11.3" @@ -3064,7 +3073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -3098,6 +3107,15 @@ dependencies = [ "siphasher 1.0.2", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher 1.0.2", +] + [[package]] name = "php-parser-rs" version = "0.1.3" @@ -3169,9 +3187,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56201207dac53e2f38e848e31b4b91616a6bb6e0c7205b77718994a7f49e70fc" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" dependencies = [ "base64 0.22.1", "byteorder", @@ -3187,9 +3205,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.9" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" dependencies = [ "array-init", "bit-vec 0.6.3", @@ -3197,7 +3215,7 @@ dependencies = [ "chrono", "fallible-iterator", "postgres-protocol", - "serde", + "serde_core", "serde_json", "uuid", ] @@ -3394,15 +3412,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "aws-lc-rs", "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.0", + "rand 0.9.5", "ring", "rustc-hash 2.1.1", "rustls", @@ -3457,9 +3475,9 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -3468,13 +3486,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", - "zerocopy", ] [[package]] @@ -3752,7 +3769,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "503c78f59814e2664c9980b739b19e40f549233ecf39928040ee54fdd431f614" dependencies = [ - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -3828,7 +3845,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "rkyv", "serde", "serde_json", @@ -3944,9 +3961,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -4574,7 +4591,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "rsa", "serde", "sha1", @@ -4585,7 +4602,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -4616,7 +4633,7 @@ dependencies = [ "memchr", "num-bigint", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "sha2 0.10.9", @@ -4626,7 +4643,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "uuid", - "whoami", + "whoami 1.6.1", ] [[package]] @@ -5043,9 +5060,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", @@ -5254,9 +5271,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.13" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" dependencies = [ "async-trait", "byteorder", @@ -5267,15 +5284,15 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf 0.11.3", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", - "socket2 0.5.10", + "rand 0.10.1", + "socket2 0.6.3", "tokio", "tokio-util", - "whoami", + "whoami 2.1.0", ] [[package]] @@ -5777,7 +5794,7 @@ dependencies = [ "getopts", "log", "phf_codegen", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -5899,6 +5916,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -5923,6 +5949,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.103" @@ -6134,7 +6169,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "wasite", + "wasite 0.1.0", +] + +[[package]] +name = "whoami" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fae98cf96deed1b7572272dfc777713c249ae40aa1cf8862e091e8b745f5361" +dependencies = [ + "libredox", + "wasite 1.0.2", "web-sys", ] @@ -6216,7 +6261,7 @@ dependencies = [ "pkcs1", "postgres-native-tls", "quick_cache", - "rand 0.9.0", + "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", @@ -6576,7 +6621,7 @@ dependencies = [ "chrono", "hex", "itertools 0.14.0", - "rand 0.9.0", + "rand 0.9.5", "serde", "serde_json", "sqlx", From 5798bd07e7aaef3c1381d8c9422af88240981944 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 09:25:12 +0200 Subject: [PATCH 06/34] build derived release images from this run's image digest, not :dev (#11280) * fix: build derived release images from this run's image digest, not :dev Co-Authored-By: Claude Opus 5 (1M context) * fix: scope the digest comment to jobs that build from, extract or retag Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/docker-image.yml | 39 +++++++++++++++++++++--------- docker/DockerfileCuda | 3 ++- docker/DockerfileFull | 3 ++- docker/DockerfileFullEe | 4 ++- docker/DockerfileSlim | 5 +++- docker/DockerfileSlimEe | 5 +++- 6 files changed, 43 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a8b20623bc..b468387057 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -40,6 +40,11 @@ jobs: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || (github.event.inputs && !github.event.inputs.ee) + # Jobs that build from, extract or retag this image must reference it by + # this digest: concurrent main-branch runs push :dev too, so a tag run + # reading :dev can ship a main build under the release version. + outputs: + digest: ${{ steps.docker_build.outputs.digest }} steps: - uses: actions/checkout@v4 with: @@ -112,6 +117,8 @@ jobs: build_ee: runs-on: ubicloud if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee + outputs: + digest: ${{ steps.docker_build.outputs.digest }} steps: - uses: actions/checkout@v4 with: @@ -192,31 +199,31 @@ jobs: - uses: actions/checkout@v4 - run: | # pulling docker image with desired arch so that actions-docker-extract doesn't do it - docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} - docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} + docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} - run: | # Checks the image is in docker prior to running actions-docker-extract. It fails if not # Also useful to visually check that the arch is the right opencontainers - docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} - docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} + docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} - uses: shrink/actions-docker-extract@v3 id: extract with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} path: "/usr/src/app/windmill" - uses: shrink/actions-docker-extract@v3 id: extract-duckdb-ffi-internal with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so" - uses: shrink/actions-docker-extract@v3 id: extract-ee with: - image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} + image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} path: "/usr/src/app/windmill" - name: Rename binary with corresponding architecture @@ -373,8 +380,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tag main and latest run: | - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main - uses: sigstore/cosign-installer@v4.1.2 if: startsWith(github.ref, 'refs/tags/v') @@ -409,8 +416,8 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Tag main and latest for ee run: | - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest - docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest + docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main - uses: sigstore/cosign-installer@v4.1.2 if: startsWith(github.ref, 'refs/tags/v') @@ -549,6 +556,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileCuda" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -602,6 +611,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlim" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -656,6 +667,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileSlimEe" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | @@ -709,6 +722,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFull" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ needs.build.outputs.digest }} tags: | ${{ steps.meta-public.outputs.tags }} labels: | @@ -761,6 +776,8 @@ jobs: push: true sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} file: "./docker/DockerfileFullEe" + build-args: | + BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/docker/DockerfileCuda b/docker/DockerfileCuda index 098670f9a3..031921070b 100644 --- a/docker/DockerfileCuda +++ b/docker/DockerfileCuda @@ -1,4 +1,5 @@ -FROM ghcr.io/windmill-labs/windmill-ee:dev +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev +FROM ${BASE_IMAGE} # Install any necessary dependencies for managing repositories and packages RUN apt-get update && apt-get install -y curl gnupg2 diff --git a/docker/DockerfileFull b/docker/DockerfileFull index b827208c1d..8ef6be78bb 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,4 +1,5 @@ -FROM ghcr.io/windmill-labs/windmill:dev +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev +FROM ${BASE_IMAGE} # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 5b5347558a..88ed619c19 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -1,3 +1,5 @@ +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev + FROM alpine:3.14 AS oracledb-client ARG TARGETPLATFORM @@ -17,7 +19,7 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ mv /opt/oracle/23/amd64/lib /opt/oracle/23/lib; \ fi -FROM ghcr.io/windmill-labs/windmill-ee:dev +FROM ${BASE_IMAGE} # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 64fd1b735e..6418a2cfda 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -1,4 +1,7 @@ ARG DEBIAN_IMAGE=debian:trixie-slim +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev + +FROM ${BASE_IMAGE} AS windmill_image FROM debian:trixie-slim AS nsjail @@ -103,7 +106,7 @@ RUN arch="$(dpkg --print-architecture)"; \ WORKDIR ${APP} -COPY --from=ghcr.io/windmill-labs/windmill:dev --chmod=755 ${APP}/windmill ${APP}/windmill +COPY --from=windmill_image --chmod=755 ${APP}/windmill ${APP}/windmill RUN ln -s ${APP}/windmill /usr/local/bin/windmill diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index de0d61a780..24ef941b87 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -1,4 +1,7 @@ ARG DEBIAN_IMAGE=debian:trixie-slim +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee:dev + +FROM ${BASE_IMAGE} AS windmill_image FROM debian:trixie-slim AS nsjail @@ -103,7 +106,7 @@ RUN arch="$(dpkg --print-architecture)"; \ WORKDIR ${APP} -COPY --from=ghcr.io/windmill-labs/windmill-ee:dev --chmod=755 ${APP}/windmill ${APP}/windmill +COPY --from=windmill_image --chmod=755 ${APP}/windmill ${APP}/windmill RUN ln -s ${APP}/windmill /usr/local/bin/windmill From 69fafb9262f90184e6c2dc9a84f1c3feeb9d9638 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 09:30:48 +0200 Subject: [PATCH 07/34] feat: check dynamic worker tags on the tag they resolve to (#11271) * feat: check dynamic worker tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * test: pin the tag check after a flow preprocessor resolves the tag Co-Authored-By: Claude Opus 5 (1M context) * fix: keep tied tag placeholders exact and drop an unresolvable flow tag Co-Authored-By: Claude Opus 5 (1M context) * fix: describe tied tag placeholders in the custom tags editor Co-Authored-By: Claude Opus 5 (1M context) * fix: check workflow-as-code task tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * fix: judge a preprocessed flow's tag as written when it is run Co-Authored-By: Claude Opus 5 (1M context) * fix: check a preprocessed flow's own tag, as written only where it reads args Co-Authored-By: Claude Opus 5 (1M context) * feat: explain dynamic tags in the worker tag picker Co-Authored-By: Claude Opus 5 (1M context) * fix: look up worker availability for typed static tags Co-Authored-By: Claude Opus 5 (1M context) * fix: poll worker availability with the tag list as it is at each run Co-Authored-By: Claude Opus 5 (1M context) * feat: warn on custom tags with nothing fixed at their start or end Co-Authored-By: Claude Opus 5 (1M context) * fix: open the broad custom tag warning from the keyboard Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/worker.rs | 180 ++++++++++++++++-- backend/windmill-api-jobs/src/execution.rs | 65 ++++++- backend/windmill-api/src/apps.rs | 6 +- backend/windmill-api/src/jobs.rs | 48 ++--- backend/windmill-common/src/jobs.rs | 66 +++++-- backend/windmill-common/src/worker.rs | 115 +++++++++++ backend/windmill-queue/src/jobs.rs | 82 ++++++-- backend/windmill-queue/src/schedule.rs | 6 +- .../windmill-trigger/src/trigger_helpers.rs | 5 +- backend/windmill-worker/src/bun_executor.rs | 44 +++-- backend/windmill-worker/src/worker_flow.rs | 97 +++++----- .../lib/components/AssignableTagsInner.svelte | 82 +++++++- .../src/lib/components/WorkerTagSelect.svelte | 59 +++--- .../components/select/GenericDropdown.svelte | 3 + 14 files changed, 678 insertions(+), 180 deletions(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b8bd6b30a7..74a10488e6 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5439,7 +5439,7 @@ async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { } /// Test that flow substeps with tags that are not available for the workspace fail. -/// This validates that `check_tag_available_for_workspace_internal` is properly called +/// This validates that `check_tag_available_for_push` is properly called /// when pushing jobs from worker_flow. #[sqlx::test(fixtures("base"))] #[serial] @@ -5512,17 +5512,178 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: Ok(()) } +/// A step's `$flow_expr[...]` tag is checked on the queue it resolves to, not on its text: the +/// custom tags list `bun` alone, which admits the step resolving to it and refuses the one that +/// lands elsewhere. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "bun".to_string() + ]))); + + let step = |id: &str, tag: Option<&str>| { + json!({ + "id": id, + "value": { + "type": "rawscript", + "language": "deno", + "content": "export function main() { return { lang: 'bun' } }", + "tag": tag, + }, + }) + }; + let flow: FlowValue = serde_json::from_value(json!({ + "modules": [ + step("a", None), + step("b", Some("$flow_expr[results.a.lang]")), + step("c", Some("$flow_expr[results.a.lang]-gpu")), + ], + }))?; + + // A non-superadmin, so the custom tags apply. + let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, server.addr.port()) + .await; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + let b_tag = sqlx::query_scalar::<_, String>( + "SELECT tag FROM v2_job WHERE parent_job = $1 AND flow_step_id = 'b'", + ) + .bind(job.id) + .fetch_one(&db) + .await?; + assert_eq!(b_tag, "bun"); + + assert!(!job.success); + let result = job.json_result().unwrap(); + let message = result["error"]["message"].as_str().unwrap_or_default(); + assert!( + message.contains("(resolved to bun-gpu) is not included in the allowed CUSTOM_TAGS"), + "got {result:?}" + ); + + Ok(()) +} + +/// A flow whose preprocessor runs lands on its tag filled in from the preprocessor's output, so +/// running it judges the tag as written: even raw args that fill it in to a listed tag do not +/// admit a template nobody listed. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "hello"))] +#[serial] +async fn test_flow_tag_judged_as_written_before_preprocessor( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let set_flow_tag = |tag: &'static str| { + let db = db.clone(); + async move { + sqlx::query("UPDATE flow SET tag = $1 WHERE path = 'f/system/hello_with_preprocessor'") + .bind(tag) + .execute(&db) + .await?; + // Flow info is cached per version, which a real redeploy would bump. + windmill_common::FLOW_INFO_CACHE + .remove(&("test-workspace".to_string(), 1443253234253456)); + anyhow::Ok(()) + } + }; + // As a non-superadmin, so the custom tags apply. + let run_as_user = |entries: &[&str], query: &str| { + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from( + entries.iter().map(|e| e.to_string()).collect(), + ))); + reqwest::Client::new() + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/f/f/system/hello_with_preprocessor{query}" + )) + .bearer_auth("SECRET_TOKEN_2") + .json(&json!({ "foo": "bar" })) + .send() + }; + + sqlx::query( + "UPDATE flow SET extra_perms = '{\"u/test-user-2\": false}' + WHERE path = 'f/system/hello_with_preprocessor'", + ) + .execute(&db) + .await?; + set_flow_tag("pp-$args[foo]").await?; + let refused = run_as_user(&["pp-bar"], "").await?; + let refused_status = refused.status(); + let refused_body = refused.text().await?; + // `push` drops a `?tag=` of such a flow, so it does not stand in for the flow's own tag. + let overridden = run_as_user(&["pp-bar"], "?tag=pp-bar").await?.status(); + let admitted = run_as_user(&["pp-$args[foo]"], "").await?.status(); + // A tag the preprocessor cannot change is known now, so patterns apply to it. + set_flow_tag("pp-large").await?; + let static_admitted = run_as_user(&["pp-$args[size]"], "").await?.status(); + + // Nothing a flow's tag can read resolves `$flow_expr[...]`, so the flow keeps its tag. + set_flow_tag("$flow_expr[flow_input.foo]").await?; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + let unresolvable = RunJob::from(JobPayload::Flow { + path: "f/system/hello_with_preprocessor".to_string(), + dedicated_worker: None, + apply_preprocessor: true, + version: 1443253234253456, + labels: None, + }) + .as_user("test-user-2", "test2@windmill.dev") + .run_until_complete(&db, false, port) + .await; + + assert_eq!(refused_status, 400, "got {refused_body}"); + assert!( + refused_body.contains("Tag pp-$args[foo] is not included"), + "got {refused_body}" + ); + assert_eq!(overridden, 400); + assert!(admitted.is_success(), "got {admitted}"); + assert!(static_admitted.is_success(), "got {static_admitted}"); + + assert!(unresolvable.success, "got {:?}", unresolvable.json_result()); + let tag = sqlx::query_scalar::<_, String>("SELECT tag FROM v2_job WHERE id = $1") + .bind(unresolvable.id) + .fetch_one(&db) + .await?; + assert_eq!(tag, "flow"); + + Ok(()) +} + /// The `*` fork marker only grants through a real `parent_workspace_id` lineage lookup, which the /// parse-level unit tests cannot reach: they hand `applies_to_workspace` a synthetic chain, so a /// regression in the lookup or in the `is_fork_scoped()` gate that skips it would pass them. #[sqlx::test(fixtures("base"))] #[serial] async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> anyhow::Result<()> { - use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; initialize_tracing().await; + // For a non-superadmin caller (a superadmin would bypass the scope check entirely). + async fn allowed(db: &Pool, w_id: &str, tag: &str) -> bool { + let no_args = std::collections::HashMap::new(); + let args = windmill_queue::PushArgs::from(&no_args); + windmill_queue::check_tag_available_for_push(db, w_id, tag, &args, false, None) + .await + .is_ok() + } + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. let fork = "wm-fork-tagmarker"; sqlx::query!( @@ -5538,27 +5699,18 @@ async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> a "bare(test-workspace)".to_string(), ]))); - // A non-superadmin caller (a superadmin would bypass the scope check entirely). - let is_super_admin = false; - for (w_id, tag) in [("test-workspace", "bare"), ("test-workspace", "forky")] { assert!( - check_tag_available_for_workspace_internal(&db, w_id, tag, is_super_admin, None) - .await - .is_ok(), + allowed(&db, w_id, tag).await, "{tag} should be available in the workspace it names" ); } assert!( - check_tag_available_for_workspace_internal(&db, fork, "forky", is_super_admin, None) - .await - .is_ok(), + allowed(&db, fork, "forky").await, "a `*` tag must be granted to a fork through its parent lineage" ); assert!( - check_tag_available_for_workspace_internal(&db, fork, "bare", is_super_admin, None) - .await - .is_err(), + !allowed(&db, fork, "bare").await, "an unmarked tag must not reach a fork of the workspace it names" ); diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 7184f8ecec..9ad7189434 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -40,8 +40,8 @@ use windmill_common::{ FlowVersionInfo, DB, }; use windmill_queue::{ - cancel_job, get_result_and_success_by_id_from_flow, parse_result_object, push, PushArgs, - PushArgsOwned, PushIsolationLevel, + cancel_job, check_tag_available_for_push, get_result_and_success_by_id_from_flow, + parse_result_object, push, tag_reads_args, PushArgs, PushArgsOwned, PushIsolationLevel, }; use crate::types::RunJobQuery; @@ -50,7 +50,27 @@ use crate::types::RunJobQuery; // Tag / license checks // --------------------------------------------------------------------------- +/// `args` must be the ones the job is pushed with, see [`windmill_queue::check_tag_available_for_push`]. pub async fn check_tag_available_for_workspace( + db: &DB, + w_id: &str, + tag: &Option, + args: &PushArgs<'_>, + authed: &ApiAuthed, +) -> error::Result<()> { + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { + let tags = get_scope_tags(authed); + // Job-aware: a WM_TOKEN running as a superadmin must not unlock restricted tags. + let is_super_admin = windmill_api_auth::is_super_admin_authed(db, authed).await?; + check_tag_available_for_push(db, w_id, tag, args, is_super_admin, tags).await + } else { + Ok(()) + } +} + +/// [`check_tag_available_for_workspace`] for a tag filled in from values nothing can check yet, +/// such as a flow's preprocessor output: only an entry spelled exactly like it admits it. +pub async fn check_tag_as_written_available_for_workspace( db: &DB, w_id: &str, tag: &Option, @@ -60,7 +80,16 @@ pub async fn check_tag_available_for_workspace( let tags = get_scope_tags(authed); // Job-aware: a WM_TOKEN running as a superadmin must not unlock restricted tags. let is_super_admin = windmill_api_auth::is_super_admin_authed(db, authed).await?; - check_tag_available_for_workspace_internal(db, w_id, tag, is_super_admin, tags).await + check_tag_available_for_workspace_internal( + db, + w_id, + tag, + None, + std::future::ready(w_id.to_string()), + is_super_admin, + tags, + ) + .await } else { Ok(()) } @@ -760,9 +789,25 @@ pub async fn run_flow<'c>( .. } = flow_version_info; - let tag = run_query.tag.clone().or(tag); + let flow_tag = tag; + let tag = run_query.tag.clone().or(flow_tag.clone()); + let push_args = PushArgs { args: &args.args, extra: args.extra }; + let apply_preprocessor = + !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + if !apply_preprocessor || run_query.tag.is_some() { + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + } + // `push` drops the tag of a flow whose preprocessor runs, and the worker then puts the flow on + // its own tag filled in from the preprocessor's output, after this caller's identity is gone. + // So that tag is the one to check, and an `$args[...]` in it is judged as written. + if apply_preprocessor { + if flow_tag.as_deref().is_some_and(tag_reads_args) { + check_tag_as_written_available_for_workspace(&db, &w_id, &flow_tag, &authed).await?; + } else { + check_tag_available_for_workspace(&db, &w_id, &flow_tag, &push_args, &authed).await?; + } + } let scheduled_for = run_query.get_scheduled_for(&db).await?; let return_tx = tx_o.is_some(); @@ -798,11 +843,10 @@ pub async fn run_flow<'c>( path: flow_path.to_string(), dedicated_worker, version, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), + apply_preprocessor, labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -988,7 +1032,8 @@ pub async fn push_script_job_by_path_into_queue<'c>( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let return_tx = tx_o.is_some(); @@ -1020,7 +1065,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 432af1bef0..edd2040143 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -4450,9 +4450,11 @@ async fn execute_component( // — like `/jobs/run/preview` — confine it to worker tags the caller may use // (a `if_jobs:filter_tags`-restricted token must not escape its filter). // `is_preview` implies an authed caller (the guard above returns otherwise). + let push_args = PushArgs { args: &args.args, extra: args.extra }; if is_preview { if let Some(authed) = opt_authed.as_ref() { - crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, authed).await?; + crate::jobs::check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, authed) + .await?; } } // Identity is already resolved to the requesting user in preview mode (the @@ -4483,7 +4485,7 @@ async fn execute_component( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, &username, email, permissioned_as, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 18ddc0b787..052c9df4bb 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7226,7 +7226,10 @@ pub async fn run_workflow_as_code( ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &authed).await?; + let mut extra = HashMap::new(); + extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); + let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; + check_tag_available_for_workspace(&db, &w_id, &run_query.tag, &args, &authed).await?; check_scopes(&authed, || format!("jobs:run"))?; if !is_valid_entrypoint_name(&entrypoint) { @@ -7320,10 +7323,6 @@ pub async fn run_workflow_as_code( i += 1; } - let mut extra = HashMap::new(); - extra.insert(ENTRYPOINT_OVERRIDE.to_string(), to_raw_value(&entrypoint)); - - let args = PushArgs { args: &task.args.unwrap_or_else(HashMap::new), extra: Some(extra) }; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag).or(Some(job.tag)); @@ -7360,7 +7359,7 @@ pub async fn run_workflow_as_code( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + args, authed.display_username(), email, permissioned_as, @@ -7646,7 +7645,8 @@ pub async fn run_wait_result_job_by_path_get( .await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_authed.as_ref() { @@ -7670,7 +7670,7 @@ pub async fn run_wait_result_job_by_path_get( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -7791,7 +7791,8 @@ pub async fn run_wait_result_script_by_path_internal( .await?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(on_behalf_of) = on_behalf_of.as_ref() { @@ -7815,7 +7816,7 @@ pub async fn run_wait_result_script_by_path_internal( tx, &w_id, job_payload, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -7904,7 +7905,8 @@ pub async fn run_wait_result_script_by_hash( check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(obo) = on_behalf_of.as_ref() { ( @@ -7940,7 +7942,7 @@ pub async fn run_wait_result_script_by_hash( && has_preprocessor.unwrap_or(false), labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, @@ -8465,7 +8467,6 @@ async fn run_preview_script( require_path_read_access_for_preview(&authed, &preview.path)?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let preview_args = preview.args.unwrap_or_default(); @@ -8484,6 +8485,7 @@ async fn run_preview_script( } let extra = if extra.is_empty() { None } else { Some(extra) }; let push_args = PushArgs { extra, args: &preview_args }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (uuid, tx) = push( &db, @@ -8845,7 +8847,6 @@ async fn run_bundle_preview_script( let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(preview.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let ltx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let args = preview.args.unwrap_or_default(); @@ -8860,6 +8861,7 @@ async fn run_bundle_preview_script( m }); let push_args = PushArgs { extra, args: &args }; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; is_tar = match preview.kind { Some(PreviewKind::Tarbundle) => true, @@ -9511,7 +9513,6 @@ async fn run_preview_flow_job( require_path_read_access_for_preview(&authed, &raw_flow.path)?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(raw_flow.tag.clone()); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false); @@ -9529,6 +9530,8 @@ async fn run_preview_flow_job( to_raw_value(temp_script_refs), ); } + check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&flow_args), &authed) + .await?; let (uuid, mut tx) = push( &db, @@ -9774,11 +9777,6 @@ async fn run_dynamic_select( } } - // Same tag-permission gate a normal run gets (run_flow / push_script_job_by_path_into_queue): - // a caller allowed to read the flow must still be allowed to use its worker tag. No-op for - // inline (tag is None); the script branch checked this inside its helper and returned above. - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - // Invoke the dyn-select entrypoint instead of `main`. let mut args = request.args.unwrap_or_default(); args.insert( @@ -9786,6 +9784,11 @@ async fn run_dynamic_select( serde_json::value::to_raw_value(&request.entrypoint_function)?, ); + // Same tag-permission gate a normal run gets (run_flow / push_script_job_by_path_into_queue): + // a caller allowed to read the flow must still be allowed to use its worker tag. No-op for + // inline (tag is None); the script branch checked this inside its helper and returned above. + check_tag_available_for_workspace(&db, &w_id, &tag, &PushArgs::from(&args), &authed).await?; + let scheduled_for = run_query.get_scheduled_for(&db).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); @@ -9920,8 +9923,9 @@ pub async fn run_job_by_hash_inner( } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); + let push_args = PushArgs { args: &args.args, extra: args.extra }; - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let (email, permissioned_as, push_authed, tx) = if let Some(obo) = on_behalf_of.as_ref() { ( @@ -9957,7 +9961,7 @@ pub async fn run_job_by_hash_inner( && has_preprocessor.unwrap_or(false), labels, }, - PushArgs { args: &args.args, extra: args.extra }, + push_args, authed.display_username(), email, permissioned_as, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 6f3248f3fb..f2b0e16dac 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -17,7 +17,7 @@ use crate::{ get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::{StripPath, HTTP_CLIENT}, - worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, + worker::{custom_tag_matches, to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, WINDMILL_DIR}, workspaces::workspace_with_fork_ancestors, FlowVersionInfo, ScriptHashInfo, Tag, }; @@ -337,32 +337,54 @@ lazy_static::lazy_static! { // `is_super_admin` is passed in (not derived from an email here) so callers can // make it job-token-aware: a job's WM_TOKEN must never count as superadmin // (GHSA-hfh4-cx4h-3fcr). See `is_super_admin_authed` at the request wrapper. +// +// Custom tags judge `resolved_tag`, the queue the job actually lands on, and `tag` only as the +// job's author wrote it: a dynamic tag names any queue its values do. Callers go through +// `windmill_queue::check_tag_available_for_push`, which resolves it the way `push` does. +// `resolved_tag` is `None` when the values are not known yet: only an entry spelled like `tag` +// admits it then. `tag_workspace` resolves to what the job's `$workspace` stands for. pub async fn check_tag_available_for_workspace_internal( db: &DB, w_id: &str, tag: &str, + resolved_tag: Option<&str>, + tag_workspace: impl Future, is_super_admin: bool, scope_tags: Option>, ) -> error::Result<()> { - let mut is_tag_in_scope_tags = None; - let mut is_tag_in_workspace_custom_tags = false; - - if let Some(scope_tags) = scope_tags.as_ref() { - is_tag_in_scope_tags = Some(scope_tags.contains(&tag)); - } + let is_tag_in_scope_tags = scope_tags.as_ref().map(|scope_tags| { + scope_tags.contains(&tag) || resolved_tag.is_some_and(|r| scope_tags.contains(&r)) + }); let custom_tags_per_w = CUSTOM_TAGS_PER_WORKSPACE.load(); - if custom_tags_per_w.global.contains(&tag.to_string()) { - is_tag_in_workspace_custom_tags = true; - } else if let Some(specific_tag) = custom_tags_per_w.specific.get(tag) { - // Only a fork-scoped tag can match through the lineage, so every other tag keeps the - // ancestor lookup off the push path entirely. - let chain = if specific_tag.is_fork_scoped() { - workspace_with_fork_ancestors(db, w_id).await? - } else { - vec![w_id.to_string()] - }; - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); + // Only an entry reading `$workspace` needs it, so every other tag keeps its lookup off the + // push path. + let tag_workspace = if resolved_tag.is_some() + && custom_tags_per_w + .global + .iter() + .any(|t| t.contains("$workspace")) + { + tag_workspace.await + } else { + w_id.to_string() + }; + // A job whose tag is written exactly as an entry resolves inside what that entry admits, which + // is the only way in for an entry `custom_tag_matches` cannot turn into a pattern. + let mut is_tag_in_workspace_custom_tags = custom_tags_per_w.global.iter().any(|entry| { + entry == tag || resolved_tag.is_some_and(|r| custom_tag_matches(entry, r, &tag_workspace)) + }); + if !is_tag_in_workspace_custom_tags { + if let Some(specific_tag) = custom_tags_per_w.specific.get(resolved_tag.unwrap_or(tag)) { + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if specific_tag.is_fork_scoped() { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); + } } match is_tag_in_scope_tags { @@ -375,6 +397,12 @@ pub async fn check_tag_available_for_workspace_internal( } if !is_super_admin { + let tag = match resolved_tag { + Some(resolved_tag) if resolved_tag != tag => { + format!("{tag} (resolved to {resolved_tag})") + } + _ => tag.to_string(), + }; if scope_tags.is_some() && is_tag_in_scope_tags.is_some() { return Err(Error::BadRequest(format!( "Tag {tag} is not available in your scope" @@ -385,7 +413,7 @@ pub async fn check_tag_available_for_workspace_internal( return Err(Error::BadRequest(format!("{tag} is not available to you"))); } else { return Err(error::Error::BadRequest(format!( - "Only super admins are allowed to use tags that are not included in the allowed CUSTOM_TAGS: {:?}", + "Tag {tag} is not included in the allowed CUSTOM_TAGS, which only super admins can go beyond: {:?}", custom_tags_per_w ))); } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index c742a3efcb..548dbe07b4 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -104,6 +104,84 @@ impl CustomTags { } } +/// Whether a job whose tag resolved to `tag` falls under the custom tag `entry`. An entry holding +/// placeholders is a pattern over resolved tags: `$args[...]` and `$flow_expr[...]` match any +/// text, since whoever pushes the job picks their values, and `$workspace` matches +/// `tag_workspace`, what the job's own `$workspace` resolves to. The text around them must match +/// as written: it is what confines `gpu-$args[size]` to the `gpu-` tags. +/// +/// Placeholders whose values are tied, a repeat or one reading inside another, make an entry no +/// wildcard pattern describes (`t-$args[id]-$args[id]` never resolves to `t-a-b`). Such an entry +/// matches nothing here: it admits only a job whose tag is written exactly as the entry is. +pub fn custom_tag_matches(entry: &str, tag: &str, tag_workspace: &str) -> bool { + if !entry.contains('$') { + return entry == tag; + } + let dynamic: Vec<&str> = CUSTOM_TAG_PLACEHOLDER + .find_iter(entry) + .map(|m| m.as_str()) + .filter(|p| *p != "$workspace") + .collect(); + let tied = dynamic + .iter() + .enumerate() + .any(|(i, a)| dynamic[i + 1..].iter().any(|b| placeholders_tied(a, b))); + if tied { + return false; + } + // The literal runs between wildcards, with `$workspace` substituted. + let mut pieces = vec![]; + let mut current = String::new(); + let mut last_end = 0; + for m in CUSTOM_TAG_PLACEHOLDER.find_iter(entry) { + current.push_str(&entry[last_end..m.start()]); + if m.as_str() == "$workspace" { + current.push_str(tag_workspace); + } else { + pieces.push(std::mem::take(&mut current)); + } + last_end = m.end(); + } + current.push_str(&entry[last_end..]); + pieces.push(current); + + let [first, rest @ ..] = pieces.as_slice() else { + return false; + }; + let Some((last, middle)) = rest.split_last() else { + return tag == first; + }; + let Some(mut inner) = tag + .strip_prefix(first.as_str()) + .and_then(|t| t.strip_suffix(last.as_str())) + else { + return false; + }; + for piece in middle { + match inner.find(piece.as_str()) { + Some(i) => inner = &inner[i + piece.len()..], + None => return false, + } + } + true +} + +/// Whether two `$args[...]` / `$flow_expr[...]` placeholders read the same value, or one reads a +/// value inside the other's. +fn placeholders_tied(a: &str, b: &str) -> bool { + let (Some((kind_a, path_a)), Some((kind_b, path_b))) = (a.split_once('['), b.split_once('[')) + else { + return false; + }; + let (path_a, path_b) = (path_a.trim_end_matches(']'), path_b.trim_end_matches(']')); + let inside = |outer: &str, inner: &str| { + inner + .strip_prefix(outer) + .is_some_and(|rest| rest.starts_with('.')) + }; + kind_a == kind_b && (path_a == path_b || inside(path_a, path_b) || inside(path_b, path_a)) +} + /// Marker suffixed to a workspace id inside a custom tag's scope (`mytag(prod*)`) to extend the /// entry to that workspace's forks. `*` cannot appear in a workspace id (the `proper_id` check /// constraint restricts them to `^\w+(-\w+)*$`), so it can never collide with a real id. @@ -476,6 +554,9 @@ lazy_static::lazy_static! { // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); + // The placeholders a job's tag is resolved from when it is pushed, see [`custom_tag_matches`]. + static ref CUSTOM_TAG_PLACEHOLDER: Regex = Regex::new(r"\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\]").unwrap(); + pub static ref DISABLE_BUNDLING: bool = std::env::var("DISABLE_BUNDLING") .ok() .and_then(|x| x.parse::().ok()) @@ -3070,6 +3151,40 @@ mod tests { assert_eq!(result, vec!["foo", "legacy(^ws1^ws2)", "urgent(ws1+ws2)"]); } + #[test] + fn test_custom_tag_matches_resolved_tags() { + let matches = |entry, tag| custom_tag_matches(entry, tag, "ws1"); + + assert!(matches("gpu", "gpu")); + assert!(!matches("gpu", "gpu-large")); + + // The text around a placeholder fences what its value can make of the tag. + assert!(matches("gpu-$args[size]", "gpu-large")); + assert!(matches("gpu-$flow_expr[results.a.size]", "gpu-")); + assert!(!matches("gpu-$args[size]", "prod")); + assert!(!matches("gpu-$args[size]", "xgpu-large")); + assert!(matches("$args[region]-gpu", "eu-gpu")); + assert!(!matches("$args[region]-gpu", "eu-gpu-x")); + assert!(matches("a-$args[x]-b-$args[y]-c", "a-1-b-2-c")); + assert!(!matches("a-$args[x]-b-$args[y]-c", "a-1-c")); + assert!(!matches("ab$args[x]ba", "aba")); + + // A bare placeholder admits every tag. + assert!(matches("$flow_expr[results.a.tag]", "anything")); + + // Tied placeholders are no pattern: their entry admits only its own text. + assert!(!matches("t-$args[id]-$args[id]", "t-a-a")); + assert!(!matches("t-$args[a]-$args[a.b]", "t-x-y")); + assert!(matches("t-$args[a.x]-$args[a.y]", "t-x-y")); + assert!(matches("t-$args[id]-$flow_expr[flow_input.id]", "t-x-y")); + + // `$workspace` stands for the job's own workspace only. + assert!(matches("tag-$workspace", "tag-ws1")); + assert!(!matches("tag-$workspace", "tag-ws2")); + assert!(matches("$workspace-$args[size]", "ws1-large")); + assert!(!matches("$workspace-$args[size]", "ws2-large")); + } + #[test] fn test_fork_marker_parses_and_round_trips() { let tags = CustomTags::from(vec![ diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 429701f7e0..9eae01c476 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4696,6 +4696,72 @@ pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> Strin } } +/// The queue an explicit `tag` sends a job pushed with `args` to, or `None` when `push` drops the +/// tag and the job runs on its default one. +pub async fn resolve_push_tag( + tag: &str, + args: &PushArgs<'_>, + workspace_id: &str, + db: &DB, +) -> Option { + // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here + // was pushed with no flow state to read (a step test, a dependency job) and would name a + // queue no worker serves: the job runs on its default tag instead. + if tag.is_empty() || tag_reads_flow_expr(tag) { + return None; + } + // `$workspace` must resolve the same way the default tags do, or an explicit tag and a default + // tag from the same workspace address two different worker pools. Resolving costs a lookup, + // so pay it only for tags that actually interpolate `$workspace`. + let tag_ws = if tag.contains("$workspace") { + crate::tags::tag_workspace_id(workspace_id, db).await + } else { + workspace_id.to_string() + }; + Some(interpolate_args(tag.to_string(), args, &tag_ws)) +} + +/// Refuses a `tag` the caller chose that the instance's custom tags do not let `w_id` use, +/// judging the queue it resolves to. `args` must be the ones the job is pushed with: resolving +/// with any others checks a queue the job does not land on. +pub async fn check_tag_available_for_push( + db: &DB, + w_id: &str, + tag: &str, + args: &PushArgs<'_>, + is_super_admin: bool, + scope_tags: Option>, +) -> Result<(), Error> { + check_tag_written_as_available_for_push(db, w_id, tag, tag, args, is_super_admin, scope_tags) + .await +} + +/// [`check_tag_available_for_push`] for a `tag` the flow runtime already partly resolved from +/// `written_tag`, the step's tag as its author wrote it. +pub async fn check_tag_written_as_available_for_push( + db: &DB, + w_id: &str, + written_tag: &str, + tag: &str, + args: &PushArgs<'_>, + is_super_admin: bool, + scope_tags: Option>, +) -> Result<(), Error> { + let Some(resolved_tag) = resolve_push_tag(tag, args, w_id, db).await else { + return Ok(()); + }; + windmill_common::jobs::check_tag_available_for_workspace_internal( + db, + w_id, + written_tag, + Some(&resolved_tag), + crate::tags::tag_workspace_id(w_id, db), + is_super_admin, + scope_tags, + ) + .await +} + pub fn fullpath_with_workspace( workspace_id: &str, script_path: Option<&String>, @@ -6535,23 +6601,9 @@ async fn push_inner<'c, 'd>( ); windmill_common::worker::dedicated_worker_tag(workspace_id, &full_path) } else { - // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here - // was pushed with no flow state to read (a step test, a dependency job) and would name a - // queue no worker serves: the job runs on its default tag instead. - if tag == Some("".to_string()) || tag.as_deref().is_some_and(tag_reads_flow_expr) { - tag = None; - } - - // `$workspace` must resolve the same way the default tags below do, or an explicit tag and - // a default tag from the same workspace address two different worker pools. Resolving costs - // a lookup, so pay it only for tags that actually interpolate `$workspace`. let interpolated_tag = match tag { + Some(x) => resolve_push_tag(&x, &args, workspace_id, db).await, None => None, - Some(x) if x.contains("$workspace") => { - let tag_ws = crate::tags::tag_workspace_id(&workspace_id, db).await; - Some(interpolate_args(x, &args, &tag_ws)) - } - Some(x) => Some(interpolate_args(x, &args, workspace_id)), }; let effective_ws = per_workspace_tag(&workspace_id, db).await; diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 5d5cf005fc..7a9a36ff19 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -19,7 +19,6 @@ use windmill_common::ee_oss::LICENSE_KEY_VALID; use windmill_common::flows::Retry; use windmill_common::get_flow_version_info_from_version; use windmill_common::get_latest_flow_version_id_for_path; -use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::jobs::JobPayload; use windmill_common::jobs::JobTriggerKind; use windmill_common::jobs::OnBehalfOf; @@ -511,14 +510,15 @@ pub async fn push_scheduled_job<'c>( if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { let is_super_admin = windmill_common::auth::is_super_admin_email(db, &email).await?; - check_tag_available_for_workspace_internal( + crate::check_tag_available_for_push( db, &schedule.workspace_id, &tag, + &crate::PushArgs::from(&args), is_super_admin, None, // no token for schedules so no scopes so no scope_tags ) - .warn_after_seconds_with_sql(1, "check_tag_available_for_workspace_internal".to_string()) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) .await?; } diff --git a/backend/windmill-trigger/src/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs index 6a37bd0da1..7e47613d1a 100644 --- a/backend/windmill-trigger/src/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -880,7 +880,8 @@ async fn trigger_script_with_retry_and_error_handler<'c>( delete_after_secs, ); - check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; + let push_args = PushArgs { args: &args.args, extra: args.extra }; + check_tag_available_for_workspace(&db, &workspace_id, &tag, &push_args, &authed).await?; let return_tx = tx_o.is_some(); @@ -907,8 +908,6 @@ async fn trigger_script_with_retry_and_error_handler<'c>( ) }; - let push_args = PushArgs { args: &args.args, extra: args.extra }; - let retryable_job_payload = match job_payload { JobPayload::ScriptHash { hash, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7ed7b8d5da..841ea8861a 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2716,18 +2716,15 @@ pub async fn handle_wac_v2_output( ) -> error::Result> { use crate::wac_executor::{ load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, - wac_cancelled_mid_segment, WacOutput, WacPark, + wac_cancelled_mid_segment, WacOutput, WacPark, WacStepDispatch, }; use serde_json::Value; use windmill_common::get_latest_flow_version_info_for_path; - use windmill_common::jobs::{ - check_tag_available_for_workspace_internal, script_path_to_payload, JobKind, JobPayload, - RawCode, - }; + use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode}; use windmill_common::runnable_settings::{ ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, }; - use windmill_queue::{push, PushArgs, PushIsolationLevel}; + use windmill_queue::{check_tag_available_for_push, push, PushArgs, PushIsolationLevel}; let output = parse_wac_output(&result)?; @@ -2970,6 +2967,18 @@ pub async fn handle_wac_v2_output( email: String, permissioned_as: String, } + // The input a task with a runnable of its own is pushed with; the parent re-run gets + // the parent's. Both the tag check and the push build it here, so the check resolves + // `$args[...]` against what the child actually receives. + let own_args = |step: &WacStepDispatch| -> HashMap> { + step.args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect() + }; let mut children: Vec = Vec::with_capacity(num_steps); for step in &steps { let (runnable, on_behalf_of) = match step.dispatch_type.as_str() { @@ -3038,17 +3047,19 @@ pub async fn handle_wac_v2_output( { let is_super_admin = windmill_common::auth::is_super_admin_email(db, &email).await?; - check_tag_available_for_workspace_internal( + let step_args = + (!matches!(runnable, ChildRunnable::Parent)).then(|| own_args(step)); + let args = + PushArgs { args: step_args.as_ref().unwrap_or(&parent_args), extra: None }; + check_tag_available_for_push( db, &job.workspace_id, tag, + &args, is_super_admin, None, ) - .warn_after_seconds_with_sql( - 1, - "check_tag_available_for_workspace_internal".to_string(), - ) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) .await .map_err(|e| match e { error::Error::BadRequest(msg) => error::Error::BadRequest(format!( @@ -3174,16 +3185,7 @@ pub async fn handle_wac_v2_output( }), ChildRunnable::Deployed(payload) => payload, }; - let step_args: Option>> = - is_external.then(|| { - step.args - .iter() - .map(|(k, v)| { - let raw = serde_json::value::to_raw_value(v).unwrap(); - (k.clone(), raw) - }) - .collect() - }); + let step_args = is_external.then(|| own_args(step)); let push_args = PushArgs { args: step_args.as_ref().unwrap_or(&parent_args), extra: None }; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index dbdba3e6ed..0ab4e2871c 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -44,8 +44,7 @@ use windmill_common::flow_status::{ }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - check_tag_available_for_workspace_internal, script_path_to_payload, JobKind, JobPayload, - OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::runnable_settings::{ ConcurrencySettingsWithCustom, DebouncingSettings, RunnableSettingsTrait, @@ -68,11 +67,12 @@ use windmill_common::{ }; use windmill_queue::schedule::get_schedule_opt; use windmill_queue::{ - add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job, - insert_concurrency_key_capped, interpolate_args, render_tag_path, - report_error_to_workspace_handler_or_critical_side_channel, tag_reads_args, - tag_reads_flow_expr, try_schedule_next_job, CanceledBy, FlowRunners, MiniCompletedJob, - MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, RE_FLOW_EXPR_TAG, + add_completed_job, add_completed_job_error, append_logs, + check_tag_written_as_available_for_push, get_mini_pulled_job, insert_concurrency_key_capped, + render_tag_path, report_error_to_workspace_handler_or_critical_side_channel, resolve_push_tag, + tag_reads_args, tag_reads_flow_expr, try_schedule_next_job, CanceledBy, FlowRunners, + MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, + RE_FLOW_EXPR_TAG, }; use windmill_audit::audit_oss::audit_log; @@ -1520,15 +1520,6 @@ pub async fn update_flow_status_after_job_completion_internal( .is_some_and(|ck| ck.contains("$args")) }); let require_args = concurrency_requires_args || has_debouncing; - let mut tag = tag_and_concurrency_key.as_ref().and_then(|x| x.tag.clone()); - // `$workspace` does not depend on the preprocessor's output, so it has to resolve even - // when nothing forced us to fetch args. Leaving it to the `$args` branch below writes a - // `$workspace`-only tag back verbatim, naming a queue no worker serves. - if let Some(t) = tag.as_ref().filter(|t| t.contains("$workspace")) { - let tag_ws = - windmill_queue::tags::tag_workspace_id(&flow_job.workspace_id, db).await; - tag = Some(t.replace("$workspace", &tag_ws)); - } let concurrency_key = tag_and_concurrency_key .as_ref() .and_then(|x| x.concurrency_key.clone()); @@ -1576,10 +1567,6 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; } - if let Some(t) = tag { - // `$workspace` is already resolved above; this fills in `$args`. - tag = Some(interpolate_args(t, &args, &flow_job.workspace_id)); - } } else if concurrent_limit.is_some() { insert_concurrency_key_capped( &flow_job.workspace_id, @@ -1594,6 +1581,23 @@ pub async fn update_flow_status_after_job_completion_internal( .await?; } + // Resolved as `push` resolves a tag, so a `$flow_expr[...]` or empty one comes out as + // `None` and the job keeps its current tag instead of queueing on the literal text. + let tag = match tag_and_concurrency_key + .as_ref() + .and_then(|x| x.tag.as_deref()) + { + // `run_flow` checked this tag for whoever ran the flow, an `$args[...]` in it as + // written since only this point knows its values: an entry admitting it as + // written admits every value here. + Some(t) => { + let no_args = HashMap::new(); + let args = PushArgs::from(fetched_args.as_ref().unwrap_or(&no_args)); + resolve_push_tag(t, &args, &flow_job.workspace_id, db).await + } + None => None, + }; + let scheduled_for: Option> = { #[cfg(feature = "private")] { @@ -4494,33 +4498,7 @@ async fn push_next_flow_job( ) }; - // Check tag availability for flow substeps to prevent abuse. - // Skip when the substep inherits the parent flow's tag: that tag was already - // validated at flow push time, and for dedicated flows it is the auto-generated - // `{workspace_id}:flow/{path}` tag which is never in CUSTOM_TAGS. - if let Some(tag_str) = tag - .as_deref() - .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) - { - // A step with its own on-behalf-of carries a cached dispatch address, up to one - // notify poll stale; accepted, see `get_email_from_permissioned_as`. - let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; - check_tag_available_for_workspace_internal( - db, - &flow_job.workspace_id, - tag_str, - is_super_admin, - None, // no token for flow substeps so no scopes so no scope_tags - ) - .warn_after_seconds_with_sql( - 1, - "check_tag_available_for_workspace_internal".to_string(), - ) - .await?; - } - - // Resolved only after the check: CUSTOM_TAGS allows the template, so its value may name - // any queue, as the value of an `$args[...]` tag does. + let written_tag = tag.clone(); let mut tag_err = None; let tag = match tag { Some(t) if err.is_none() && tag_reads_flow_expr(&t) => { @@ -4543,6 +4521,31 @@ async fn push_next_flow_job( }; let err = err.or(tag_err.as_ref()); + // Check tag availability for flow substeps to prevent abuse. It runs on the tag with its + // `$flow_expr[...]` filled in, as `push` fills in the rest, since that is what picks the + // queue. Skip when the substep inherits the parent flow's tag: that tag was already + // validated at flow push time, and for dedicated flows it is the auto-generated + // `{workspace_id}:flow/{path}` tag which is never in CUSTOM_TAGS. + if let Some(tag_str) = tag + .as_deref() + .filter(|t| !t.is_empty() && *t != flow_job.tag.as_str()) + { + // A step with its own on-behalf-of carries a cached dispatch address, up to one + // notify poll stale; accepted, see `get_email_from_permissioned_as`. + let is_super_admin = windmill_common::auth::is_super_admin_email(db, email).await?; + check_tag_written_as_available_for_push( + db, + &flow_job.workspace_id, + written_tag.as_deref().unwrap_or(tag_str), + tag_str, + &push_args, + is_super_admin, + None, // no token for flow substeps so no scopes so no scope_tags + ) + .warn_after_seconds_with_sql(1, "check_tag_available_for_push".to_string()) + .await?; + } + let evaluated_timeout = if let Some(timeout_transform) = &module.timeout { let ctx = get_transform_context(&flow_job, &previous_id, &status); diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index 33a51be87c..f385ff2b49 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -2,7 +2,8 @@ import { preventDefault, stopPropagation } from 'svelte/legacy' import { Button } from './common' - import { ExternalLink, Loader2, X } from 'lucide-svelte' + import { AlertTriangle, ExternalLink, Loader2, X } from 'lucide-svelte' + import Popover from './meltComponents/Popover.svelte' import { SettingService, WorkerService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { superadmin, devopsRole } from '$lib/stores' @@ -53,6 +54,43 @@ return matched ? { kind: matched[1], path: matched[2] } : undefined }) + // Mirrors custom_tag_matches in backend/windmill-common/src/worker.rs: a job's tag is judged on + // what it resolves to, and a placeholder in a custom tag stands for any text, unless two of them + // are tied (same kind, and the same path or one inside the other). A tied entry admits only a + // tag written exactly like it; an untied one is a pattern fenced only by its fixed text. + function placeholdersOf(entry: string) { + return [...entry.matchAll(new RegExp(dynamicTagRegex.source, 'g'))].map((m) => ({ + kind: m[1], + path: m[2], + start: m.index, + end: m.index + m[0].length + })) + } + function isTied(entry: string) { + const placeholders = placeholdersOf(entry) + return placeholders.some((a, i) => + placeholders + .slice(i + 1) + .some( + (b) => + a.kind == b.kind && + (a.path == b.path || a.path.startsWith(b.path + '.') || b.path.startsWith(a.path + '.')) + ) + ) + } + // With nothing fixed at its start or its end, a pattern reaches almost any tag. + function reachesAnyTag(entry: string) { + const placeholders = placeholdersOf(entry) + return ( + placeholders.length > 0 && + !isTied(entry) && + placeholders[0].start == 0 && + placeholders[placeholders.length - 1].end == entry.length + ) + } + let dynamicTagTied = $derived(isTied(newTag.trim())) + let dynamicTagReachesAnyTag = $derived(reachesAnyTag(newTag.trim())) + let extractedCustomTag = $derived.by(() => { let r = newTag.trim() if (r == '') return undefined @@ -126,6 +164,12 @@ +{#snippet reachesAnyTagWarning()} + Nothing fixed at its start or its end, so it allows anyone to reach almost any tag. We highly + recommend a prefix or a suffix, ideally both, to limit its reach: for instance + gpu-$args[size] or gpu-$args[size]-eu. +{/snippet} +
{customTag} + {#if reachesAnyTag(customTag)} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+ {@render reachesAnyTagWarning()} +
+ {/snippet} + + {/if} {#if tagEditor}
{/if} + {#if dynamicTagTied} +
+ Its placeholders read the same value, or one reads inside the other, so it allows only + jobs whose tag is written exactly like this +
+ {:else if dynamicTagReachesAnyTag} +
+ {@render reachesAnyTagWarning()} +
+ {:else if dynamicTag} +
+ Allows any tag it resolves to, with the text around the placeholder as written +
+ {/if} {/if} {/if} @@ -269,6 +344,11 @@ 
a
is the step id, or 
$flow_expr[flow_input.a.b.c]
and 
$flow_expr[flow_env.a.b.c]
. +
{#if variant !== 'drawer'}
{/if} + A dynamic tag is checked on the tag it resolves to: it is allowed when that tag is listed here, + or fits a listed dynamic tag such as + 
gpu-$args[size]
, which allows any tag starting with + 
gpu-
. {/if} diff --git a/frontend/src/lib/components/WorkerTagSelect.svelte b/frontend/src/lib/components/WorkerTagSelect.svelte index 624aeb1bcd..503c8f7f66 100644 --- a/frontend/src/lib/components/WorkerTagSelect.svelte +++ b/frontend/src/lib/components/WorkerTagSelect.svelte @@ -92,6 +92,8 @@ } loading = false } + // A dynamic tag is filled in when the job runs, so no worker can be looked up for its text. + const dynamicTagRegex = /\$(workspace|args\[|flow_expr\[)/ let items = $derived([ // ...(tag ? ['reset to default'] : [nullTag ? `default: ${nullTag}` : '']), ...(tag && tag != '' && !(currentTags ?? []).includes(tag) ? [tag] : []), @@ -99,23 +101,25 @@ ]) let lastCheck: number | undefined = undefined - async function loadTagsToWorkerExists(tags: string[]) { - if (lastCheck && Date.now() - lastCheck < 5000) { - return - } + // Each run reads the list as it is then, so a tag typed in since the last run is looked up + // too, and a run too soon after the last one is deferred rather than dropped. + async function loadTagsToWorkerExists() { if (timeout) { clearTimeout(timeout) } + const wait = lastCheck ? 5000 - (Date.now() - lastCheck) : 0 + if (wait > 0) { + timeout = setTimeout(loadTagsToWorkerExists, wait) + return + } if (open) { tagsToWorkerExists = await WorkerService.existsWorkersWithTags({ - tags: tags.join(','), + tags: items.filter((t) => !dynamicTagRegex.test(t)).join(','), workspace: effectiveWorkspace }) lastCheck = Date.now() if (visible) { - timeout = setTimeout(() => { - loadTagsToWorkerExists(tags) - }, 5000) + timeout = setTimeout(loadTagsToWorkerExists, 5000) } } } @@ -136,7 +140,7 @@ $effect(() => { if (currentTags && open) { - loadTagsToWorkerExists(currentTags) + loadTagsToWorkerExists() } }) @@ -144,7 +148,8 @@ {#snippet startSnippet({ item })} - {#if tagsToWorkerExists} + + {#if tagsToWorkerExists && !item.__is_create && item.value in tagsToWorkerExists} {#if tagsToWorkerExists[item.value]} {#snippet text()} @@ -162,7 +167,7 @@ {/if} {/if} {/snippet} -
+
{#if !noLabel}
{placeholder ?? 'tag'}
{/if} @@ -176,22 +181,30 @@ placeholder={nullTag ? nullTag : (placeholder ?? 'lang default')} items={safeSelectItems(items)} bind:value={() => tag, (value) => ((tag = value), dispatch('change', value))} + onCreateItem={(value) => ((tag = value), dispatch('change', value))} + createText="Press Enter to use this tag" {startSnippet} bottomSnippet={refreshAll} />
{#snippet refreshAll()} - +
+ + $workspace, $args[…] and $flow_expr[…] are filled in when + the job runs, and must land on an allowed tag + + +
{/snippet} diff --git a/frontend/src/lib/components/select/GenericDropdown.svelte b/frontend/src/lib/components/select/GenericDropdown.svelte index 78a88a6457..a2bd6a044b 100644 --- a/frontend/src/lib/components/select/GenericDropdown.svelte +++ b/frontend/src/lib/components/select/GenericDropdown.svelte @@ -53,6 +53,9 @@ const spaceAbove = inputR.y const isBelow = fitsBelow || spaceBelow >= spaceAbove let [x, y] = disablePortal ? [0, 0] : [inputR.x, inputR.y] + // The list grows to its widest item, so one wider than the room right of its input + // would run off screen: slide it left instead. + if (!disablePortal) x = Math.max(0, Math.min(x, window.innerWidth - listR.width)) if (isBelow) return { width: inputR.width, height: listR.height, x: x, y: y + inputR.height, isBelow } else { From 46206a2787cbdd8b698c5db2ec22468787868d52 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Tue, 22 Sep 2026 09:42:36 +0200 Subject: [PATCH 08/34] refactor(ai-chat): assemble the global prompt and tools in one place (#11252) * refactor(ai-chat): assemble the global prompt and tools in one place Co-Authored-By: Claude Opus 5 (1M context) * refactor(ai-chat): derive assembly opts, harden omission tests Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/AIChatManager.svelte.ts | 88 ++++++------------- .../chat/global/globalAssembly.test.ts | 63 +++++++++++++ .../copilot/chat/global/globalAssembly.ts | 46 ++++++++++ 3 files changed, 137 insertions(+), 60 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/global/globalAssembly.ts diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 922f64ad93..c39325c84c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -125,27 +125,25 @@ import { } from '$lib/aiStore' import type { WorkspaceMutationTarget } from './workspaceTools' import { - globalToolsFor, loadWorkspaceSkills, - prepareGlobalSystemMessage, resolveGlobalPromptIdentity, type GlobalPromptIdentity, prepareGlobalUserMessage, type AiSkillListItem, type ChatCommandItem, type SessionPromptContext, - getSessionContextPromptSection, type GlobalToolHelpers, type GlobalActivePreviewContext } from './global/core' +import { + assembleGlobalSystemMessage, + assembleGlobalTools, + type GlobalAssemblyOpts +} from './global/globalAssembly' import { formatChatJobCompletion } from './datatableTools' import { isGlobalAiEnabled } from './global/gate' -import { createMcpTools, loadMcpServers, type McpServer } from './global/mcpTools' -import { - pipelineTools, - getPipelinePromptSection, - type PipelineAIChatHelpers -} from './pipeline/core' +import { loadMcpServers, type McpServer } from './global/mcpTools' +import type { PipelineAIChatHelpers } from './pipeline/core' import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userScopedStorage' import { getLocalSetting, storeLocalSetting } from '$lib/utils' import { AttachedFilesStore } from './files/attachedFiles.svelte' @@ -2408,27 +2406,11 @@ export class AIChatManager implements ChatViewHost { } } - // Fetch the workspace's AI skills and, if GLOBAL mode is still active, rebuild - // the system message so the next chat-loop iteration advertises them. Ignore - // stale resolves so workspace changes cannot overwrite newer skills. - // Build the global-mode system message, tools, and helpers, layering on the - // pipeline surface when a /pipeline editor has registered helpers. Centralized - // so changeMode, refreshGlobalSkills, and setPipelineHelpers stay consistent — - // each rebuild would otherwise drop the pipeline augmentation the others added. - // // Public because it is purely local, unlike `changeMode(GLOBAL)`, which also // fires the three network refreshes. configureGlobalMode = () => { - const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { - previewTools: this.isSessionChat, - user: this.globalIdentity, - skills: this.globalSkills, - mcpServers: this.mcpServers - }) - const sessionCtx = this.sessionContextResolver?.() - if (sessionCtx) { - systemMessage.content += getSessionContextPromptSection(sessionCtx) - } + const pipeline = this.pipelineAiChatHelpers + const opts = this.globalAssemblyOpts() const baseHelpers: GlobalToolHelpers = { // A session targets its own fixed (possibly forked) workspace, so capture it for // permission gating. The global side-panel chat follows the live navigation @@ -2458,24 +2440,24 @@ export class AIChatManager implements ChatViewHost { this.rebuildGlobalSystemMessage() } } - const pipeline = this.pipelineAiChatHelpers - const mcpTools = createMcpTools(this.mcpServers) - if (pipeline) { - systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) - this.tools = [ - ...globalToolsFor({ sessionPreview: this.isSessionChat }), - ...pipelineTools, - ...mcpTools - ] - this.helpers = { ...baseHelpers, pipeline } - } else { - this.tools = [...globalToolsFor({ sessionPreview: this.isSessionChat }), ...mcpTools] - this.helpers = baseHelpers - } - this.systemMessage = systemMessage + this.tools = assembleGlobalTools(opts) + this.helpers = pipeline ? { ...baseHelpers, pipeline } : baseHelpers + this.systemMessage = assembleGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), opts) this.syncArtifactsSession() } + private globalAssemblyOpts = (): GlobalAssemblyOpts => ({ + previewTools: this.isSessionChat, + user: this.globalIdentity, + skills: this.globalSkills, + mcpServers: this.mcpServers, + sessionContext: this.sessionContextResolver?.(), + pipelineContext: this.pipelineAiChatHelpers?.getPipelineContext() + }) + + // Fetch the workspace's AI skills and, if GLOBAL mode is still active, rebuild + // the system message so the next chat-loop iteration advertises them. Ignore + // stale resolves so workspace changes cannot overwrite newer skills. refreshGlobalSkills = async (workspace = this.operatingWorkspace ?? '') => { const refreshId = ++this.globalSkillsRefreshId const skills = await loadWorkspaceSkills(workspace) @@ -2550,24 +2532,10 @@ export class AIChatManager implements ChatViewHost { if (this.mode !== AIMode.GLOBAL) { return } - const systemMessage = prepareGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), { - previewTools: this.isSessionChat, - user: this.globalIdentity, - skills: this.globalSkills, - mcpServers: this.mcpServers - }) - // Preserve the session-state and active pipeline-editor augmentations that - // configureGlobalMode adds — otherwise update_user_instructions (which calls - // this) would drop them mid-session. - const sessionCtx = this.sessionContextResolver?.() - if (sessionCtx) { - systemMessage.content += getSessionContextPromptSection(sessionCtx) - } - const pipeline = this.pipelineAiChatHelpers - if (pipeline) { - systemMessage.content += getPipelinePromptSection(pipeline.getPipelineContext()) - } - this.systemMessage = systemMessage + this.systemMessage = assembleGlobalSystemMessage( + getCustomPromptParts(AIMode.GLOBAL), + this.globalAssemblyOpts() + ) } private expandGlobalSkillCommand = (instructions: string): string => { diff --git a/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts b/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts new file mode 100644 index 0000000000..feffda425e --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/globalAssembly.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +import { getSessionContextPromptSection, type SessionPromptContext } from './core' +import { getPipelinePromptSection, type PipelineContext } from '../pipeline/core' +import { assembleGlobalSystemMessage, assembleGlobalTools } from './globalAssembly' + +const sessionContext: SessionPromptContext = { workspaceId: 'ws', parentWorkspaceId: 'parent' } +const pipelineContext: PipelineContext = { + folder: 'myfolder', + mode: 'edit', + nodes: [], + assets: [] +} + +describe('assembleGlobalTools', () => { + it('includes the pipeline tools only when a pipeline editor is active', () => { + const names = (opts: Parameters[0]) => + assembleGlobalTools(opts).map((t) => t.def.function.name) + expect(names({ pipelineContext })).toContain('get_pipeline_graph') + expect(names({})).not.toContain('get_pipeline_graph') + }) +}) + +describe('assembleGlobalSystemMessage', () => { + // The hazard this module exists to remove: every caller gets every section, so a + // mid-session rebuild (update_user_instructions) cannot silently drop one. + it('carries both optional sections when both contexts are given', () => { + const content = assembleGlobalSystemMessage(undefined, { + sessionContext, + pipelineContext + }).content as string + expect(content).toContain(getSessionContextPromptSection(sessionContext)) + expect(content).toContain(getPipelinePromptSection(pipelineContext)) + // Order is part of the assembled output, so it is pinned too. + expect(content.indexOf(getSessionContextPromptSection(sessionContext))).toBeLessThan( + content.indexOf(getPipelinePromptSection(pipelineContext)) + ) + }) + + it('omits them when the contexts are absent', () => { + const content = assembleGlobalSystemMessage(undefined, {}).content as string + // Each section's own heading, read from the code rather than retyped: a reword + // cannot quietly make these vacuous, and a heading is the one part every variant + // shares, so a stray section built from any other context still trips them. + const heading = (s: string) => s.trim().split('\n')[0] + expect(content).not.toContain(heading(getSessionContextPromptSection(sessionContext))) + expect(content).not.toContain(heading(getPipelinePromptSection(pipelineContext))) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts new file mode 100644 index 0000000000..21ba3643de --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/global/globalAssembly.ts @@ -0,0 +1,46 @@ +/** + * The whole assembled surface of a GLOBAL-mode chat — prompt sections and tools — + * in one place, so a new section cannot be added to one rebuild path and forgotten + * on another. Plan mode's decoration is not here: it is applied per request, later. + */ +import type { ChatCompletionSystemMessageParam } from 'openai/resources/index.mjs' +import type { Tool } from '../shared' +import { + getSessionContextPromptSection, + globalToolsFor, + prepareGlobalSystemMessage, + type SessionPromptContext +} from './core' +import { createMcpTools } from './mcpTools' +import { getPipelinePromptSection, pipelineTools, type PipelineContext } from '../pipeline/core' + +// Derived, not retyped: an option added to prepareGlobalSystemMessage is reachable +// here at once. Hand-listing the four would compile fine while leaving the new one +// unreachable from this path, and only the eval harness — which calls the builder +// directly — would get it. +export type GlobalAssemblyOpts = NonNullable[1]> & { + sessionContext?: SessionPromptContext + pipelineContext?: PipelineContext +} + +export function assembleGlobalSystemMessage( + instructions: Parameters[0], + opts: GlobalAssemblyOpts +): ChatCompletionSystemMessageParam { + const systemMessage = prepareGlobalSystemMessage(instructions, opts) + if (opts.sessionContext) { + systemMessage.content += getSessionContextPromptSection(opts.sessionContext) + } + if (opts.pipelineContext) { + systemMessage.content += getPipelinePromptSection(opts.pipelineContext) + } + return systemMessage +} + +export function assembleGlobalTools(opts: GlobalAssemblyOpts): Tool[] { + return [ + ...globalToolsFor({ sessionPreview: opts.previewTools ?? false }), + ...(opts.pipelineContext ? pipelineTools : []), + ...createMcpTools(opts.mcpServers ?? []) + ] +} From 58217006ece56bbaf0447af67cac487e72b928c7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 10:07:50 +0200 Subject: [PATCH 09/34] fix: stop cutting DeepSeek chat turns off mid-thought (#11282) * fix: stop cutting DeepSeek chat turns off mid-thought Co-Authored-By: Claude Opus 5 (1M context) * test: pin the Anthropic max_tokens stop as a turn failure Co-Authored-By: Claude Opus 5 (1M context) * fix: drop the send-failure prefix from the output-limit toast Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/AIChatManager.svelte.ts | 5 +++ .../components/copilot/chat/anthropic.test.ts | 32 +++++++++++++++++- .../lib/components/copilot/chat/anthropic.ts | 4 +++ .../components/copilot/chat/chatLoop.test.ts | 18 ++++++++++ .../lib/components/copilot/chat/chatLoop.ts | 6 ++++ .../copilot/chat/openai-responses.test.ts | 33 +++++++++++++++++++ .../copilot/chat/openai-responses.ts | 11 +++++++ .../copilot/chat/outputTokenLimit.ts | 12 +++++++ .../src/lib/components/copilot/lib.test.ts | 1 + .../components/copilot/lib.toolCalls.test.ts | 30 +++++++++++++++++ frontend/src/lib/components/copilot/lib.ts | 16 +++++++-- .../src/lib/components/copilot/modelConfig.ts | 8 +++-- 12 files changed, 170 insertions(+), 6 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/outputTokenLimit.ts diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index c39325c84c..871dead4bc 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -110,6 +110,7 @@ import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { closeInterruptedToolBatch, runChatLoop, truncateToToolPairedPrefix } from './chatLoop' +import { OutputTokenLimitError } from './outputTokenLimit' import { sanitizeToolCallArguments } from './toolCallArguments' import { billedTokens, normalizeContextUsage, type ChatTokenUsage } from './tokenUsage' import { logAiUsage } from '$lib/utils/aiUsageReporter' @@ -375,6 +376,10 @@ function isImageRejection(err: unknown, models: (string | undefined)[] = []): bo } function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean): string { + // The request went through; only its response was cut short. + if (err instanceof OutputTokenLimitError) { + return err.message + } const errorMessage = err instanceof Error ? err.message : typeof err === 'string' ? err : undefined const message = errorMessage diff --git a/frontend/src/lib/components/copilot/chat/anthropic.test.ts b/frontend/src/lib/components/copilot/chat/anthropic.test.ts index 1ac413973f..9f6b49a9e9 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.test.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it, vi } from 'vitest' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' -import { convertOpenAIToAnthropicMessages, partialWebSearchQuery } from './anthropic' +import { + convertOpenAIToAnthropicMessages, + parseAnthropicCompletion, + partialWebSearchQuery +} from './anthropic' +import { OutputTokenLimitError } from './outputTokenLimit' // anthropic.ts pulls in the chat client/registry layer at import time; the // converter under test is pure, so stub those side-effecting modules away. @@ -217,3 +222,28 @@ describe('partialWebSearchQuery', () => { expect(partialWebSearchQuery('{"query": ')).toBeUndefined() }) }) + +describe('parseAnthropicCompletion output token limit', () => { + it('fails a message that stopped at max_tokens instead of ending the turn', async () => { + const stream = { + on: () => undefined, + done: async () => undefined, + finalMessage: async () => ({ + content: [{ type: 'thinking', thinking: 'I need to plan the flow so each' }], + stop_reason: 'max_tokens', + usage: { input_tokens: 20000, output_tokens: 64000 } + }) + } + + const parsed = parseAnthropicCompletion( + stream as any, + { onNewToken: vi.fn(), onMessageEnd: vi.fn(), setToolStatus: vi.fn() } as any, + [], + [], + [], + {} + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 8d38f10916..a311d16990 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -24,6 +24,7 @@ import { type ToolCallbacks, type WebSearchSource } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import { anthropicUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' import { parseImageDataUrl } from './imageUtils' @@ -453,6 +454,9 @@ export async function parseAnthropicCompletion( return { shouldContinue: true, tokenUsage } } + if (finalMessage.stop_reason === 'max_tokens') { + throw new OutputTokenLimitError() + } return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 251c2ff858..144617779f 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -8,6 +8,7 @@ import { type ChatLoopConfig } from './chatLoop' import type { ReasoningProviderModel } from '../reasoningRegistry' +import { OutputTokenLimitError } from './outputTokenLimit' const mocks = vi.hoisted(() => ({ getCompletion: vi.fn(), @@ -540,6 +541,23 @@ describe('runChatLoop lastIterationUsage', () => { }) }) +describe('runChatLoop output token limit', () => { + beforeEach(() => { + vi.resetAllMocks() + mocks.resolveRequestReasoning.mockReturnValue(undefined) + }) + + it('fails a truncated Responses iteration instead of replaying it on the Completions API', async () => { + mocks.getOpenAIResponsesCompletion.mockResolvedValue({}) + mocks.parseOpenAIResponsesCompletion.mockRejectedValue(new OutputTokenLimitError()) + + await expect( + runChatLoop(createConfig({ workspace: `workspace-${randomUUID()}` })) + ).rejects.toBeInstanceOf(OutputTokenLimitError) + expect(mocks.getCompletion).not.toHaveBeenCalled() + }) +}) + // Builders for the message shapes the chat loop accumulates. const assistant = (content: string): ChatCompletionMessageParam => ({ role: 'assistant', content }) const assistantTools = (...ids: string[]): ChatCompletionMessageParam => ({ diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index 110e4f1154..5091d76af5 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -20,6 +20,7 @@ import { parseOpenAIResponsesCompletion } from './openai-responses' import type { Tool, ToolCallbacks } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import { sanitizeToolCallArguments } from './toolCallArguments' import { addChatTokenUsage, emptyChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -501,6 +502,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise ({ getProviderAndCompletionConfig: vi.fn(), @@ -153,3 +155,34 @@ describe('openAIWebSearchDetails', () => { ).toEqual({ query: undefined, sources: [{ url: 'https://a.dev' }] }) }) }) + +describe('parseOpenAIResponsesCompletion output token limit', () => { + it('fails a response the stream reports incomplete at max_output_tokens', async () => { + const handlers: Record void> = {} + // The SDK's final snapshot of an incomplete response still reads in_progress; + // only the response.incomplete event carries the reason. + const runner = { + on: (name: string, fn: (event: any) => void) => { + handlers[name] = fn + }, + done: async () => { + handlers['response.incomplete']?.({ + type: 'response.incomplete', + response: { incomplete_details: { reason: 'max_output_tokens' } } + }) + }, + finalResponse: async () => ({ status: 'in_progress', output: [] }) + } + + const parsed = parseOpenAIResponsesCompletion( + runner as any, + { onNewToken: vi.fn(), onMessageEnd: vi.fn(), setToolStatus: vi.fn() } as any, + [], + [], + [], + {} + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 4118eb492d..b216a08107 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -21,6 +21,7 @@ import { type ToolCallbacks, type WebSearchSource } from './shared' +import { OutputTokenLimitError } from './outputTokenLimit' import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' import type { AIProviderModel } from '$lib/gen' import { openAIResponsesUsageToChatTokenUsage, type ChatTokenUsage } from './tokenUsage' @@ -542,6 +543,13 @@ export async function parseOpenAIResponsesCompletion( }) }) + // The stream's final snapshot keeps the in-progress status, so an + // incomplete response is only visible through its own event. + let hitOutputTokenLimit = false + runner.on('response.incomplete', (event) => { + hitOutputTokenLimit = event.response.incomplete_details?.reason === 'max_output_tokens' + }) + // Handle errors runner.on('error', (err: OpenAIError | ResponseErrorEvent) => { currentStreamingTool = undefined @@ -600,6 +608,9 @@ export async function parseOpenAIResponsesCompletion( return { shouldContinue: true, tokenUsage } } + if (hitOutputTokenLimit) { + throw new OutputTokenLimitError() + } return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts new file mode 100644 index 0000000000..b1127a1db7 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts @@ -0,0 +1,12 @@ +/** The provider ended the response at the request's output token cap. A turn + * cut off there must fail rather than end quietly: thinking counts toward the + * cap, so the stop often lands mid-thought and would read as the model giving + * up. Failing also keeps the turn's output so far for a follow-up. */ +export class OutputTokenLimitError extends Error { + constructor() { + super( + "The response was cut off at the model's output token limit. Ask it to continue, or raise the limit in the workspace AI settings under Model output limits." + ) + this.name = 'OutputTokenLimitError' + } +} diff --git a/frontend/src/lib/components/copilot/lib.test.ts b/frontend/src/lib/components/copilot/lib.test.ts index 5f1c657b39..8e06e0e2b9 100644 --- a/frontend/src/lib/components/copilot/lib.test.ts +++ b/frontend/src/lib/components/copilot/lib.test.ts @@ -263,6 +263,7 @@ describe('model context windows', () => { expect(getKnownModelContextWindow('gemini-3-flash')).toBe(1000000) expect(getKnownModelContextWindow('gemini-2.5-flash')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-v4-pro')).toBe(1000000) + expect(getKnownModelContextWindow('deepseek-flash')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-chat')).toBe(1000000) expect(getKnownModelContextWindow('deepseek-reasoner')).toBe(1000000) }) diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts index c665cc964e..b011c1f672 100644 --- a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -285,3 +285,33 @@ describe('parseOpenAICompletion tool call arguments', () => { expect(toolResult.content).toBe('tool ok') }) }) + +describe('output token limit', () => { + it("gives DeepSeek DeepSeek's own thinking-mode budget rather than the 8192 fallback", async () => { + const { getModelMaxTokens } = await import('./lib') + expect(getModelMaxTokens('deepseek', 'deepseek-flash')).toBe(131072) + expect(getModelMaxTokens('deepseek', 'deepseek-v4-pro')).toBe(131072) + }) + + it('fails a response cut off at the output token limit instead of ending the turn', async () => { + const { parseOpenAICompletion } = await import('./lib') + const { OutputTokenLimitError } = await import('./chat/outputTokenLimit') + + // Thinking counts toward max_tokens, so the cap can land before any answer. + const parsed = parseOpenAICompletion( + streamOf([ + { choices: [{ delta: { reasoning_content: 'I need to plan the flow so each' } }] }, + { choices: [{ delta: {}, finish_reason: 'length' }] } + ]), + createCallbacks(), + [], + [], + [], + {}, + undefined, + { workspace: 'test' } + ) + + await expect(parsed).rejects.toBeInstanceOf(OutputTokenLimitError) + }) +}) diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 19f350a665..5094764807 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -28,6 +28,7 @@ import { type Tool, type ToolCallbacks } from './chat/shared' +import { OutputTokenLimitError } from './chat/outputTokenLimit' import { hasValidToolCallArguments } from './chat/toolCallArguments' import { getNonStreamingOpenAIResponsesCompletion, @@ -116,7 +117,7 @@ export const AI_PROVIDERS: Record = { }, deepseek: { label: 'DeepSeek', - defaultModels: ['deepseek-v4-pro', 'deepseek-v4-flash'] + defaultModels: ['deepseek-v4-pro', 'deepseek-flash'] }, groq: { label: 'Groq', @@ -310,7 +311,12 @@ export async function fetchAvailableModels( } export function getModelMaxTokens(provider: AIProvider, model: string) { - if (model.includes('gpt-5')) { + if (provider === 'deepseek') { + // DeepSeek thinks by default and counts the thinking toward max_tokens, so + // the 8192 fallback cuts long turns off mid-thought. 131072 is DeepSeek's + // own default at the highest effort (65536 at the default one). + return 131072 + } else if (model.includes('gpt-5')) { return 128000 } else if ( (provider === 'azure_openai' || provider === 'openai' || provider === 'azure_foundry') && @@ -1218,6 +1224,7 @@ export async function parseOpenAICompletion( // to the next call, the previous one is demoted to queued. let streamingToolCallId: string | undefined = undefined let malformedFunctionCallError = false + let hitOutputTokenLimit = false let tokenUsage = emptyChatTokenUsage() let answer = '' @@ -1244,6 +1251,9 @@ export async function parseOpenAICompletion( ) { malformedFunctionCallError = true } + if (finishReason === 'length') { + hitOutputTokenLimit = true + } // Mistral nests reasoning inside structured content parts; split them out // so a content delta never leaks "[object Object]" into the answer. @@ -1460,6 +1470,8 @@ export async function parseOpenAICompletion( } messages.push(toolResponse) addedMessages.push(toolResponse) + } else if (hitOutputTokenLimit) { + throw new OutputTokenLimitError() } else { return { shouldContinue: false, tokenUsage } } diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index e5a66248b3..be49907ed4 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -96,10 +96,12 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['gemini-3.1', 1_000_000], ['gemini-3', 1_000_000], ['gemini-2.5', 1_000_000], - // DeepSeek — the V4 family (pro / flash) is 1M. The deepseek-chat / - // deepseek-reasoner aliases were retired 2026-07-24 but can still sit in a - // saved selection, so they keep resolving to the window they had. + // DeepSeek — the V4 family (pro / flash) is 1M; V4.1 Flash is served as + // `deepseek-flash`. The deepseek-chat / deepseek-reasoner aliases were retired + // 2026-07-24 but can still sit in a saved selection, so they keep resolving to + // the window they had. ['deepseek-v4', 1_000_000], + ['deepseek-flash', 1_000_000], ['deepseek-chat', 1_000_000], ['deepseek-reasoner', 1_000_000], ['deepseek', 128_000], From 9ad2c91ddb661ab60ada9ec4039ef8d025602969 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 10:13:58 +0200 Subject: [PATCH 10/34] fix: enforce token path scopes on GET /raw_apps/list (#11284) * fix: enforce token path scopes on GET /raw_apps/list Co-Authored-By: Claude Opus 5 (1M context) * fix: apply the raw app scope filter before the page limit Co-Authored-By: Claude Opus 5 (1M context) * test: cover the bare prefix path in the raw app scope test Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/raw_apps_list_scope.rs | 76 ++++++++++++++++++++++++++++ backend/windmill-api/src/raw_apps.rs | 30 ++++++++++- 2 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 backend/tests/raw_apps_list_scope.rs diff --git a/backend/tests/raw_apps_list_scope.rs b/backend/tests/raw_apps_list_scope.rs new file mode 100644 index 0000000000..503bca7577 --- /dev/null +++ b/backend/tests/raw_apps_list_scope.rs @@ -0,0 +1,76 @@ +//! `GET /raw_apps/list` must honor a path-scoped token. The route layer only checks +//! the scope domain and RLS knows nothing of token scopes, so the per-path filter +//! lives in the handler, ahead of the page's LIMIT. + +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +async fn list_paths(port: u16, token: &str, query: &str) -> anyhow::Result> { + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{port}/api/w/test-workspace/raw_apps/list?{query}" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + .await?; + assert_eq!(resp.status(), 200, "list: {}", resp.text().await?); + let body: Vec = resp.json().await?; + let mut paths: Vec = body + .iter() + .map(|a| a["path"].as_str().unwrap().to_string()) + .collect(); + paths.sort(); + Ok(paths) +} + +#[sqlx::test(fixtures("base"))] +async fn test_raw_apps_list_filters_by_token_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // The list sorts by edited_at desc, so the out-of-scope apps fill the first page + // unless the scope is applied before the LIMIT. + for (path, age_minutes) in [ + ("f/other/c", 0), + ("u/test-userx/d", 1), + ("u/test-user/a", 2), + ("u/test-user/sub/b", 3), + // A `prefix/*` grant also covers the path at the prefix itself. + ("u/test-user", 4), + ] { + sqlx::query( + "INSERT INTO raw_app (path, workspace_id, data, edited_at) + VALUES ($1, 'test-workspace', '', now() - make_interval(mins => $2))", + ) + .bind(path) + .bind(age_minutes) + .execute(&db) + .await?; + } + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, scopes) + VALUES (encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', + 'test@windmill.dev', 'scoped', ARRAY['raw_apps:read:u/test-user/*'])", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // The same user through an unscoped token sees every row. + assert_eq!( + list_paths(port, "SECRET_TOKEN", "").await?, + [ + "f/other/c", + "u/test-user", + "u/test-user/a", + "u/test-user/sub/b", + "u/test-userx/d" + ] + ); + assert_eq!( + list_paths(port, "SCOPED_TOKEN", "per_page=3").await?, + ["u/test-user", "u/test-user/a", "u/test-user/sub/b"] + ); + Ok(()) +} diff --git a/backend/windmill-api/src/raw_apps.rs b/backend/windmill-api/src/raw_apps.rs index d7bc3bc6c0..35e4f693a4 100644 --- a/backend/windmill-api/src/raw_apps.rs +++ b/backend/windmill-api/src/raw_apps.rs @@ -5,7 +5,10 @@ * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ -use crate::{db::ApiAuthed, utils::check_scopes}; +use crate::{ + db::ApiAuthed, + utils::{build_scope_path_filter, check_scopes, ScopePathFilter}, +}; use axum::{ body::Body, extract::{Extension, Json, Path, Query}, @@ -103,6 +106,31 @@ async fn list_apps( } } + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would return short pages and let a page's size report how + // many raw apps the token may not read. One `?` per term, since a chained `.bind` + // substitutes into a `?` inside the value an earlier one inserted; `starts_with`, + // since LIKE reads a `_` in the path as a wildcard. + if let ScopePathFilter::Restricted { exact, prefix } = + build_scope_path_filter(&authed, "raw_apps", "read") + { + let terms: Vec = exact + .iter() + .chain(&prefix) + .map(|p| "app.path = ?".bind(p)) + .chain( + prefix + .iter() + .map(|p| "starts_with(app.path, ?)".bind(&format!("{p}/"))), + ) + .collect(); + sqlb.and_where(if terms.is_empty() { + "false".to_string() + } else { + format!("({})", terms.join(" OR ")) + }); + } + let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?; let mut tx = user_db.begin(&authed).await?; let rows = sqlx::query_as::<_, ListableApp>(&sql) From 586699c4938b5203920a43e69a706310078b93f5 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 10:14:12 +0200 Subject: [PATCH 11/34] fix: apply SSRF validation to workspace webhook URLs (#11285) * fix: apply SSRF validation to workspace webhook URLs Co-Authored-By: Claude Opus 5 * test: assert edit_webhook refuses a private webhook URL Co-Authored-By: Claude Opus 5 * fix: stop the webhook sender following redirects past the SSRF check Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/THREAT_MODEL.md | 2 +- .../tests/workspaces.rs | 8 ++ .../windmill-api-workspaces/src/workspaces.rs | 11 ++ backend/windmill-common/src/ssrf.rs | 103 ++++++++++++++++++ backend/windmill-common/src/webhook.rs | 5 +- .../(logged)/workspace_settings/+page.svelte | 2 +- 6 files changed, 128 insertions(+), 3 deletions(-) diff --git a/backend/THREAT_MODEL.md b/backend/THREAT_MODEL.md index af33877078..73a4b53227 100644 --- a/backend/THREAT_MODEL.md +++ b/backend/THREAT_MODEL.md @@ -97,7 +97,7 @@ published advisory history (73 GHSA advisories, several rated 9.9 critical). | id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence | |---|---|---|---|---|---|---|---|---|---| | T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b | -| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; SAML IdP metadata URLs are SSRF-validated at load time behind the `ALLOW_PRIVATE_SAML_METADATA_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | +| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; MCP private URL access requires the instance-wide `ALLOW_PRIVATE_MCP_SERVER_URLS` opt-in; WebSocket trigger URLs (stored, test, and runnable-resolved) are SSRF-validated at connect time behind the `ALLOW_PRIVATE_WEBSOCKET_URLS` opt-in, and the trigger test route now requires `:write` scope; SAML IdP metadata URLs are SSRF-validated at load time behind the `ALLOW_PRIVATE_SAML_METADATA_URLS` opt-in; workspace webhook URLs are SSRF-checked when saved, not when sent, behind the `ALLOW_PRIVATE_WEBHOOK_URLS` opt-in; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 | | T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 | | T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 | | T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e | diff --git a/backend/windmill-api-integration-tests/tests/workspaces.rs b/backend/windmill-api-integration-tests/tests/workspaces.rs index ff603269e7..29b8a0a611 100644 --- a/backend/windmill-api-integration-tests/tests/workspaces.rs +++ b/backend/windmill-api-integration-tests/tests/workspaces.rs @@ -319,6 +319,14 @@ async fn test_workspace_endpoints(db: Pool) -> anyhow::Result<()> { .unwrap(); assert_eq!(resp.status(), 200); + let resp = authed(client().post(format!("{base}/edit_webhook"))) + .json(&json!({"webhook": "http://127.0.0.1/hook"})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await?.contains("ALLOW_PRIVATE_WEBHOOK_URLS")); + // --- edit_auto_invite (EE-gated) --- let resp = authed(client().post(format!("{base}/edit_auto_invite"))) .json(&json!({"operator": false, "invite_all": false, "auto_add": false})) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 029f5d3945..8ebd4eeb26 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_common::ssrf::{validate_webhook_url, webhook_ssrf_error_message}; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; use windmill_common::webhook::WebhookShared; use windmill_common::{BASE_URL, DB}; @@ -1923,6 +1924,16 @@ async fn edit_webhook( )); } + // An empty URL is stored as-is and means "no webhook" to the sender. + if let Some(webhook) = ew.webhook.as_deref().filter(|w| !w.is_empty()) { + validate_webhook_url(webhook).await.map_err(|e| { + Error::BadRequest(format!( + "Webhook URL is not allowed: {}", + webhook_ssrf_error_message(&e) + )) + })?; + } + let mut tx = db.begin().await?; if let Some(webhook) = &ew.webhook { diff --git a/backend/windmill-common/src/ssrf.rs b/backend/windmill-common/src/ssrf.rs index 31dee749e6..0cdf7ffa4f 100644 --- a/backend/windmill-common/src/ssrf.rs +++ b/backend/windmill-common/src/ssrf.rs @@ -8,6 +8,8 @@ pub const ALLOW_PRIVATE_SAML_METADATA_URLS_ENV: &str = "ALLOW_PRIVATE_SAML_METAD pub const ALLOW_PRIVATE_GUEST_JWKS_URLS_ENV: &str = "ALLOW_PRIVATE_GUEST_JWKS_URLS"; +pub const ALLOW_PRIVATE_WEBHOOK_URLS_ENV: &str = "ALLOW_PRIVATE_WEBHOOK_URLS"; + /// Lets every git call reach hosts on a private network, whoever it is made for. /// Without it, [`private_git_host_allowed`] decides. pub const ALLOW_LOCAL_GIT_REMOTES_ENV: &str = "ALLOW_LOCAL_GIT_REMOTES"; @@ -207,6 +209,12 @@ pub fn allow_private_saml_metadata_urls() -> bool { .is_some_and(|v| v == "true" || v == "1") } +fn allow_private_webhook_urls() -> bool { + std::env::var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV) + .ok() + .is_some_and(|v| v == "true" || v == "1") +} + fn allow_local_git_remotes() -> bool { std::env::var(ALLOW_LOCAL_GIT_REMOTES_ENV) .ok() @@ -321,6 +329,26 @@ pub async fn validate_mcp_server_url(url: &str) -> Result Result { + let parsed = + url::Url::parse(url).map_err(|e| SsrfValidationError::InvalidUrl(e.to_string()))?; + + match parsed.scheme() { + "http" | "https" => {} + scheme => return Err(SsrfValidationError::DisallowedScheme(scheme.to_string())), + } + + let host = parsed.host_str().ok_or(SsrfValidationError::MissingHost)?; + + if allow_private_webhook_urls() { + return Ok(ValidatedTarget::unpinned(host)); + } + + validate_url_for_ssrf(url).await +} + /// Validate an MCP-related URL and return the [`ValidatedTarget`] so the caller /// can pin the connect: the OAuth registration/discovery/token requests carry /// secrets, so they must target the validated address (see @@ -358,6 +386,16 @@ pub fn saml_ssrf_error_message(e: &SsrfValidationError) -> String { } } +pub fn webhook_ssrf_error_message(e: &SsrfValidationError) -> String { + match e { + SsrfValidationError::Private { .. } => format!( + "{e}. If you need to use private/internal webhook URLs, \ + set the {ALLOW_PRIVATE_WEBHOOK_URLS_ENV}=true environment variable" + ), + _ => e.to_string(), + } +} + fn is_private_ip(ip: &IpAddr) -> bool { match ip { IpAddr::V4(ipv4) => is_private_ipv4(ipv4), @@ -684,6 +722,71 @@ mod tests { )); } + struct PrivateWebhookUrlsEnvGuard { + previous: Option, + } + + impl PrivateWebhookUrlsEnvGuard { + fn set(value: Option<&str>) -> Self { + let previous = std::env::var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV).ok(); + match value { + Some(value) => std::env::set_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV), + } + Self { previous } + } + } + + impl Drop for PrivateWebhookUrlsEnvGuard { + fn drop(&mut self) { + match &self.previous { + Some(value) => std::env::set_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV, value), + None => std::env::remove_var(ALLOW_PRIVATE_WEBHOOK_URLS_ENV), + } + } + } + + #[tokio::test] + async fn validate_webhook_url_blocks_private_by_default_with_env_hint() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateWebhookUrlsEnvGuard::set(None); + + let private_error = validate_webhook_url("http://127.0.0.1/hook") + .await + .unwrap_err(); + assert!(matches!( + private_error, + SsrfValidationError::Private { resolved: false } + )); + assert!( + webhook_ssrf_error_message(&private_error).contains("ALLOW_PRIVATE_WEBHOOK_URLS=true") + ); + + let invalid_error = validate_webhook_url("ftp://example.com/hook") + .await + .unwrap_err(); + assert!( + !webhook_ssrf_error_message(&invalid_error).contains(ALLOW_PRIVATE_WEBHOOK_URLS_ENV) + ); + } + + #[tokio::test] + async fn validate_webhook_url_allows_private_when_env_is_set_but_keeps_syntax_guards() { + let _lock = TEST_ENV_LOCK.lock().await; + let _guard = PrivateWebhookUrlsEnvGuard::set(Some("true")); + + assert!(validate_webhook_url("http://127.0.0.1/hook").await.is_ok()); + assert!(validate_webhook_url("http://10.0.0.1/hook").await.is_ok()); + assert!(matches!( + validate_webhook_url("file:///etc/passwd").await, + Err(SsrfValidationError::DisallowedScheme(_)) + )); + assert!(matches!( + validate_webhook_url("not-a-url").await, + Err(SsrfValidationError::InvalidUrl(_)) + )); + } + #[test] fn private_git_hosts_are_refused_to_non_admins_and_on_cloud() { use GitRemoteCaller::{AdminOrSystem, NonAdmin}; diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index 02d6f7ddc5..2f0d83f7e2 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -164,7 +164,10 @@ impl WebhookShared { reqwest::Client::builder() .connect_timeout(Duration::from_secs(5)) // TODO: investigate pool timeouts and such if TCP load is high - .timeout(Duration::from_secs(5)), + .timeout(Duration::from_secs(5)) + // The workspace webhook URL is SSRF-checked when saved; a redirect + // would send the POST past that check. + .redirect(reqwest::redirect::Policy::none()), ) .build() .unwrap(); diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 053ec2f2a1..6ab4fe6558 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -518,7 +518,7 @@ webhook = '' } } catch (error) { - sendUserToast(`Failed to save webhook: ${error}`, true) + sendUserToast(`Failed to save webhook: ${error?.body ?? error}`, true) } } From faf7b22be0827738c90c1f3ed6cd3e214d6c63d3 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 10:14:41 +0200 Subject: [PATCH 12/34] fix: apply token path scopes to the native trigger list (#11281) Co-authored-by: Claude Opus 5 --- ...b983774ee2127dceb790181ed91b987f44ff.json} | 9 ++-- .../tests/native_triggers.rs | 52 ++++++++++++++++++- backend/windmill-api/src/workspaces_export.rs | 14 +++-- .../windmill-native-triggers/src/handler.rs | 4 +- backend/windmill-native-triggers/src/lib.rs | 22 ++++++-- backend/windmill-native-triggers/src/sync.rs | 13 ++++- 6 files changed, 101 insertions(+), 13 deletions(-) rename backend/.sqlx/{query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json => query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json} (85%) diff --git a/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json b/backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json similarity index 85% rename from backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json rename to backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json index 0f43afed39..6964dfa8d6 100644 --- a/backend/.sqlx/query-a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a.json +++ b/backend/.sqlx/query-f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary,\n nt.enabled\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n ) AND\n ( $7\n OR nt.script_path = ANY($8)\n OR EXISTS ( SELECT 1 FROM unnest($9::text[]) AS pfx\n WHERE nt.script_path = pfx\n OR left(nt.script_path, length(pfx) + 1) = pfx || '/' ) )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -93,7 +93,10 @@ "Int8", "Int8", "Text", - "Bool" + "Bool", + "Bool", + "TextArray", + "TextArray" ] }, "nullable": [ @@ -111,5 +114,5 @@ false ] }, - "hash": "a2122e8520268919e2ddc85ef46b5f9322229bb2a1e104b6eabaf2a697c2776a" + "hash": "f3cd54017710536747b8aca797a8b983774ee2127dceb790181ed91b987f44ff" } diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 059b3aa2d2..e7f0147323 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -12,7 +12,7 @@ use serde_json::json; use sqlx::{Pool, Postgres}; use axum::http::StatusCode; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{ApiAuthed, ScopePathFilter}; use windmill_common::{ error::Error, variables::{build_crypt, encrypt}, @@ -661,6 +661,7 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< None, Some("f/test/after"), Some(false), + ScopePathFilter::AllowAll, ) .await?; assert_eq!( @@ -672,6 +673,55 @@ async fn test_rename_moves_native_trigger(db: Pool) -> anyhow::Result< Ok(()) } +// ============================================================================ +// 6. Token scope filtering on list +// ============================================================================ + +/// A `native_triggers:read:f/test/*` token must only list triggers under `f/test/`, on the `/` +/// boundary (`f/testx/` is out of scope). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_native_triggers_scope_filter(db: Pool) -> anyhow::Result<()> { + for (i, path) in ["f/testx/out", "f/test/in"].into_iter().enumerate() { + insert_test_script(&db, path).await?; + store_native_trigger( + &db, + "test-workspace", + ServiceName::Nextcloud, + &format!("ext-{i}"), + &NativeTriggerConfig { + script_path: path.to_string(), + is_flow: false, + webhook_token: format!("abcdefghij123456789{i}"), + }, + json!({"event": "OCP\\Files\\Events\\Node\\NodeCreatedEvent"}), + None, + true, + ) + .await?; + } + + let triggers = list_native_triggers( + &db, + "test-workspace", + ServiceName::Nextcloud, + None, + None, + None, + None, + ScopePathFilter::Restricted { exact: vec![], prefix: vec!["f/test".to_string()] }, + ) + .await?; + assert_eq!( + triggers + .iter() + .map(|t| t.script_path.as_str()) + .collect::>(), + vec!["f/test/in"] + ); + + Ok(()) +} + // --- parse_stop_channel_params --- #[test] diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 4f6a6d9efb..e29113ecd2 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -1425,9 +1425,17 @@ pub(crate) async fn tarball_workspace( use strum::IntoEnumIterator; for service_name in ServiceName::iter() { - let native_triggers = - list_native_triggers(&mut *tx, &w_id, service_name, None, None, None, None) - .await?; + let native_triggers = list_native_triggers( + &mut *tx, + &w_id, + service_name, + None, + None, + None, + None, + windmill_api_auth::ScopePathFilter::AllowAll, + ) + .await?; // Native triggers (Nextcloud, Google Drive, GitHub) are never // cloned into a fork — a fork only has one if its owner created diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index 27a50ba07a..2c785a4e26 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -16,7 +16,8 @@ use serde::{Deserialize, Serialize}; use sqlx::PgConnection; use std::sync::Arc; use windmill_api_auth::{ - check_scopes, create_token_internal, require_is_writer, ApiAuthed, NewToken, + build_scope_path_filter, check_scopes, create_token_internal, require_is_writer, ApiAuthed, + NewToken, }; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ @@ -701,6 +702,7 @@ async fn list_native_triggers_handler( query.per_page, query.path.as_deref(), query.is_flow, + build_scope_path_filter(&authed, "native_triggers", "read"), ) .await?; tx.commit().await?; diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 570319ee0e..6398e8a6d9 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -56,7 +56,7 @@ use windmill_queue::PushArgsOwned; #[cfg(feature = "native_trigger")] use windmill_oauth::{ErrorField, ExecuteError, OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{ApiAuthed, ScopePathFilter}; pub mod handler; pub(crate) mod lock; pub mod sync; @@ -1493,9 +1493,17 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> per_page: Option, path: Option<&str>, is_flow: Option, + scope: ScopePathFilter, ) -> Result> { let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; let limit = per_page.unwrap_or(100) as i64; + // In the WHERE, not a retain after the fetch: the result is paginated, and a + // post-fetch filter would let a page's size report how many rows the token + // may not read. + let (scope_all, scope_exact, scope_prefix) = match scope { + ScopePathFilter::AllowAll => (true, Vec::new(), Vec::new()), + ScopePathFilter::Restricted { exact, prefix } => (false, exact, prefix), + }; let triggers = sqlx::query_as!( NativeTrigger, @@ -1532,7 +1540,12 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> WHERE f.workspace_id = nt.workspace_id AND f.path = nt.script_path )) - ) + ) AND + ( $7 + OR nt.script_path = ANY($8) + OR EXISTS ( SELECT 1 FROM unnest($9::text[]) AS pfx + WHERE nt.script_path = pfx + OR left(nt.script_path, length(pfx) + 1) = pfx || '/' ) ) LIMIT $3 OFFSET $4 "#, @@ -1541,7 +1554,10 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> limit, offset, path, - is_flow + is_flow, + scope_all, + &scope_exact[..], + &scope_prefix[..], ) .fetch_all(db) .await?; diff --git a/backend/windmill-native-triggers/src/sync.rs b/backend/windmill-native-triggers/src/sync.rs index bc259f048e..4f75845429 100644 --- a/backend/windmill-native-triggers/src/sync.rs +++ b/backend/windmill-native-triggers/src/sync.rs @@ -214,8 +214,17 @@ pub async fn sync_workspace_triggers( workspace_id ); - let windmill_triggers = - list_native_triggers(db, workspace_id, T::SERVICE_NAME, None, None, None, None).await?; + let windmill_triggers = list_native_triggers( + db, + workspace_id, + T::SERVICE_NAME, + None, + None, + None, + None, + windmill_api_auth::ScopePathFilter::AllowAll, + ) + .await?; if windmill_triggers.is_empty() { tracing::info!( From e7ba5954b69514d72c140f9b1e697fb2236df23e Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 22 Sep 2026 10:19:38 +0200 Subject: [PATCH 13/34] feat: add workspace mentions to AI composer (#11264) * feat: add workspace mentions to AI composer * fix: ignore embedded mention text in composer * fix: sync only standalone composer mentions * fix: keep punctuation-adjacent mentions active * fix: use unicode mention boundaries * fix: handle unicode mention prefixes * fix: avoid rescanning mention prefixes --- .../lib/components/DropdownSubmenuItem.svelte | 25 ++- frontend/src/lib/components/DropdownV2.svelte | 5 +- .../src/lib/components/DropdownV2Inner.svelte | 7 +- .../copilot/chat/AIChatDisplay.svelte | 19 +++ .../copilot/chat/AIChatInput.svelte | 41 ++--- .../copilot/chat/ContextTextarea.svelte | 39 +++-- .../chat/WorkspaceMentionPicker.svelte | 148 ++++++++++++++++++ .../components/copilot/chat/mention.test.ts | 89 ++++++++++- .../lib/components/copilot/chat/mention.ts | 56 +++++++ frontend/src/lib/utils.ts | 2 + 10 files changed, 384 insertions(+), 47 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte diff --git a/frontend/src/lib/components/DropdownSubmenuItem.svelte b/frontend/src/lib/components/DropdownSubmenuItem.svelte index 60f7ad2d64..a67411a039 100644 --- a/frontend/src/lib/components/DropdownSubmenuItem.svelte +++ b/frontend/src/lib/components/DropdownSubmenuItem.svelte @@ -13,9 +13,10 @@ item: Item builders: ReturnType['builders'] meltItem: MenubarMenuElements['item'] + close?: (afterClose?: () => void) => void } - let { item, builders, meltItem }: Props = $props() + let { item, builders, meltItem, close }: Props = $props() const { elements: { subTrigger, subMenu }, @@ -23,6 +24,8 @@ } = untrack(() => builders).createSubmenu() let subItems = $derived((item.submenuItems ?? []).filter((i) => !i.hide)) + const CustomSubmenu = $derived(item.customSubmenu) + const closeMenu = $derived(close ?? (() => {}))
{/if} diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 5061636835..1d86519293 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -132,8 +132,9 @@ }) }) - export function close() { + export function close(afterClose?: () => void) { open = false + if (afterClose) setTimeout(afterClose, 10) } async function computeItems(): Promise { @@ -209,7 +210,7 @@ class="bg-surface-tertiary dark:border w-56 origin-top-right rounded-lg shadow-lg focus:outline-none overflow-y-auto py-1" style={`${customWidth ? `width: ${customWidth}px;` : ''} max-height: ${maxHeight || '50vh'};`} > - + {/if} diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index 7749310b5b..31fd9426f5 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -13,9 +13,10 @@ items?: Item[] | (() => Item[]) | (() => Promise) meltItem: MenubarMenuElements['item'] builders?: ReturnType['builders'] + close?: (afterClose?: () => void) => void } - let { aiId, items = [], meltItem, builders }: Props = $props() + let { aiId, items = [], meltItem, builders, close }: Props = $props() let computedItems: Item[] | undefined = $state(undefined) async function computeItems() { @@ -102,8 +103,8 @@ {#if item.separatorTop}
{/if} - {#if item.submenuItems && builders} - + {#if (item.submenuItems || item.customSubmenu) && builders} + {})} /> {:else if item.disabled && item.tooltip} diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index cf27a45d89..17d752ae39 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -3,6 +3,7 @@ import AIChatMessage from './AIChatMessage.svelte' import AppAvailableContextList from './AppAvailableContextList.svelte' import ChatContextPicker from './ChatContextPicker.svelte' + import WorkspaceMentionPicker from './WorkspaceMentionPicker.svelte' import { type Snippet } from 'svelte' import { AlertTriangle, @@ -548,6 +549,11 @@ await imageWork } + function mentionWorkspaceItem(element: ContextElement) { + void aiChatInput?.addContextToSelection(element) + aiChatInput?.insertMention(element.title) + } + function onFolderInputChange(e: Event) { const input = e.currentTarget as HTMLInputElement // webkitdirectory files carry webkitRelativePath (`folder/sub/file`); addFiles groups @@ -1055,6 +1061,19 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> } ] : []), + ...(inGlobal + ? [ + { + displayName: 'Mention file', + icon: AtSign, + customSubmenu: WorkspaceMentionPicker, + customSubmenuProps: { + onSelect: mentionWorkspaceItem, + onAfterClose: () => aiChatInput?.focusInput() + } + } + ] + : []), ...(skillItems ? [ { diff --git a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte index 6f15937cf2..bcea3a26d6 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatInput.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatInput.svelte @@ -14,7 +14,7 @@ import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' import { getChatViewHost } from './chatViewHost' import { composerBoxClass, COMPOSER_FIELD_RESET } from './composerBox' - import { formatMention } from './mention' + import { formatMention, hasMention, removeMentionFromText } from './mention' import { twMerge } from 'tailwind-merge' import { tick, untrack, type Snippet } from 'svelte' import Portal from '$lib/components/Portal.svelte' @@ -588,40 +588,33 @@ const contextKey = contextElementKey - /** Append `@title` to the textarea so the button-picker path stays in - * sync with the inline `@` mention path — both leave a visible - * token tied to the selectedContext entry, which the textarea diffs on - * to auto-remove items when the user deletes them. No-op when the - * mention is already present so re-picking the same item doesn't - * leave duplicate tokens. */ + /** Insert `@title` so button/menu picker paths stay in sync with + * the inline `@` mention path — both leave a visible token tied + * to the selectedContext entry, which the textarea diffs on to + * auto-remove items when the user deletes them. No-op when the same + * standalone mention is already present, so picking the same item + * again doesn't leave duplicate tokens. */ export function insertMention(title: string) { - const target = `@${title}` - if (draft.text.split(/\s+/).includes(target)) return + if (hasMention(draft.text, title)) return + if (contextTextareaComponent) { + void contextTextareaComponent.insertMention(title) + return + } const sep = draft.text.length === 0 || /\s$/.test(draft.text) ? '' : ' ' - draft.text = `${draft.text}${sep}${target} ` + draft.text = `${draft.text}${sep}${formatMention(title)} ` } /** Strip every `@title` token from the textarea — used when the user * deletes the corresponding badge so the badge X-button mirrors the - * inverse (text-delete-to-badge-remove) sync. Only matches `@title` as a - * standalone token (boundary on both sides) so substring matches don't - * bleed into other words; only the whitespace adjacent to the removed - * mention is collapsed so unrelated double-spaces stay intact. */ + * inverse (text-delete-to-badge-remove) sync. Only matches tokens whose + * `@` starts a word, so embedded text like `owner@app.ts` is left alone + * while punctuation and non-spacing text after the token still work. */ export function removeMention(title: string) { // Pre-zap the textarea's mention diff snapshot so the upcoming strip // doesn't refire the removal effect on a same-title sibling — the host // has already mutated `selectedContext` to drop the targeted entry. contextTextareaComponent?.unsyncMention(title) - const escaped = title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - const re = new RegExp(`(^|\\s)@${escaped}(\\s|$)`, 'g') - draft.text = draft.text.replace(re, (_m, lead, trail) => { - // Boundary on at least one side → drop the mention entirely. - if (!lead || !trail) return '' - // Middle of text: keep ONE of the bracketing whitespace chars so - // the surviving tokens are still separated; preserve the leading - // one verbatim so newlines/tabs aren't downgraded to spaces. - return lead - }) + draft.text = removeMentionFromText(draft.text, title) } export function focusInput() { diff --git a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte index 976b969b3f..dd3a011da2 100644 --- a/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextTextarea.svelte @@ -10,7 +10,13 @@ import { zIndexes } from '$lib/zIndexes' import { twMerge } from 'tailwind-merge' import { CHAT_INPUT_PADDING, getAiChatManager } from './aiChatManagerContext' - import { MENTION_RE, mentionTitle, formatMention } from './mention' + import { + MENTION_RE, + mentionTitle, + formatMention, + hasMentionLeadingBoundary, + mentionTitlesInText + } from './mention' import { createFloatingActions, createVirtualElement } from 'svelte-floating-ui' import { flip, offset, shift } from 'svelte-floating-ui/dom' import { @@ -65,17 +71,11 @@ const aiChatManager = getAiChatManager() - function extractMentions(text: string): Set { - const out = new Set() - for (const m of text.matchAll(MENTION_RE)) out.add(mentionTitle(m[0])) - return out - } - // Titles currently appearing as `@title` mentions in the textarea. Compared // against the previous snapshot in a $effect (NOT inside handleInput — // the picker mutates `value` programmatically via `updateInstructionsWithContext`, // which doesn't fire `oninput`, so a handleInput-only diff goes stale). - const mentionedTitles = $derived(extractMentions(value)) + const mentionedTitles = $derived(mentionTitlesInText(value)) let prevMentionedTitles = $state>(new Set()) let showContextTooltip = $state(false) @@ -268,7 +268,11 @@ if (!att) return match return `${match}` }) - html = html.replace(MENTION_RE, (match) => { + html = html.replace(MENTION_RE, (match, ...args) => { + const offset = args[args.length - 2] + if (typeof offset !== 'number' || !hasMentionLeadingBoundary(html, offset)) { + return match + } const title = unescapeHtml(mentionTitle(match)) const inContext = availableContext.find((c) => c.title === title) || @@ -566,6 +570,23 @@ showContextTooltip = false } + export async function insertMention(title: string) { + const token = formatMention(title) + const selectionStart = textarea?.selectionStart ?? value.length + const selectionEnd = textarea?.selectionEnd ?? selectionStart + const { from, to, ids } = tokensOverlapping(selectionStart, selectionEnd) + const before = value.slice(0, from) + const after = value.slice(to) + const prefix = before.length === 0 || /\s$/.test(before) ? '' : ' ' + const suffix = after.length === 0 ? ' ' : /^\s/.test(after) ? '' : ' ' + const inserted = `${prefix}${token}${suffix}` + replacePasteRange(from, to, ids, inserted) + await tick() + const pos = from + inserted.length + textarea?.setSelectionRange(pos, pos) + textarea?.focus() + } + function refreshCommandSkills() { if (commandSkillsRefreshInFlight) return commandSkillsRefreshInFlight = true diff --git a/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte b/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte new file mode 100644 index 0000000000..0fb21f684f --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/WorkspaceMentionPicker.svelte @@ -0,0 +1,148 @@ + + + +{#snippet leafIcon(leaf: DrillLeaf)} + +{/snippet} + +{#snippet branchIcon(branch: DrillBranch)} + {#if branch.icon} + {@const Icon = branch.icon} + + {:else} + + {/if} +{/snippet} + + +
+ relativizeWorkspacePath(workspaceItemDisplayPath(leaf.data), scope)} + onScopeChange={handleScopeChange} + onFilterChange={loader.onFilterChange} + rootLoading={WORKSPACE_KINDS.some((k) => !loader.loaded[k] && loader.loadingKind[k])} + /> +
diff --git a/frontend/src/lib/components/copilot/chat/mention.test.ts b/frontend/src/lib/components/copilot/chat/mention.test.ts index 8d379ce3b4..ae021bf1de 100644 --- a/frontend/src/lib/components/copilot/chat/mention.test.ts +++ b/frontend/src/lib/components/copilot/chat/mention.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest' -import { MENTION_RE, mentionTitle, formatMention } from './mention' +import { + MENTION_RE, + mentionTitle, + formatMention, + hasMention, + mentionTitlesInText, + removeMentionFromText +} from './mention' describe('formatMention', () => { it('leaves a simple name bare', () => { @@ -52,3 +59,83 @@ describe('MENTION_RE', () => { expect(mentionTitle(m[0])).toBe(name) }) }) + +describe('removeMentionFromText', () => { + it('removes a bare mention and keeps neighboring words separated', () => { + expect(removeMentionFromText('before @app.ts after', 'app.ts')).toBe('before after') + }) + + it('removes a bracketed mention', () => { + expect(removeMentionFromText('before @[my folder/a b.ts] after', 'my folder/a b.ts')).toBe( + 'before after' + ) + }) + + it('removes an escaped bracketed mention', () => { + expect(removeMentionFromText('before @[notes \\] draft.md] after', 'notes ] draft.md')).toBe( + 'before after' + ) + }) + + it('removes edge mentions without leaving extra whitespace', () => { + expect(removeMentionFromText('@app.ts after', 'app.ts')).toBe('after') + expect(removeMentionFromText('before @app.ts', 'app.ts')).toBe('before') + }) + + it('keeps embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact owner@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact owner@app.ts about' + ) + }) + + it('keeps Unicode-prefixed embedded mention-like text intact', () => { + expect(removeMentionFromText('Contact 用户@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 用户@app.ts about' + ) + expect(removeMentionFromText('Contact हिंदी@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact हिंदी@app.ts about' + ) + expect(removeMentionFromText('Contact 𐐀@app.ts about @app.ts', 'app.ts')).toBe( + 'Contact 𐐀@app.ts about' + ) + }) +}) + +describe('hasMention', () => { + it('does not treat embedded mention-like text as a selected context mention', () => { + expect(hasMention('Contact owner@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact owner@app.ts about @app.ts', 'app.ts')).toBe(true) + }) + + it('does not treat Unicode-prefixed embedded text as a mention', () => { + expect(hasMention('Contact 用户@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact हिंदी@app.ts', 'app.ts')).toBe(false) + expect(hasMention('Contact 𐐀@app.ts', 'app.ts')).toBe(false) + }) + + it('treats punctuation next to a mention as a boundary', () => { + expect(hasMention('use @app.ts, then compare', 'app.ts')).toBe(true) + expect(hasMention('open (@app.ts)', 'app.ts')).toBe(true) + }) +}) + +describe('mentionTitlesInText', () => { + it('extracts only standalone mentions for context synchronization', () => { + expect([...mentionTitlesInText('Contact owner@app.ts about @app.ts')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('Contact owner@app.ts')]).toEqual([]) + }) + + it('ignores Unicode-prefixed embedded text for context synchronization', () => { + expect([...mentionTitlesInText('Contact 用户@app.ts')]).toEqual([]) + }) + + it('keeps punctuation-adjacent mentions synchronized', () => { + expect([...mentionTitlesInText('use @app.ts, then compare')]).toEqual(['app.ts']) + expect([...mentionTitlesInText('open (@app.ts)')]).toEqual(['app.ts']) + }) + + it('ignores combining-mark and astral-letter embedded text', () => { + expect([...mentionTitlesInText('Contact हिंदी@app.ts')]).toEqual([]) + expect([...mentionTitlesInText('Contact 𐐀@app.ts')]).toEqual([]) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/mention.ts b/frontend/src/lib/components/copilot/chat/mention.ts index 65a2b3b7c6..d1d42696b0 100644 --- a/frontend/src/lib/components/copilot/chat/mention.ts +++ b/frontend/src/lib/components/copilot/chat/mention.ts @@ -29,3 +29,59 @@ const BARE_SAFE = /^[\w/.\-]+$/ export function formatMention(name: string): string { return BARE_SAFE.test(name) ? `@${name}` : `@[${name.replace(/[\\\]]/g, '\\$&')}]` } + +function isWordChar(char: string | undefined): boolean { + return char !== undefined && /[\p{L}\p{N}\p{M}_]/u.test(char) +} + +function codePointBefore(text: string, index: number): string | undefined { + if (index <= 0) return undefined + const prev = text.charCodeAt(index - 1) + if (prev >= 0xdc00 && prev <= 0xdfff && index > 1) return text.slice(index - 2, index) + return text[index - 1] +} + +// A mention is owned by this parser when the `@` starts a token. The trailing side is +// intentionally open: non-spacing scripts often continue immediately after a mention, +// while embedded forms like `owner@app.ts` are rejected by the leading boundary. +export function hasMentionLeadingBoundary(text: string, index: number): boolean { + return !isWordChar(codePointBefore(text, index)) +} + +export function isStandaloneMention(text: string, match: RegExpMatchArray): boolean { + if (match.index === undefined) return false + return hasMentionLeadingBoundary(text, match.index) +} + +export function hasMention(text: string, title: string): boolean { + return [...text.matchAll(MENTION_RE)].some( + (m) => mentionTitle(m[0]) === title && isStandaloneMention(text, m) + ) +} + +export function mentionTitlesInText(text: string): Set { + const out = new Set() + for (const m of text.matchAll(MENTION_RE)) { + if (isStandaloneMention(text, m)) out.add(mentionTitle(m[0])) + } + return out +} + +export function removeMentionFromText(text: string, title: string): string { + let out = '' + let last = 0 + for (const m of text.matchAll(MENTION_RE)) { + if (m.index === undefined || mentionTitle(m[0]) !== title) continue + let start = m.index + let end = m.index + m[0].length + if (!isStandaloneMention(text, m)) continue + const hasLead = start > 0 && /\s/.test(text[start - 1]) + const hasTrail = end < text.length && /\s/.test(text[end]) + if (hasLead && !hasTrail) start -= 1 + if (!hasLead && hasTrail) end += 1 + if (hasLead && hasTrail) end += 1 + out += text.slice(last, start) + last = end + } + return out + text.slice(last) +} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 0f9eb78da4..b90751b9d3 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1677,6 +1677,8 @@ export type Item = { iconColor?: string /** Extra props for `icon`, for an icon that does not take lucide's `size`. */ iconProps?: Record + customSubmenu?: import('svelte').Component + customSubmenuProps?: Record href?: string hrefTarget?: '_blank' | '_self' | '_parent' | '_top' disabled?: boolean From efa7a0a70af43f8827542059dd450326a9b88d62 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 10:48:55 +0200 Subject: [PATCH 14/34] feat: allow workspace-scoped dynamic custom tags (#11283) * feat: check dynamic worker tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * test: pin the tag check after a flow preprocessor resolves the tag Co-Authored-By: Claude Opus 5 (1M context) * fix: keep tied tag placeholders exact and drop an unresolvable flow tag Co-Authored-By: Claude Opus 5 (1M context) * fix: describe tied tag placeholders in the custom tags editor Co-Authored-By: Claude Opus 5 (1M context) * fix: check workflow-as-code task tags on the tag they resolve to Co-Authored-By: Claude Opus 5 (1M context) * fix: judge a preprocessed flow's tag as written when it is run Co-Authored-By: Claude Opus 5 (1M context) * fix: check a preprocessed flow's own tag, as written only where it reads args Co-Authored-By: Claude Opus 5 (1M context) * feat: explain dynamic tags in the worker tag picker Co-Authored-By: Claude Opus 5 (1M context) * fix: look up worker availability for typed static tags Co-Authored-By: Claude Opus 5 (1M context) * fix: poll worker availability with the tag list as it is at each run Co-Authored-By: Claude Opus 5 (1M context) * feat: allow workspace-scoped dynamic custom tags Co-Authored-By: Claude Opus 5 (1M context) * test: pin the $workspace lookup for scoped custom tag patterns Co-Authored-By: Claude Opus 5 (1M context) * feat: keep a tag listed with its own scope inside it past matching patterns Co-Authored-By: Claude Opus 5 (1M context) * fix: warn when a scoped custom tag is also listed for every workspace Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/tests/worker.rs | 116 ++++++++++++++++++ backend/windmill-common/src/jobs.rs | 69 ++++++++--- backend/windmill-common/src/worker.rs | 94 +++++++++++--- .../lib/components/AssignableTagsInner.svelte | 113 +++++++++++------ 4 files changed, 322 insertions(+), 70 deletions(-) diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 74a10488e6..d7fb992280 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -5719,6 +5719,122 @@ async fn test_fork_marker_tag_admission_through_lineage(db: Pool) -> a Ok(()) } +/// A workspace-scoped pattern admits the tags in its range only from the workspaces its scope +/// allows, and a tag listed by its own name with a scope stays inside that scope whatever pattern +/// it fits. The fork lineage and `$workspace` are resolved for a matching pattern. +#[sqlx::test(fixtures("base"))] +#[serial] +async fn test_scoped_custom_tag_pattern_admission(db: Pool) -> anyhow::Result<()> { + use windmill_common::worker::{CustomTags, CUSTOM_TAGS_PER_WORKSPACE}; + + initialize_tracing().await; + + // For a non-superadmin caller (a superadmin would bypass the scope check entirely). + async fn check(db: &Pool, w_id: &str, tag: &str, size: &str) -> Result<(), String> { + let args = std::collections::HashMap::from([( + "size".to_string(), + serde_json::value::to_raw_value(size).unwrap(), + )]); + let args = windmill_queue::PushArgs::from(&args); + windmill_queue::check_tag_available_for_push(db, w_id, tag, &args, false, None) + .await + .map_err(|e| e.to_string()) + } + async fn allowed(db: &Pool, w_id: &str, tag: &str, size: &str) -> bool { + check(db, w_id, tag, size).await.is_ok() + } + async fn allowed_as_written(db: &Pool, w_id: &str, tag: &str) -> bool { + windmill_common::jobs::check_tag_available_for_workspace_internal( + db, + w_id, + tag, + None, + std::future::ready(w_id.to_string()), + false, + None, + ) + .await + .is_ok() + } + + // The ancestor chain is cached process-wide by workspace id, so use one no other test takes. + let fork = "wm-fork-tagpattern"; + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ($1, $1, 'test-user', 'test-workspace')", + fork + ) + .execute(&db) + .await?; + + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::from(vec![ + "gpu-$args[size]".to_string(), + "$args[size]-secret(test-workspace)".to_string(), + "gpu-secret(other)".to_string(), + "$workspace-$args[size](test-workspace*)".to_string(), + "urgent".to_string(), + "urgent(other)".to_string(), + ]))); + + let confined = check(&db, "test-workspace", "gpu-$args[size]", "secret").await; + let checks = [ + ( + "scoped pattern from its workspace", + allowed(&db, "test-workspace", "cpu-secret", "").await, + ), + ( + "scoped pattern refused elsewhere", + !allowed(&db, "other", "cpu-secret", "").await, + ), + ("confined tag refused past the patterns it fits", confined.is_err()), + ( + "global pattern for the tags nothing confines", + allowed(&db, "test-workspace", "gpu-large", "").await, + ), + ( + "confined tag from its workspace", + allowed(&db, "other", "gpu-secret", "").await, + ), + // Listing the tag by name without a scope as well keeps it open everywhere. + ( + "tag listed both globally and scoped", + allowed(&db, "test-workspace", "urgent", "").await, + ), + // The fork's `$workspace` is its parent's, whose tags its lineage reaches. Written as the + // resolved tag, not as the entry, so only the pattern can admit it. + ( + "fork-scoped `$workspace` pattern from a fork", + allowed(&db, fork, "test-workspace-large", "").await, + ), + ( + "fork-scoped `$workspace` pattern refused elsewhere", + !allowed(&db, "other", "$workspace-$args[size]", "large").await, + ), + ( + "as written from its workspace", + allowed_as_written(&db, "test-workspace", "$args[size]-secret").await, + ), + ( + "as written refused elsewhere", + !allowed_as_written(&db, "other", "$args[size]-secret").await, + ), + ]; + CUSTOM_TAGS_PER_WORKSPACE.store(std::sync::Arc::new(CustomTags::default())); + + for (what, ok) in checks { + assert!(ok, "{what}"); + } + let confined = confined.unwrap_err(); + assert!( + confined.contains("the custom tag gpu-secret(other) restricts it to other workspaces") + && confined.contains("gpu-$args[size]") + && confined.contains("$args[size]-secret(test-workspace)"), + "got {confined}" + ); + + Ok(()) +} + #[cfg(all(feature = "quickjs", feature = "python"))] #[sqlx::test(fixtures("base"))] async fn test_whileloop_propagates_inner_iterator_eval_failure( diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index f2b0e16dac..796d1c999a 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -363,6 +363,7 @@ pub async fn check_tag_available_for_workspace_internal( && custom_tags_per_w .global .iter() + .chain(custom_tags_per_w.specific.keys()) .any(|t| t.contains("$workspace")) { tag_workspace.await @@ -371,21 +372,35 @@ pub async fn check_tag_available_for_workspace_internal( }; // A job whose tag is written exactly as an entry resolves inside what that entry admits, which // is the only way in for an entry `custom_tag_matches` cannot turn into a pattern. - let mut is_tag_in_workspace_custom_tags = custom_tags_per_w.global.iter().any(|entry| { + let admits = |entry: &str| { entry == tag || resolved_tag.is_some_and(|r| custom_tag_matches(entry, r, &tag_workspace)) - }); - if !is_tag_in_workspace_custom_tags { - if let Some(specific_tag) = custom_tags_per_w.specific.get(resolved_tag.unwrap_or(tag)) { - // Only a fork-scoped tag can match through the lineage, so every other tag keeps the - // ancestor lookup off the push path entirely. - let chain = if specific_tag.is_fork_scoped() { - workspace_with_fork_ancestors(db, w_id).await? - } else { - vec![w_id.to_string()] - }; - is_tag_in_workspace_custom_tags = specific_tag.applies_to_workspace(&chain); - } - } + }; + let scoped = custom_tags_per_w + .specific + .iter() + .filter(|(name, _)| admits(name)) + .collect::>(); + // Only a fork-scoped tag can match through the lineage, so every other tag keeps the + // ancestor lookup off the push path entirely. + let chain = if scoped.iter().any(|(_, t)| t.is_fork_scoped()) { + workspace_with_fork_ancestors(db, w_id).await? + } else { + vec![w_id.to_string()] + }; + // A tag listed by its own name with a scope stays inside that scope: a pattern it fits does + // not open it to other workspaces. Only listing it by name without a scope does. + let lands_on = resolved_tag.unwrap_or(tag); + let confined_by = custom_tags_per_w + .specific + .get_key_value(lands_on) + .filter(|(name, t)| { + !name.contains('$') + && !t.applies_to_workspace(&chain) + && !custom_tags_per_w.global.iter().any(|e| e == lands_on) + }); + let is_tag_in_workspace_custom_tags = confined_by.is_none() + && (custom_tags_per_w.global.iter().any(|entry| admits(entry)) + || scoped.iter().any(|(_, t)| t.applies_to_workspace(&chain))); match is_tag_in_scope_tags { Some(true) | None => { @@ -411,6 +426,32 @@ pub async fn check_tag_available_for_workspace_internal( if *TAGS_ARE_SENSITIVE { return Err(Error::BadRequest(format!("{tag} is not available to you"))); + } else if let Some((name, confining)) = confined_by { + let overridden = custom_tags_per_w + .global + .iter() + .filter(|entry| admits(entry)) + .cloned() + .chain( + scoped + .iter() + .filter(|(_, t)| t.applies_to_workspace(&chain)) + .map(|(n, t)| t.authored(n)), + ) + .collect::>(); + let overridden = if overridden.is_empty() { + String::new() + } else { + format!( + ", which takes precedence over the custom tags it also fits: {}", + overridden.join(", ") + ) + }; + return Err(Error::BadRequest(format!( + "Tag {tag} is not available in workspace {w_id}: the custom tag {} restricts it \ + to other workspaces{overridden}", + confining.authored(name) + ))); } else { return Err(error::Error::BadRequest(format!( "Tag {tag} is not included in the allowed CUSTOM_TAGS, which only super admins can go beyond: {:?}", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 548dbe07b4..ad0a5068e8 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -41,6 +41,8 @@ use crate::{ #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] pub struct CustomTags { pub global: Vec, + /// Keyed by the entry's name, which may hold placeholders like a global entry: only a name + /// without any can be looked up by the tag it admits, the others through [`custom_tag_matches`]. pub specific: HashMap, } @@ -83,20 +85,7 @@ impl CustomTags { } else { self.specific .iter() - .map(|(tag, tag_data)| { - let separator = tag_data.tag_type.corresponding_separator(); - let mut workspaces = tag_data - .workspaces - .iter() - .map(|w| w.to_string()) - .collect::>() - .join(&*separator.to_string()); - if tag_data.tag_type == SpecificTagType::AllExcluding { - // the AllExcluding tag syntax has a leading separator - workspaces.insert(0, separator); - } - format!("{}({})", tag, workspaces) - }) + .map(|(tag, tag_data)| tag_data.authored(tag)) .collect::>() }; let all_tags = self.global.clone(); @@ -261,6 +250,22 @@ impl SpecificTagData { pub fn is_fork_scoped(&self) -> bool { self.workspaces.iter().any(|w| w.include_forks) } + + /// The entry named `name` with this scope, as written in the custom tags: `tag(ws1+ws2)`. + pub fn authored(&self, name: &str) -> String { + let separator = self.tag_type.corresponding_separator(); + let mut workspaces = self + .workspaces + .iter() + .map(|w| w.to_string()) + .collect::>() + .join(&*separator.to_string()); + if self.tag_type == SpecificTagType::AllExcluding { + // the AllExcluding tag syntax has a leading separator + workspaces.insert(0, separator); + } + format!("{}({})", name, workspaces) + } } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum SpecificTagType { @@ -542,7 +547,9 @@ lazy_static::lazy_static! { - // ^([\w-]+) # Group 1: tag name + // ^( # Group 1: tag name, a pattern when it holds placeholders + // (?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+ + // ) // \( # Literal '(' // ( # Group 2: the full workspace list // (?:[\w-]+\*?\+)*[\w-]+\*? # NoneExcept pattern: ws1+ws2* @@ -551,8 +558,9 @@ lazy_static::lazy_static! { // ) // \)$ # Closing ')' // - // The optional `*` after each workspace id is the fork marker, see [`WorkspaceMatcher`]. - static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); + // The placeholders are CUSTOM_TAG_PLACEHOLDER's. The optional `*` after each workspace id is + // the fork marker, see [`WorkspaceMatcher`]. + static ref CUSTOM_TAG_REGEX: Regex = Regex::new(r"^((?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$").unwrap(); // The placeholders a job's tag is resolved from when it is pushed, see [`custom_tag_matches`]. static ref CUSTOM_TAG_PLACEHOLDER: Regex = Regex::new(r"\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\]").unwrap(); @@ -3185,6 +3193,58 @@ mod tests { assert!(!matches("$workspace-$args[size]", "ws2-large")); } + #[test] + fn test_scoped_custom_tag_patterns_parse_and_round_trip() { + let input = vec![ + "gpu-$args[size](ws1+ws2)".to_string(), + "cpu-$flow_expr[results.a.size](^ws1)".to_string(), + "$workspace-$args[x](prod*)".to_string(), + "t-$args[id]-$args[id](ws1)".to_string(), + ]; + let tags = CustomTags::from(input.clone()); + + assert!(tags.global.is_empty()); + assert_eq!( + tags.specific["gpu-$args[size]"], + SpecificTagData { + tag_type: SpecificTagType::NoneExcept, + workspaces: vec![matcher("ws1"), matcher("ws2")], + } + ); + assert_eq!( + tags.specific["cpu-$flow_expr[results.a.size]"], + SpecificTagData { + tag_type: SpecificTagType::AllExcluding, + workspaces: vec![matcher("ws1")], + } + ); + assert!(tags.specific["$workspace-$args[x]"].is_fork_scoped()); + assert!(tags.specific.contains_key("t-$args[id]-$args[id]")); + + let mut result = tags.to_string_vec(None); + result.sort(); + let mut expected = input; + expected.sort(); + assert_eq!(result, expected); + + let mut for_ws1 = tags.to_string_vec(Some(&chain(&["ws1"]))); + for_ws1.sort(); + assert_eq!(for_ws1, vec!["gpu-$args[size]", "t-$args[id]-$args[id]"]); + + // Only the placeholders a tag resolves from make a name a pattern: anything else is no + // scoped entry at all. + for literal in [ + "gpu-$foo(ws1)", + "gpu-$args[](ws1)", + "gpu-$args[a.](ws1)", + "gpu.x(ws1)", + ] { + let tags = CustomTags::from(vec![literal.to_string()]); + assert_eq!(tags.global, vec![literal]); + assert!(tags.specific.is_empty()); + } + } + #[test] fn test_fork_marker_parses_and_round_trips() { let tags = CustomTags::from(vec![ diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index f385ff2b49..fd5859b784 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -39,7 +39,8 @@ const dispatch = createEventDispatcher() // Mirrors CUSTOM_TAG_REGEX in backend/windmill-common/src/worker.rs — keep both in sync. - const customTagRegex = /^([\w-]+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ + const customTagRegex = + /^((?:[\w-]|\$workspace|\$(?:args|flow_expr)\[(?:\w+\.)*\w+\])+)\(((?:[\w-]+\*?\+)*[\w-]+\*?|(?:\^[\w-]+\*?)+)\)$/ // Mirrors RE_ARG_TAG and RE_FLOW_EXPR_TAG in backend/windmill-queue/src/jobs.rs. const dynamicTagRegex = /\$(args|flow_expr)\[((?:\w+\.)*\w+)\]/ @@ -47,10 +48,16 @@ return w.includeForks ? `${w.id} (and its forks)` : w.id } + // What an entry matches job tags against: a workspace specific entry's name, which may be + // dynamic like any other entry. + function nameOf(entry: string) { + return entry.match(customTagRegex)?.[1] ?? entry + } + let tagName = $derived(nameOf(newTag.trim())) + let dynamicTag = $derived.by(() => { - let r = newTag.trim() - if (r == '') return undefined - let matched = r.match(dynamicTagRegex) + if (tagName == '') return undefined + let matched = tagName.match(dynamicTagRegex) return matched ? { kind: matched[1], path: matched[2] } : undefined }) @@ -88,8 +95,8 @@ placeholders[placeholders.length - 1].end == entry.length ) } - let dynamicTagTied = $derived(isTied(newTag.trim())) - let dynamicTagReachesAnyTag = $derived(reachesAnyTag(newTag.trim())) + let dynamicTagTied = $derived(isTied(tagName)) + let dynamicTagReachesAnyTag = $derived(reachesAnyTag(tagName)) let extractedCustomTag = $derived.by(() => { let r = newTag.trim() @@ -164,10 +171,40 @@ -{#snippet reachesAnyTagWarning()} - Nothing fixed at its start or its end, so it allows anyone to reach almost any tag. We highly - recommend a prefix or a suffix, ideally both, to limit its reach: for instance - gpu-$args[size] or gpu-$args[size]-eu. +{#snippet reachesAnyTagWarning(scoped: boolean)} + Nothing fixed at its start or its end, so it allows anyone{scoped ? ' in its workspaces' : ''} to reach + almost any tag. We highly recommend a prefix or a suffix, ideally both, to limit its reach: for instance + gpu-$args[size] + or gpu-$args[size]-eu. +{/snippet} + +{#snippet dynamicTagInfo(scoped: boolean)} + {#if tagName.includes('$workspace')} +
Interpolated tag based on workspace id the job was created in
+ {/if} + {#if dynamicTag?.kind == 'flow_expr'} +
+ Interpolated tag based on the flow value at {dynamicTag.path}, resolved when the flow + step starts +
+ {:else if dynamicTag} +
Interpolated tag based on args input of {dynamicTag.path}
+ {/if} + {#if dynamicTagTied} +
+ Its placeholders read the same value, or one reads inside the other, so it allows only jobs + whose tag is written exactly like this +
+ {:else if dynamicTagReachesAnyTag} +
+ {@render reachesAnyTagWarning(scoped)} +
+ {:else if dynamicTag} +
+ Allows any tag it resolves to{scoped ? ' in these workspaces' : ''}, with the text around the + placeholder as written +
+ {/if} {/snippet}
{#each customTags as customTag} + {@const name = nameOf(customTag)} {customTag} - {#if reachesAnyTag(customTag)} + {#if reachesAnyTag(name)} - {@render reachesAnyTagWarning()} + {@render reachesAnyTagWarning(name != customTag)}
{/snippet} @@ -229,8 +267,11 @@ {#if extractedCustomTag} + {@const dynamic = tagName.includes('$workspace') || dynamicTag != undefined}
-
Workspace specific tag
+
+ Workspace specific {#if dynamic}dynamic tag{:else}tag{/if} +
Tag: {extractedCustomTag.tag} @@ -243,6 +284,20 @@ All workspaces except {extractedCustomTag.workspaces?.map(formatWorkspace).join(', ')} {/if}
+ {@render dynamicTagInfo(true)} + + {#if !dynamic && tagEditor} + {#if (customTags ?? []).includes(tagName)} +
+ {tagName} is also listed without workspaces, which keeps it open to every + workspace +
+ {:else} +
+ Workspaces outside this scope cannot use it, even through a dynamic tag it fits +
+ {/if} + {/if}
{:else if newTag.trim()} {#if newTag.includes('(') || newTag.includes(')') || newTag.includes('+') || newTag.includes('^') || newTag.includes('*') || ((newTag.includes('.') || newTag.includes('$args[') || newTag.includes('$flow_expr[')) && !dynamicTag)} @@ -266,31 +321,7 @@ Tag: {newTag.trim()} - {#if newTag.includes('$workspace') && !dynamicTag} -
Interpolated tag based on workspace id the job was created in
- {/if} - {#if dynamicTag?.kind == 'flow_expr'} -
- Interpolated tag based on the flow value at {dynamicTag.path}, resolved when - the flow step starts -
- {:else if dynamicTag} -
Interpolated tag based on args input of {dynamicTag.path}
- {/if} - {#if dynamicTagTied} -
- Its placeholders read the same value, or one reads inside the other, so it allows only - jobs whose tag is written exactly like this -
- {:else if dynamicTagReachesAnyTag} -
- {@render reachesAnyTagWarning()} -
- {:else if dynamicTag} -
- Allows any tag it resolves to, with the text around the placeholder as written -
- {/if} + {@render dynamicTagInfo(false)} {/if} {/if} @@ -348,7 +379,11 @@ A dynamic tag is checked on the tag it resolves to: it is allowed when that tag is listed here, or fits a listed dynamic tag such as 
gpu-$args[size]
, which allows any tag starting with - 
gpu-
. + 
gpu-
. Scope it like any other tag, e.g. + 
gpu-$args[size](workspace1+workspace2)
, to allow those + tags only in some workspaces. A tag listed by its own name for some workspaces, e.g. + 
gpu-secret(workspace1)
, stays limited to them even + when a dynamic tag fits it, unless it is also listed without workspaces. {/if} From 7d0f2ce3bf920a68f1ea22d59481d239947f1b51 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 11:32:11 +0200 Subject: [PATCH 15/34] fix: size the AI chat output budget by model, not provider (#11286) * fix: size the AI chat output budget by model, not provider Co-Authored-By: Claude Opus 5 (1M context) * fix: keep self-hosted open-weight models on the fallback output budget Co-Authored-By: Claude Opus 5 (1M context) * chore: point ee-repo-ref at the free tier output clamp change Co-Authored-By: Claude Opus 5 (1M context) * fix: treat codestral as open-weight for the output budget Co-Authored-By: Claude Opus 5 (1M context) * chore: update ee-repo-ref to 559a9ff78ea03439553cf2b412e73765762e9013 This commit updates the EE repository reference after PR #821 was merged in windmill-ee-private. Previous ee-repo-ref: 953f7c1cc9a70b740d5d1b129cb1401950854b02 New ee-repo-ref: 559a9ff78ea03439553cf2b412e73765762e9013 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../copilot/chat/AIChatManager.svelte.ts | 4 +- .../lib/components/copilot/chat/anthropic.ts | 3 +- .../copilot/chat/openai-responses.ts | 6 +- .../copilot/chat/outputTokenLimit.ts | 5 + .../components/copilot/lib.toolCalls.test.ts | 36 ++++++- frontend/src/lib/components/copilot/lib.ts | 100 ++++++++++-------- .../src/lib/components/copilot/modelConfig.ts | 65 ++++++++++++ .../components/copilot/reasoningRegistry.ts | 17 ++- .../workspaceSettings/AISettings.svelte | 4 +- 10 files changed, 185 insertions(+), 57 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c336c4fdd3..b82e25d93b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5370ae3a95a3dc10171f72da74286a654996eee6 +559a9ff78ea03439553cf2b412e73765762e9013 diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 871dead4bc..27f9bfc43f 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -110,7 +110,7 @@ import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { closeInterruptedToolBatch, runChatLoop, truncateToToolPairedPrefix } from './chatLoop' -import { OutputTokenLimitError } from './outputTokenLimit' +import { FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE, OutputTokenLimitError } from './outputTokenLimit' import { sanitizeToolCallArguments } from './toolCallArguments' import { billedTokens, normalizeContextUsage, type ChatTokenUsage } from './tokenUsage' import { logAiUsage } from '$lib/utils/aiUsageReporter' @@ -378,7 +378,7 @@ function isImageRejection(err: unknown, models: (string | undefined)[] = []): bo function getSendRequestErrorMessage(err: unknown, webSearchUnavailable: boolean): string { // The request went through; only its response was cut short. if (err instanceof OutputTokenLimitError) { - return err.message + return get(copilotInfo).freeTier ? FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE : err.message } const errorMessage = err instanceof Error ? err.message : typeof err === 'string' ? err : undefined diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index a311d16990..122425e264 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -141,7 +141,8 @@ export async function getAnthropicCompletion( const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) let anthropicTools = convertOpenAIToolsToAnthropic(tools) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index b216a08107..0e460f1921 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -272,7 +272,8 @@ export async function getOpenAIResponsesCompletion( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = applyReasoningToConfig( @@ -331,7 +332,8 @@ export async function* getOpenAIResponsesCompletionStream( messages, stream: true, tools, - forceModelProvider: options?.forceModelProvider + forceModelProvider: options?.forceModelProvider, + reasoningEffort: options?.reasoningEffort }) const { instructions, input } = convertMessagesToResponsesInput(messages) // No prompt cache key here: a rejected key has to be retried without it, and this is diff --git a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts index b1127a1db7..19b74f0c68 100644 --- a/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts +++ b/frontend/src/lib/components/copilot/chat/outputTokenLimit.ts @@ -10,3 +10,8 @@ export class OutputTokenLimitError extends Error { this.name = 'OutputTokenLimitError' } } + +/** Shown instead on Windmill's free tier, whose server clamps the output below any + * workspace setting: pointing at that setting would send the user to a no-op. */ +export const FREE_TIER_OUTPUT_TOKEN_LIMIT_MESSAGE = + "The response was cut off at the free tier's output token limit. Ask it to continue, or add your own API key in the workspace AI settings for a higher limit." diff --git a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts index b011c1f672..6966e018d1 100644 --- a/frontend/src/lib/components/copilot/lib.toolCalls.test.ts +++ b/frontend/src/lib/components/copilot/lib.toolCalls.test.ts @@ -287,10 +287,42 @@ describe('parseOpenAICompletion tool call arguments', () => { }) describe('output token limit', () => { - it("gives DeepSeek DeepSeek's own thinking-mode budget rather than the 8192 fallback", async () => { + it('follows the model through every host that serves it', async () => { const { getModelMaxTokens } = await import('./lib') expect(getModelMaxTokens('deepseek', 'deepseek-flash')).toBe(131072) - expect(getModelMaxTokens('deepseek', 'deepseek-v4-pro')).toBe(131072) + expect(getModelMaxTokens('azure_foundry', 'DeepSeek-V4-Pro')).toBe(131072) + expect(getModelMaxTokens('openrouter', 'deepseek/deepseek-v4-pro')).toBe(131072) + // gpt-oss reasons whether or not the chat sends an effort. + expect(getModelMaxTokens('groq', 'openai/gpt-oss-120b')).toBe(16000) + expect(getModelMaxTokens('openrouter', 'openai/o3')).toBe(100000) + // A legacy `/thinking` selection is its base model, not the `thinking` row. + expect(getModelMaxTokens('anthropic', 'claude-sonnet-4-5/thinking')).toBe(64000) + }) + + it('gives a model outside the table room to think only when the request asks it to', async () => { + const { getModelMaxTokens } = await import('./lib') + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'high')).toBe(32768) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3', 'none')).toBe(8192) + expect(getModelMaxTokens('openrouter', 'x-ai/grok-4.3')).toBe(8192) + // The registry does not know Custom AI models, so an effort says nothing about them. + expect(getModelMaxTokens('customai', 'local-model', 'high')).toBe(8192) + // A self-hosted open-weight model runs at its operator's context length, not the + // vendor's, while a Custom AI proxy in front of a hosted model keeps its row. + expect(getModelMaxTokens('customai', 'deepseek-v4-pro', 'high')).toBe(8192) + expect(getModelMaxTokens('customai', 'gpt-5')).toBe(128000) + }) + + it('sizes the budget from the effort the same request carries', async () => { + const { getCompletion } = await import('./lib') + const create = vi.fn() + + await getCompletion([], new AbortController(), undefined, { + forceModelProvider: { provider: 'mistral', model: 'mistral-medium-latest' }, + openaiClient: { chat: { completions: { create } } } as any, + reasoningEffort: 'high' + }) + + expect(create.mock.calls[0][0]).toMatchObject({ max_tokens: 32768, reasoning_effort: 'high' }) }) it('fails a response cut off at the output token limit instead of ending the turn', async () => { diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 5094764807..b55aa06c92 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -15,11 +15,17 @@ import { get, type Writable } from 'svelte/store' import { OpenAPI, ResourceService, type Script } from '../../gen' import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { + getKnownModelMaxOutputTokens, requiresMaxCompletionTokens, usesAnthropicMessagesApi, usesOpenRouterPromptCaching } from './modelConfig' -import { applyReasoningToConfig } from './reasoningRegistry' +import { + applyReasoningToConfig, + requestsReasoning, + stripLegacyThinkingSuffix, + type ReasoningEffort +} from './reasoningRegistry' import { formatResourceTypes } from './utils' import { appendPendingToolImages, @@ -310,45 +316,31 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -export function getModelMaxTokens(provider: AIProvider, model: string) { - if (provider === 'deepseek') { - // DeepSeek thinks by default and counts the thinking toward max_tokens, so - // the 8192 fallback cuts long turns off mid-thought. 131072 is DeepSeek's - // own default at the highest effort (65536 at the default one). - return 131072 - } else if (model.includes('gpt-5')) { - return 128000 - } else if ( - (provider === 'azure_openai' || provider === 'openai' || provider === 'azure_foundry') && - model.startsWith('o') - ) { - return 100000 - } else if ( - // Raising this further would also raise the worst case of the - // non-streaming completion path, which the Anthropic SDK refuses once - // the request could run past ~10 minutes. - model.includes('claude-sonnet') || - model.includes('claude-haiku') || - model.includes('claude-fable') || - model.includes('claude-mythos') || - // Opus only from 4.5 on. Opus 4.1 and older cap at 32K and fall through - // to the row below. Dots are normalized because OpenRouter writes - // `anthropic/claude-opus-4.5` where Anthropic writes `claude-opus-4-5`. - /claude-opus-(4-(5|6|7|8)|5)(?!\d)/.test(model.replace(/\./g, '-')) || - model.includes('gemini-2.5') || - model.includes('gemini-3') - ) { - return 64000 - } else if (model.includes('gpt-4.1')) { - return 32768 - } else if (model.includes('claude-opus')) { - return 32000 - } else if (model.includes('gpt-4o') || model.includes('codestral')) { - return 16384 - } else if (model.includes('gpt-4-turbo') || model.includes('gpt-3.5')) { - return 4096 - } - return 8192 +// Thinking counts toward max_tokens, so a model outside the table that is asked to +// reason gets more room than the plain fallback. A host rejects a budget above the +// model's cap, so the reasoning fallback must stay within the cap of every model it +// reaches (true of every reasoning model OpenRouter serves outside the table). Custom +// AI never gets it: the registry does not know its models, and the per-model override +// is its way to a larger budget. +const FALLBACK_MAX_TOKENS = 8192 +const REASONING_FALLBACK_MAX_TOKENS = 32768 + +/** + * The output budget a request sends when the workspace sets no override. `reasoningEffort` + * is the effort the request carries, as `resolveRequestReasoning` resolves it. + */ +export function getModelMaxTokens( + provider: AIProvider, + model: string, + reasoningEffort?: ReasoningEffort +): number { + const bareModel = stripLegacyThinkingSuffix(model) + return ( + getKnownModelMaxOutputTokens(provider, bareModel) ?? + (requestsReasoning(provider, bareModel, reasoningEffort) + ? REASONING_FALLBACK_MAX_TOKENS + : FALLBACK_MAX_TOKENS) + ) } // Resolves the completion token cap for a model: the workspace's per-model @@ -356,8 +348,16 @@ export function getModelMaxTokens(provider: AIProvider, model: string) { // Anthropic request paths so both honor the same limit. `cap` bounds the result // (used by short metadata completions, see METADATA_MAX_TOKENS) — a hard ceiling // that wins over both the workspace override and the default. -function resolveMaxTokens(modelProvider: AIProviderModel, cap?: number): number { - const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) +function resolveMaxTokens( + modelProvider: AIProviderModel, + cap?: number, + reasoningEffort?: ReasoningEffort +): number { + const defaultMaxTokens = getModelMaxTokens( + modelProvider.provider, + modelProvider.model, + reasoningEffort + ) const modelKey = `${modelProvider.provider}:${modelProvider.model}` let customMaxTokensStore: Record | undefined try { @@ -379,9 +379,10 @@ export const METADATA_MAX_TOKENS = 4096 function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[], - maxTokensCap?: number + maxTokensCap?: number, + reasoningEffort?: ReasoningEffort ) { - const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap) + const maxTokens = resolveMaxTokens(modelProvider, maxTokensCap, reasoningEffort) if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' || @@ -903,7 +904,8 @@ export function getProviderAndCompletionConfig({ tools, forceModelProvider, maxTokensCap, - promptCaching + promptCaching, + reasoningEffort }: { messages: ChatCompletionMessageParam[] stream: K @@ -913,6 +915,9 @@ export function getProviderAndCompletionConfig({ // Opt-in: a cache write costs more than an uncached read, so it only pays off where // the same prefix is sent again. True for the chat loop, false for one-shot calls. promptCaching?: boolean + // The effort the caller will add with `applyReasoningToConfig`. Only sizes the + // output budget here: a request that reasons needs room for the thinking. + reasoningEffort?: ReasoningEffort }): { provider: AIProvider config: K extends true @@ -929,7 +934,7 @@ export function getProviderAndCompletionConfig({ provider: modelProvider.provider, config: { ...providerConfig, - ...getModelSpecificConfig(modelProvider, tools, maxTokensCap), + ...getModelSpecificConfig(modelProvider, tools, maxTokensCap, reasoningEffort), messages: processedMessages, stream } as any @@ -1135,7 +1140,8 @@ export async function getCompletion( stream: true, tools, forceModelProvider: options?.forceModelProvider, - promptCaching: options?.promptCaching + promptCaching: options?.promptCaching, + reasoningEffort: options?.reasoningEffort }) // Use Responses API for OpenAI and Azure OpenAI diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index be49907ed4..77d0cf942a 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -117,6 +117,58 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['codestral', 32_000] ] +// Output token budgets, matched like the context windows above so one row covers +// every host of a model: native API, Azure AI Foundry, OpenRouter, Together, Bedrock. +// A host rejects any request above its own limit, so each value is one every host of +// the model accepts, not the vendor's maximum. Models not listed fall back in +// `getModelMaxTokens`. +const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // Anthropic — the Anthropic SDK refuses a non-streaming request whose max_tokens + // implies more than ~10 minutes, so these stay below the models' own maximum. + // Opus caps at 32K before 4.5. + ['claude-opus-4.5', 64_000], + ['claude-opus-4.6', 64_000], + ['claude-opus-4.7', 64_000], + ['claude-opus-4.8', 64_000], + ['claude-opus-5', 64_000], + ['claude-opus', 32_000], + ['claude-sonnet', 64_000], + ['claude-haiku', 64_000], + ['claude-fable', 64_000], + ['claude-mythos', 64_000], + // OpenAI + ['gpt-5', 128_000], + ['gpt-4.1', 32_768], + ['gpt-4o', 16_384], + ['gpt-4-turbo', 4_096], + ['gpt-3.5', 4_096], + ['o1', 100_000], + ['o3', 100_000], + ['o4-mini', 100_000], + // Google + ['gemini-3', 64_000], + ['gemini-2.5', 64_000] +] + +// Open-weight models, which a self-hosted server (Custom AI) caps at whatever context +// length its operator set, often below these: vLLM rejects a request whose prompt plus +// max_tokens exceeds it. So these rows only apply to the providers that host them. +const OPEN_WEIGHT_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ + // gpt-oss reasons on every request. Bedrock documents 16K, where Groq takes 65536 + ['gpt-oss', 16_000], + // DeepSeek — thinks by default and counts the thinking toward max_tokens. 131072 + // is DeepSeek's own default at the highest effort. R1 is capped by its OpenRouter + // host. + ['deepseek-v4', 131_072], + ['deepseek-flash', 131_072], + ['deepseek-r1', 16_000], + // Mistral + ['codestral', 16_384], + // Models named for thinking reason on every request, whether or not the chat sends + // an effort. Every one OpenRouter lists takes at least this much. + ['thinking', 32_768] +] + // Version separators differ by route to the same model: Anthropic writes // `claude-opus-4-8`, OpenRouter writes `anthropic/claude-opus-4.8`. Collapsing // dots to dashes on both sides keeps one table entry covering every route — @@ -197,6 +249,19 @@ export function getKnownModelContextWindow(model: string): number | undefined { return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } +const MODEL_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(MODEL_MAX_OUTPUT_TOKENS) +const OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS = buildModelMatchers(OPEN_WEIGHT_MAX_OUTPUT_TOKENS) + +export function getKnownModelMaxOutputTokens( + provider: AIProvider, + model: string +): number | undefined { + return ( + matchModel(MODEL_MAX_OUTPUT_TOKEN_MATCHERS, model) ?? + (provider === 'customai' ? undefined : matchModel(OPEN_WEIGHT_MAX_OUTPUT_TOKEN_MATCHERS, model)) + ) +} + /** Trim/compaction logic needs a number; assume a conservative window when unknown. */ export const ASSUMED_CONTEXT_WINDOW = 128000 diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 242f6cb329..9915e9c42f 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -374,7 +374,8 @@ const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/ /** * Disable token to forward when the user explicitly turns reasoning off on a * model that reasons *by default* — omitting the field would silently keep - * the default-on behavior. Undefined means omission is the correct off. + * the default-on behavior. Undefined means omission is the correct off. Every + * token is `'none'`: `requestsReasoning` reads that value as off. */ export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { switch (reasoningProviderFamily(provider, model)) { @@ -434,6 +435,20 @@ export function resolveRequestReasoning( return undefined } +/** + * Whether a request carrying `effort`, as `resolveRequestReasoning` resolves it, + * has the model reason. An effort sent to a model the registry does not mark as + * reasoning-capable answers no: an explicit choice is always sent, but that does + * not make the model think. + */ +export function requestsReasoning( + provider: AIProvider, + model: string, + effort: ReasoningEffort | undefined +): boolean { + return !!effort && effort !== 'none' && supportsReasoning(provider, model) +} + export type ReasoningApiKind = 'anthropic' | 'responses' | 'completions' | 'deepseek' | 'mistral' /** diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 3957f47b74..05159e3057 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -21,6 +21,7 @@ getKnownModelContextWindow, getModelContextWindowFromTable } from '../copilot/modelConfig' + import { resolveRequestReasoning } from '../copilot/reasoningRegistry' import { supportsAutocomplete } from '../copilot/utils' import TestAiKey from '../copilot/TestAIKey.svelte' import Label from '../Label.svelte' @@ -650,7 +651,8 @@ min={1} max={2_000_000} getDefault={(provider, model) => ({ - tokens: getModelMaxTokens(provider, model), + // What a chat sends at the default effort; the budget depends on it. + tokens: getModelMaxTokens(provider, model, resolveRequestReasoning({ provider, model })), assumed: false })} /> From 02006f364a4217e967435d9e71d8617c03b65413 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 13:03:30 +0200 Subject: [PATCH 16/34] fix: keep a cancel that lands after a worker last read its queue row (#11246) * fix: keep a cancel that lands after a worker last read its queue row Co-Authored-By: Claude Opus 5 * fix: take the cancellation from the queue row the completion deletes Co-Authored-By: Claude Opus 5 * test: wait for the completion to block on the row instead of a timer Co-Authored-By: Claude Opus 5 * refactor: settle a missed cancel from the deleted queue row alone Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- ...dde0be9e0d192e0aee224bff78d8bd2ccbd8b.json | 16 ++ ...99c2e1e9ca0ed33077ade8d7560e7cc21fa06.json | 22 --- ...933eeff7396430607c02cbd0cc682d5c34d61.json | 23 +++ ...f86505bc499320e08d1aad15a9a82c3823c0b.json | 28 +++ backend/tests/perpetual_cancel_race.rs | 183 ++++++++++++++++++ backend/windmill-queue/src/jobs.rs | 76 ++++++-- 6 files changed, 310 insertions(+), 38 deletions(-) create mode 100644 backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json delete mode 100644 backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json create mode 100644 backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json create mode 100644 backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json create mode 100644 backend/tests/perpetual_cancel_race.rs diff --git a/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json new file mode 100644 index 0000000000..0bc09d9430 --- /dev/null +++ b/backend/.sqlx/query-9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9a2345e35c9f44578aa7dde994fdde0be9e0d192e0aee224bff78d8bd2ccbd8b" +} diff --git a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json b/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json deleted file mode 100644 index 8a4b957c2a..0000000000 --- a/backend/.sqlx/query-c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c92cc71e6d10c41368f7aa75b0799c2e1e9ca0ed33077ade8d7560e7cc21fa06" -} diff --git a/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json new file mode 100644 index 0000000000..9930e0494f --- /dev/null +++ b/backend/.sqlx/query-e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e2760debbb69182f1385bfe8e18933eeff7396430607c02cbd0cc682d5c34d61" +} diff --git a/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json new file mode 100644 index 0000000000..b11e41fe89 --- /dev/null +++ b/backend/.sqlx/query-ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "canceled_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "canceled_reason", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "ed2eb32d445164e7ab5fb735e01f86505bc499320e08d1aad15a9a82c3823c0b" +} diff --git a/backend/tests/perpetual_cancel_race.rs b/backend/tests/perpetual_cancel_race.rs new file mode 100644 index 0000000000..df239eb7ab --- /dev/null +++ b/backend/tests/perpetual_cancel_race.rs @@ -0,0 +1,183 @@ +//! A cancel reaches the queue row first and the worker only later: one landing after the worker's +//! last read of that row completes the run carrying no `canceled_by` of its own, and a perpetual +//! script must still stop rather than restart as if nothing had been cancelled. + +use serde_json::json; +use sqlx::{types::Json, Pool, Postgres}; +use uuid::Uuid; +use windmill_queue::{add_completed_job, get_mini_completed_job}; + +const W_ID: &str = "test-workspace"; +const PATH: &str = "u/test-user/loop"; + +async fn start_perpetual_run(db: &Pool) -> anyhow::Result { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, summary, description, content, created_by, \ + language, lock, restart_unless_cancelled) \ + VALUES ($1, 1, $2, '', '', 'echo', 'test-user', 'bash', '', true)", + ) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + let id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, runnable_id, runnable_path, script_lang, tag, args, \ + visible_to_owner) \ + VALUES ($1, $2, 'test-user', now(), 'u/test-user', 'test@windmill.dev', 'script', 1, \ + $3, 'bash', 'bash', '{\"n\": 1}', true)", + ) + .bind(id) + .bind(W_ID) + .bind(PATH) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, started_at, tag) \ + VALUES ($1, $2, now(), true, now(), 'bash')", + ) + .bind(id) + .bind(W_ID) + .execute(db) + .await?; + Ok(id) +} + +/// Waits for another connection to be stuck on the queue row of `id`, which is where the +/// completion's delete lands while the cancel is uncommitted. +async fn wait_until_blocked_on(db: &Pool, id: Uuid) -> anyhow::Result<()> { + for _ in 0..100 { + let blocked: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM pg_locks l JOIN v2_job_queue q \ + ON q.ctid = ('(' || l.page || ',' || l.tuple || ')')::tid \ + WHERE NOT l.granted AND l.locktype = 'tuple' AND q.id = $1) \ + OR EXISTS (SELECT 1 FROM pg_stat_activity \ + WHERE wait_event_type = 'Lock' AND query LIKE '%v2_job_queue%')", + ) + .bind(id) + .fetch_one(db) + .await?; + if blocked { + return Ok(()); + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + anyhow::bail!("the completion never reached the delete, so the race was not exercised") +} + +/// The same cancel, one step earlier: still being written when the completion reads the queue row, +/// which reads it without a lock. Only the delete that follows waits for the writer. +#[sqlx::test(fixtures("base"))] +async fn a_cancel_still_being_written_is_kept(db: Pool) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + + let mut cancel = db.begin().await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&mut *cancel) + .await?; + + let completing = tokio::spawn({ + let db = db.clone(); + async move { + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + None, + false, + None, + false, + ) + .await + } + }); + // Committing on a timer would let a slow completion read the cancel through the insert + // instead, leaving the half this test is for unexercised and green. Waiting for the delete to + // block on the row is what puts it there. + wait_until_blocked_on(&db, running).await?; + cancel.commit().await?; + completing.await??; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn a_cancel_landing_after_the_worker_read_its_row_is_kept( + db: Pool, +) -> anyhow::Result<()> { + let running = start_perpetual_run(&db).await?; + sqlx::query( + "UPDATE v2_job_queue SET canceled_by = 'test-user', canceled_reason = 'stop' WHERE id = $1", + ) + .bind(running) + .execute(&db) + .await?; + + let job = get_mini_completed_job(&running, W_ID, &db).await?.unwrap(); + add_completed_job( + &db, + &job, + true, + false, + Json(&json!("done")), + None, + 0, + // What the worker carries when the cancel landed after it last read the row. + None, + false, + None, + false, + ) + .await?; + + let (status, canceled_by): (String, Option) = + sqlx::query_as("SELECT status::text, canceled_by FROM v2_job_completed WHERE id = $1") + .bind(running) + .fetch_one(&db) + .await?; + assert_eq!(status, "canceled", "the run is recorded as canceled"); + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + let queued: Vec = sqlx::query_scalar( + "SELECT q.id FROM v2_job_queue q JOIN v2_job j USING (id) \ + WHERE j.workspace_id = $1 AND j.runnable_path = $2", + ) + .bind(W_ID) + .bind(PATH) + .fetch_all(&db) + .await?; + assert!( + queued.is_empty(), + "a canceled perpetual run queues no next one: {queued:?}" + ); + Ok(()) +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 9eae01c476..50eb9c8b1f 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1285,7 +1285,25 @@ async fn commit_completed_job( } let mut _skip_downstream_error_handlers = false; - tx = delete_job(tx, &job_id).warn_after_seconds(10).await?; + let (ntx, canceled_at_delete) = delete_job(tx, &job_id).warn_after_seconds(10).await?; + tx = ntx; + // `canceled_by` is only what the worker last read from the queue row. Deleting the row waits + // for a cancel still being written, so the row it removed is the final word on whether this + // job was canceled. + if canceled_by.is_none() { + if let Some(canceled) = canceled_at_delete { + sqlx::query!( + "UPDATE v2_job_completed SET status = 'canceled'::job_status, canceled_by = $2, \ + canceled_reason = $3 WHERE id = $1", + job_id, + canceled.username, + canceled.reason, + ) + .execute(&mut *tx) + .warn_after_seconds(10) + .await?; + } + } // tracing::error!("3 {:?}", start.elapsed()); if completed_job.is_flow_step() { @@ -1611,6 +1629,22 @@ async fn restart_job_if_perpetual_inner( }; if restart { + // Not `canceled_by`: a worker reads the queue row on a widening interval, up to every 5s + // once a job has run for a minute, so a cancel landing after its last read reaches a + // completion carrying none. `commit_completed_job` records it from the queue row it + // deletes, so the completed row is what says whether this loop was stopped. + let canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed \ + WHERE id = $1 AND workspace_id = $2", + queued_job.id, + &queued_job.workspace_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if canceled { + return Ok(()); + } let tx = PushIsolationLevel::IsolatedRoot(db.clone()); // perpetual jobs can run one job per 10s max. If the job was faster than 10s, schedule the next one with the appropriate delay @@ -5153,34 +5187,44 @@ async fn extract_result_from_job_result( } } +/// Also reports the cancellation the deleted row carried, if any. Unlike a plain read of the queue +/// row, this waits for a cancel that is still being written, so it is the last word on one. pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, job_id: &Uuid, -) -> windmill_common::error::Result> { +) -> windmill_common::error::Result<(Transaction<'c, Postgres>, Option)> { #[cfg(feature = "prometheus")] if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { QUEUE_DELETE_COUNT.inc(); } otel_incr_queue_delete_count(); - let job_removed = - sqlx::query_scalar!("DELETE FROM v2_job_queue WHERE id = $1 RETURNING 1", job_id,) - .fetch_optional(&mut *tx) - .await; + let job_removed = sqlx::query!( + "DELETE FROM v2_job_queue WHERE id = $1 RETURNING canceled_by, canceled_reason", + job_id, + ) + .fetch_optional(&mut *tx) + .await; - if let Err(job_removed) = job_removed { - tracing::error!( - "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" - ); - } else { - let job_removed = job_removed.unwrap().flatten().unwrap_or(0); - if job_removed != 1 { - tracing::error!("Job {job_id} could not be deleted, returned not 1: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + let canceled = match &job_removed { + Err(job_removed) => { + tracing::error!( + "Job {job_id} could not be deleted: {job_removed}. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling" + ); + None } - } + Ok(None) => { + tracing::error!("Job {job_id} could not be deleted, no row was removed. This is not necessarily an error, as the job might have been deleted by another process such as in the case of cancelling"); + None + } + Ok(Some(row)) => row.canceled_by.as_ref().map(|username| CanceledBy { + username: Some(username.clone()), + reason: row.canceled_reason.clone(), + }), + }; tracing::debug!("Job {job_id} deleted"); - Ok(tx) + Ok((tx, canceled)) } pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result { From 732214361f36eb07d65854118bd053c96ce681fb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 13:17:58 +0200 Subject: [PATCH 17/34] fix: stop the raw app editor stealing focus in ai sessions (#11287) * fix: preserve raw app editor focus during ai file updates * fix: sync ai file edits through the existing files effect * docs: explain raw app file synchronization dependency * fix: avoid echoing raw app files back into the editor * fix: synchronize raw app file operations through one effect * fix: stop session draft sync from re-applying the editor's own edits Co-Authored-By: Claude Opus 5 (1M context) * fix: open raw app files without taking focus and follow idle ai edits Co-Authored-By: Claude Opus 5 (1M context) * fix: keep selecting raw app documents that live outside the app files Co-Authored-By: Claude Opus 5 (1M context) * fix: tie a pending raw app editor focus request to the picked file Co-Authored-By: Claude Opus 5 (1M context) * chore: pin ui_builder to 60b6904 Co-Authored-By: Claude Opus 5 (1M context) * fix: always send explicit raw app file replacements to the editor Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Guilhem Lemouel Co-authored-by: Claude Opus 5 (1M context) --- frontend/scripts/ui_builder_artifact.json | 4 +- .../components/raw_apps/RawAppEditor.svelte | 106 ++++++++++++------ .../sessions/useUserDraftSync.svelte.ts | 9 +- 3 files changed, 79 insertions(+), 40 deletions(-) diff --git a/frontend/scripts/ui_builder_artifact.json b/frontend/scripts/ui_builder_artifact.json index 87ce066c87..68c1c558b1 100644 --- a/frontend/scripts/ui_builder_artifact.json +++ b/frontend/scripts/ui_builder_artifact.json @@ -1,5 +1,5 @@ { "baseUrl": "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev", - "version": "1522e43", - "sha256": "0c44db3a52e02e6530860d0e9e9031fcdad7a09a98897b6c97e6f522c1d4754c" + "version": "60b6904", + "sha256": "85b68e8418c97d96f3a419b3e345ef8a1987976943b8439a7df841f9217a9b34" } diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 9a5eb001ec..78d3ba8102 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -566,7 +566,10 @@ // The only writer of `activeTabId`, `selectedRunnable` and `selectedDocument`. // Everything switches through here, so the three can't disagree: one left // stale marks two sidebar rows selected, or leaves a tab over an empty pane. - function select(next: EditorSelection, opts?: { force?: boolean; notifyIframe?: boolean }): void { + function select( + next: EditorSelection, + opts?: { force?: boolean; notifyIframe?: boolean; focus?: boolean } + ): void { if (next.kind === 'preview') { // In split mode Preview is always shown on the right, so selecting it is // a no-op (would collapse the left pane). `force` lets closeTab fall back @@ -575,21 +578,23 @@ activeTabId = PREVIEW_TAB_ID selectedRunnable = undefined selectedDocument = undefined + iframeFocusPending = undefined return } if (next.kind === 'file') { activeTabId = ensureFileTab(next.path) selectedRunnable = undefined selectedDocument = next.path - if (opts?.notifyIframe !== false) openInIframe(next.path) + if (opts?.notifyIframe !== false) openInIframe(next.path, opts?.focus) return } activeTabId = ensureRunnableTab(next.key) selectedDocument = undefined selectedRunnable = next.key + iframeFocusPending = undefined } - function activateTab(id: string, opts?: { force?: boolean }) { + function activateTab(id: string, opts?: { force?: boolean; focus?: boolean }) { if (!tabs.some((t) => t.id === id)) return select(selectionOfTab(id), opts) } @@ -602,13 +607,20 @@ closeTab(runnableTabId(key)) } - // Ask the UI Builder iframe to open a document. `populateFiles` replays - // `iframeDocument` on iframe load, so record it even when the iframe isn't - // ready yet (the postMessage is then skipped). - function openInIframe(path: string) { + // Ask the UI Builder iframe to open a document, taking the keyboard only when + // the user picked it. For a file whose current content the iframe doesn't hold + // yet, `populateFiles` opens `iframeDocument` in the same message as that + // content: opening first errors on a new file and races the write on a changed + // one (the iframe applies the edit twice). It also opens it on iframe load. + // Documents outside `files` (wmill.ts, ui/, node_modules/) never go through it. + function openInIframe(path: string, focus = false) { iframeDocument = path - if (iframeLoaded) { - iframe?.contentWindow?.postMessage({ type: 'selectFile', path }, '*') + const content = files?.[path] + if (iframeLoaded && (content === undefined || iframeFiles?.[path] === content)) { + iframeFocusPending = undefined + iframe?.contentWindow?.postMessage({ type: 'selectFile', path, focus }, '*') + } else { + iframeFocusPending = focus ? path : undefined } } @@ -730,6 +742,7 @@ summary = update.summary } if (update.files !== undefined) { + iframeFiles = undefined files = update.files } if (update.runnables !== undefined) { @@ -810,6 +823,17 @@ } let iframeLoaded = $state(false) // @hmr:keep + // The files the iframe holds: last posted to it, or last reported by it. Explicit + // replacements (history, YAML) clear it so they are always sent: the iframe can + // hold edits it hasn't reported yet, which a skip would leave in place. + let iframeFiles: Record | undefined + // The document the user picked that `openInIframe` left for `populateFiles` to + // open. It takes the keyboard only if it is still the document being opened. + let iframeFocusPending: string | undefined + // Last keystroke or click in the editor, or edit reported by the iframe. AI + // edits only move the editor to their file once the user has paused this long. + let lastUserInputAt = 0 + const USER_IDLE_MS = 3000 // Briefly drops the `setActiveDocument` echo VS Code fires while we're // pushing the initial file set — the iframe auto-opens a default editor // during boot which we don't want to treat as a user-driven activation. @@ -857,6 +881,10 @@ function populateFiles() { if (files) { + // `files` is reassigned with unchanged content (the session draft sync, history + // restores, the iframe's own reports). Re-sending it makes the iframe reopen its + // document and rewrite the files under the user's cursor. + if (deepEqual(files, iframeFiles)) return suppressSetActiveDocument = true if (suppressTimer !== undefined) clearTimeout(suppressTimer) suppressTimer = setTimeout(() => { @@ -875,10 +903,14 @@ } } function setFilesInIframe(newFiles: Record) { + const target = iframe?.contentWindow + if (!target) return + iframeFiles = { ...newFiles } + iframeFocusPending = undefined const files = Object.fromEntries( Object.entries(newFiles).filter(([path, _]) => !path.endsWith('/')) ) - iframe?.contentWindow?.postMessage( + target.postMessage( { type: 'setFiles', files: files @@ -889,14 +921,20 @@ function setFilesAndSelectInIframe(newFiles: Record, pathToSelect: string) { iframeDocument = pathToSelect + const target = iframe?.contentWindow + if (!target) return + iframeFiles = { ...newFiles } + const focus = iframeFocusPending === pathToSelect + iframeFocusPending = undefined const files = Object.fromEntries( Object.entries(newFiles).filter(([path, _]) => !path.endsWith('/')) ) - iframe?.contentWindow?.postMessage( + target.postMessage( { type: 'setFilesAndSelect', files: files, - pathToSelect: pathToSelect + pathToSelect: pathToSelect, + focus }, '*' ) @@ -1012,14 +1050,15 @@ return frontendFiles }, setFrontendFile: (path, content): LintResult => { - console.log('setting frontend file', path, content) + console.log('setting frontend file', path, `${content.length} chars`) if (!files) { files = {} } files[path] = content - // Combined setFilesAndSelect avoids a race, so let it do the telling. - select({ kind: 'file', path }, { notifyIframe: false }) - setFilesAndSelectInIframe(files, path) + // Follow the AI to the file it edits, but never pull a user who is editing + // off their file. The files effect sends the content. + if (Date.now() - lastUserInputAt < USER_IDLE_MS) ensureFileTab(path) + else select({ kind: 'file', path }) return lint() }, deleteFrontendFile: (path) => { @@ -1027,7 +1066,6 @@ files = {} } delete files[path] - setFilesInIframe(files) }, listBackendRunnables: () => { return Object.entries(runnables).map(([key, runnable]) => ({ @@ -1399,9 +1437,11 @@ if (e.data.type === 'setFiles') { // Normalize Windows-style path separators to Linux-style const normalizedFiles = normalizeFilePaths(e.data.files) + iframeFiles = { ...normalizedFiles } // Only mark pending changes if files actually changed (ignore echo from setFilesInIframe) if (!deepEqual(files, normalizedFiles)) { files = normalizedFiles + lastUserInputAt = Date.now() historyManager.markPendingChanges() } } else if (e.data.type === 'getBundle') { @@ -2015,6 +2055,7 @@ } $effect(() => { iframe?.addEventListener('load', () => { + iframeFiles = undefined iframeLoaded = true }) }) @@ -2080,6 +2121,7 @@ } }) $effect(() => { + // populateFiles must track file entries so in-place AI edits trigger synchronization. iframe && iframeLoaded && files && populateFiles() }) $effect(() => { @@ -2108,7 +2150,7 @@ // future `FileExplorer` caller feeds folder paths back in. function handleSelectPath(path: string) { if (!path || path.endsWith('/')) return - select({ kind: 'file', path }) + select({ kind: 'file', path }, { focus: true }) } // Track previous values for change detection @@ -2202,19 +2244,12 @@ data: RawAppData }) { try { + iframeFiles = undefined files = structuredClone($state.snapshot(entry.files)) runnables = structuredClone($state.snapshot(entry.runnables)) summary = entry.summary replaceData(structuredClone($state.snapshot(entry.data))) - // If the open document survives into the new files, use the combined message - if (iframeDocument && isOpenableDocument(iframeDocument)) { - // Use combined setFilesAndSelect message to avoid race condition - setFilesAndSelectInIframe(entry.files, iframeDocument) - } else { - // Otherwise just set files normally - setFilesInIframe(entry.files) - } populateRunnables() } catch (error) { console.error('Failed to apply entry:', error) @@ -2351,7 +2386,12 @@ gateJobIds={false} extraSourceWindow={() => externalPreviewWindow} /> -
+
(lastUserInputAt = Date.now())} + onpointerdowncapture={() => (lastUserInputAt = Date.now())} + class="max-h-full overflow-hidden h-full min-h-0 flex flex-col" +> files, - (newFiles) => { - files = newFiles - setFilesInIframe(newFiles ?? {}) - } - } + bind:files onSelectPath={handleSelectPath} onSelectRunnable={(key) => select({ kind: 'runnable', key })} onDeleteRunnable={deleteRunnable} @@ -2495,7 +2529,7 @@ activateTab(id)} + onSelect={(id) => activateTab(id, { focus: true })} onClose={(id) => closeTab(id)} onReorder={(next) => reorderTabs(next)} > @@ -2580,7 +2614,7 @@ activateTab(id)} + onSelect={(id) => activateTab(id, { focus: true })} onClose={(id) => closeTab(id)} onReorder={(next) => reorderTabs(next)} > diff --git a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts index 7e89d3b945..64f7f7e6c2 100644 --- a/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts +++ b/frontend/src/lib/components/sessions/useUserDraftSync.svelte.ts @@ -94,7 +94,10 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void // re-apply only advances the sig, never reverts an edit or fires a save. if (sig === lastInboundSig) return lastInboundSig = sig - codec.applyDraftToStore(incoming) + // Copies both ways: a store sharing proxies with the draft makes every local + // edit re-fire this effect as if a draft came in, reassigning the store on + // each keystroke. + codec.applyDraftToStore($state.snapshot(incoming) as Draft) }) }) @@ -125,7 +128,9 @@ export function useUserDraftSync(opts: UserDraftSyncOptions): void untrack(() => { const current = UserDraft.get(codec.itemKind, path, { workspace }) if (current && codec.sig(current) === sig) return - const toSave = codec.storeToDraft(current) ?? draft + const toSave = $state.snapshot(codec.storeToDraft(current) ?? draft) as Draft + // Our own write comes back through the handle; the inbound side must skip it. + lastInboundSig = codec.sig(toSave) UserDraft.save(codec.itemKind, path, toSave, { workspace }) }) } From 841f7e1436ca1217ffde16f5d70c9b49ef60be13 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 14:13:11 +0200 Subject: [PATCH 18/34] fix(cli): sync workspace dependency files on Windows and fix CLI tests (#11288) * fix(cli): sync workspace dependency files on Windows Co-Authored-By: Claude Opus 5 (1M context) * test(cli): pin workspace dependency path parsing on both separators Co-Authored-By: Claude Opus 5 (1M context) * fix(cli): classify workspace dependency files on either separator Co-Authored-By: Claude Opus 5 (1M context) * test(cli): check out the fork deploy clone without autocrlf Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- cli/src/commands/sync/sync.ts | 4 ++-- cli/src/types.ts | 2 +- cli/src/utils/script_common.ts | 4 +++- cli/src/utils/utils.ts | 3 ++- cli/test/gitsync_deploy_branch_config.test.ts | 5 +++- cli/test/sync_pull_push.test.ts | 12 ++++++---- cli/test/workspace_deps_path_unit.test.ts | 23 +++++++++++++++++++ 7 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 cli/test/workspace_deps_path_unit.test.ts diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 32cff4a3cb..6185decc5a 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1153,7 +1153,7 @@ export function ZipFSElement( ? "script" : p.endsWith(".resource.json") ? "resource" - : p.startsWith("dependencies/") + : isWorkspaceDependencies(p) ? "dependencies" : "other"; @@ -2752,7 +2752,7 @@ export async function compareDynFSElement( if (skipMetadata) { continue; } - if (k.startsWith("dependencies/")) { + if (isWorkspaceDependencies(k)) { if (!workspaceDependenciesPathToLanguageAndFilename(k)) { log.warn(`Skipping unrecognized workspace dependencies file: ${k}`); continue; diff --git a/cli/src/types.ts b/cli/src/types.ts index 5713d06390..866b2be21e 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -391,7 +391,7 @@ export function getTypeStrFromPath( if (isSharedLockPath(p)) { return "shared_lock"; } - if (p.startsWith("dependencies" + SEP)) { + if (isWorkspaceDependencies(p)) { return "workspace_dependencies"; } if (isFileResource(p) || isFilesetResource(p)) { diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index d804318491..611268c27c 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -45,7 +45,9 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [ ] as const; export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined { - const relativePath = path.replace("dependencies/", ""); + // Sync paths carry the platform separator, while a set's name is always + // forward-slashed. + const relativePath = path.replaceAll("\\", "/").replace("dependencies/", ""); for (const { filename, language } of workspaceDependenciesLanguages) { if (relativePath.endsWith(filename)) { return { diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 0801c03911..526dffbde4 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -223,8 +223,9 @@ export function isRawAppFile(path: string): boolean { return isRawAppPath(path); } +/** Sync paths carry the platform separator: `dependencies\x` on Windows. */ export function isWorkspaceDependencies(path: string): boolean { - return path.startsWith("dependencies/"); + return path.startsWith("dependencies/") || path.startsWith("dependencies\\"); } export function printSync(input: string | Uint8Array) { diff --git a/cli/test/gitsync_deploy_branch_config.test.ts b/cli/test/gitsync_deploy_branch_config.test.ts index 39c435d872..eeab7a0120 100644 --- a/cli/test/gitsync_deploy_branch_config.test.ts +++ b/cli/test/gitsync_deploy_branch_config.test.ts @@ -63,7 +63,10 @@ test("git-sync fork deploy follows the fork branch's wmill.yaml", async () => { git(seed, "commit", "-m", "dedupe"); git(seed, "push", "origin", forkBranch); - git(work, "clone", `file://${bare}`, "."); + // Git for Windows defaults to autocrlf=true, which checks the pulled LF + // files out as CRLF; the deploy's pull would rewrite every one of them + // and the clean-tree assertion below would read that as a change. + git(work, "clone", "-c", "core.autocrlf=false", `file://${bare}`, "."); const deployed = await backend.runCLICommand( [ "sync", diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 24c4a282a3..fa52510e6c 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -2185,18 +2185,20 @@ describe("settings sync", () => { const files = await listFilesRecursive(tempDir); expect(files).toContain("settings.yaml"); - // Read and modify a safe setting (webhook URL) + // Read and modify a safe setting (webhook URL). The server resolves the + // webhook host when it is saved and refuses one that does not resolve to a + // public address, so this must be a real public hostname. + const webhook = "https://example.com/hook"; const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8"); let modifiedSettings: string; if (settingsContent.includes("webhook:")) { modifiedSettings = settingsContent.replace( /webhook:.*/, - 'webhook: "https://test-webhook.example.com/hook"' + `webhook: "${webhook}"` ); } else { - modifiedSettings = - settingsContent + '\nwebhook: "https://test-webhook.example.com/hook"\n'; + modifiedSettings = settingsContent + `\nwebhook: "${webhook}"\n`; } await writeFile(`${tempDir}/settings.yaml`, modifiedSettings, "utf-8"); @@ -2213,7 +2215,7 @@ describe("settings sync", () => { ); expect(apiResp.status).toEqual(200); const settingsData = await apiResp.json(); - expect(settingsData.webhook).toEqual("https://test-webhook.example.com/hook"); + expect(settingsData.webhook).toEqual(webhook); }); }); }); diff --git a/cli/test/workspace_deps_path_unit.test.ts b/cli/test/workspace_deps_path_unit.test.ts new file mode 100644 index 0000000000..45d725c6e5 --- /dev/null +++ b/cli/test/workspace_deps_path_unit.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from "bun:test"; +import { isWorkspaceDependencies } from "../src/utils/utils.ts"; +import { workspaceDependenciesPathToLanguageAndFilename } from "../src/utils/script_common.ts"; + +// Sync paths carry the platform separator. A Windows path parsed to a different +// set name pushes the file to a set the workspace does not have. +test("workspace dependency paths parse the same with either separator", () => { + for (const sep of ["/", "\\"]) { + expect(isWorkspaceDependencies(`dependencies${sep}requirements.in`)).toBe(true); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}requirements.in`), + ).toEqual({ name: undefined, language: "python3" }); + expect( + workspaceDependenciesPathToLanguageAndFilename(`dependencies${sep}team.package.json`), + ).toEqual({ name: "team", language: "bun" }); + expect( + workspaceDependenciesPathToLanguageAndFilename( + `dependencies${sep}team${sep}python.requirements.in`, + ), + ).toEqual({ name: "team/python", language: "python3" }); + } + expect(isWorkspaceDependencies("f\\dependencies\\requirements.in")).toBe(false); +}); From 25c8bfaab412d440e51be92af61066af025d334b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 22 Sep 2026 14:13:32 +0200 Subject: [PATCH 19/34] chore(main): release 1.817.0 (#11268) * chore(main): release 1.817.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- .release-please-manifest.json | 2 +- CHANGELOG.md | 24 +++ backend/Cargo.lock | 166 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- chat-sdk/package-lock.json | 4 +- chat-sdk/package.json | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- windmill-yaml-validator/package-lock.json | 4 +- windmill-yaml-validator/package.json | 2 +- 22 files changed, 154 insertions(+), 130 deletions(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 438887d2eb..6dc5574962 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.816.0" + ".": "1.817.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index e7e039fb45..595426d619 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Changelog +## [1.817.0](https://github.com/windmill-labs/windmill/compare/v1.816.0...v1.817.0) (2026-09-22) + + +### Features + +* add workspace mentions to AI composer ([#11264](https://github.com/windmill-labs/windmill/issues/11264)) ([e7ba595](https://github.com/windmill-labs/windmill/commit/e7ba5954b69514d72c140f9b1e697fb2236df23e)) +* allow workspace-scoped dynamic custom tags ([#11283](https://github.com/windmill-labs/windmill/issues/11283)) ([efa7a0a](https://github.com/windmill-labs/windmill/commit/efa7a0a70af43f8827542059dd450326a9b88d62)) +* check dynamic worker tags on the tag they resolve to ([#11271](https://github.com/windmill-labs/windmill/issues/11271)) ([69fafb9](https://github.com/windmill-labs/windmill/commit/69fafb9262f90184e6c2dc9a84f1c3feeb9d9638)) +* deploy from the UI to a workspace on another instance ([#11245](https://github.com/windmill-labs/windmill/issues/11245)) ([4d12ea4](https://github.com/windmill-labs/windmill/commit/4d12ea461454f89367bd91daa7e8c39a89035507)) + + +### Bug Fixes + +* add missing credential fields to settings redaction lists ([#11272](https://github.com/windmill-labs/windmill/issues/11272)) ([2a40f25](https://github.com/windmill-labs/windmill/commit/2a40f25bda81d41b9f578338c85656744f59ebfc)) +* apply SSRF validation to workspace webhook URLs ([#11285](https://github.com/windmill-labs/windmill/issues/11285)) ([586699c](https://github.com/windmill-labs/windmill/commit/586699c4938b5203920a43e69a706310078b93f5)) +* apply token path scopes to the native trigger list ([#11281](https://github.com/windmill-labs/windmill/issues/11281)) ([faf7b22](https://github.com/windmill-labs/windmill/commit/faf7b22be0827738c90c1f3ed6cd3e214d6c63d3)) +* check workflow-as-code task tags against CUSTOM_TAGS ([#11273](https://github.com/windmill-labs/windmill/issues/11273)) ([2699d50](https://github.com/windmill-labs/windmill/commit/2699d5065ea3dfe985636733fb6022ca23187291)) +* clear the impersonation token from session storage on logout ([#11274](https://github.com/windmill-labs/windmill/issues/11274)) ([9936673](https://github.com/windmill-labs/windmill/commit/9936673ee62fb6dd929a654a6138aeca687c0fb7)) +* enforce token path scopes on GET /raw_apps/list ([#11284](https://github.com/windmill-labs/windmill/issues/11284)) ([9ad2c91](https://github.com/windmill-labs/windmill/commit/9ad2c91ddb661ab60ada9ec4039ef8d025602969)) +* keep a cancel that lands after a worker last read its queue row ([#11246](https://github.com/windmill-labs/windmill/issues/11246)) ([02006f3](https://github.com/windmill-labs/windmill/commit/02006f364a4217e967435d9e71d8617c03b65413)) +* size the AI chat output budget by model, not provider ([#11286](https://github.com/windmill-labs/windmill/issues/11286)) ([7d0f2ce](https://github.com/windmill-labs/windmill/commit/7d0f2ce3bf920a68f1ea22d59481d239947f1b51)) +* stop cutting DeepSeek chat turns off mid-thought ([#11282](https://github.com/windmill-labs/windmill/issues/11282)) ([5821700](https://github.com/windmill-labs/windmill/commit/58217006ece56bbaf0447af67cac487e72b928c7)) +* stop the raw app editor stealing focus in ai sessions ([#11287](https://github.com/windmill-labs/windmill/issues/11287)) ([7322143](https://github.com/windmill-labs/windmill/commit/732214361f36eb07d65854118bd053c96ce681fb)) + ## [1.816.0](https://github.com/windmill-labs/windmill/compare/v1.815.0...v1.816.0) (2026-09-21) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 82c625df67..e0bfce842b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -928,9 +928,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "aws-config" -version = "1.8.3" +version = "1.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0baa720ebadea158c5bda642ac444a2af0cdf7bb66b46d1e4533de5d1f449d0" +checksum = "8a8fc176d53d6fe85017f230405e3255cedb4a02221cb55ed6d76dccbbb099b2" dependencies = [ "aws-credential-types", "aws-runtime", @@ -938,8 +938,8 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http 0.62.6", - "aws-smithy-json 0.61.9", + "aws-smithy-http 0.63.4", + "aws-smithy-json 0.62.4", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -14792,7 +14792,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -14880,7 +14880,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.816.0" +version = "1.817.0" dependencies = [ "async-stream", "async-trait", @@ -14914,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14927,7 +14927,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "argon2", @@ -15067,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15090,7 +15090,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15107,7 +15107,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15133,7 +15133,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.816.0" +version = "1.817.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15143,7 +15143,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15160,7 +15160,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15182,7 +15182,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15205,7 +15205,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15221,7 +15221,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15265,7 +15265,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15279,7 +15279,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -15314,7 +15314,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15339,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15367,7 +15367,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15389,7 +15389,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15409,7 +15409,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15447,7 +15447,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15476,7 +15476,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.816.0" +version = "1.817.0" dependencies = [ "lazy_static", "serde", @@ -15488,7 +15488,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.816.0" +version = "1.817.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15512,7 +15512,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15526,7 +15526,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.816.0" +version = "1.817.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15564,7 +15564,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.816.0" +version = "1.817.0" dependencies = [ "chrono", "lazy_static", @@ -15578,7 +15578,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15597,7 +15597,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.816.0" +version = "1.817.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15704,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.816.0" +version = "1.817.0" dependencies = [ "chrono", "futures", @@ -15724,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.816.0" +version = "1.817.0" dependencies = [ "regex", "serde", @@ -15741,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15768,7 +15768,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15785,7 +15785,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.816.0" +version = "1.817.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15801,7 +15801,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15822,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -15853,7 +15853,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "arc-swap", @@ -15878,7 +15878,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-stream", @@ -15913,7 +15913,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "futures", @@ -15931,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.816.0" +version = "1.817.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15940,7 +15940,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15952,7 +15952,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -15964,7 +15964,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -15976,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -15988,7 +15988,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -16000,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -16011,7 +16011,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16022,7 +16022,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16034,7 +16034,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16045,7 +16045,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16067,7 +16067,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -16079,7 +16079,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16093,7 +16093,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16110,7 +16110,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16123,7 +16123,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -16135,7 +16135,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16153,7 +16153,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16169,7 +16169,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16185,7 +16185,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -16199,7 +16199,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16238,7 +16238,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "const_format", @@ -16278,7 +16278,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.816.0" +version = "1.817.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16289,7 +16289,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -16324,7 +16324,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16348,7 +16348,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16381,7 +16381,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16408,7 +16408,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16441,7 +16441,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16461,7 +16461,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16495,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16531,7 +16531,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16554,7 +16554,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16578,7 +16578,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-nats", @@ -16602,7 +16602,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16637,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16665,7 +16665,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-trait", @@ -16690,7 +16690,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16709,7 +16709,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-once-cell", @@ -16827,7 +16827,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.816.0" +version = "1.817.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a911d57b81..74da8a4244 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.816.0" +version = "1.817.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.816.0" +version = "1.817.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index dcd846c0d8..ccba27bcc5 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.816.0" +version = "1.817.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.816.0" +version = "1.817.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.816.0" +version = "1.817.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.816.0" +version = "1.817.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1eacbd24d3..0244b7b106 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.816.0" +version = "1.817.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 46c498de4c..0dd9f8620e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.816.0 + version: 1.817.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 7f0649a2ab..1e6cfdd474 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.816.0"; +export const VERSION = "v1.817.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index e657c139d7..ff470d4ffe 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.816.0", + "version": "1.817.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index a157a5175a..5655adc64e 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.816.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 45eb4e941f..9728b6edd7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.816.0"; +export const VERSION = "1.817.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 072729566c..611cd0c91f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index f535994371..46db11793f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.816.0", + "version": "1.817.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 418c7a8f8e..79d2672f4a 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.816.0" +wmill = ">=1.817.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a5c0d33b62..48f573bab1 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.816.0 + version: 1.817.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 9117e1bf7d..ad6c0e8bc1 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.816.0' + ModuleVersion = '1.817.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index b493b81c41..baab12d832 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.816.0" +version = "1.817.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index c72327a9a7..26441041a1 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.816.0", + "version": "1.817.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index afa9e10871..4badfa307b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.816.0", + "version": "1.817.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 198c13b19c..2edd9ebec3 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.816.0 +1.817.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index cb1c272906..e07061650d 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index f0dffa300a..01bb378dcd 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.816.0", + "version": "1.817.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts", From 7dec3d97717406d078499ea6509cdb00c89c3480 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:15:05 +0200 Subject: [PATCH 20/34] feat(db-manager): add a schema diagram view (#11030) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(datatables): put a data table's connection under Postgres roles A data table backed by the instance database resolved to exactly one Postgres connection, `custom_instance_user`, for everyone who could reach it at all. There was no way to say this job reads, that one writes, this one never sees the salaries table. A data table role is now a real Postgres login on the cluster, defined once for the instance by a superadmin and named exactly as they named it. A script that declares `-- role analytics` connects as `analytics`, and Postgres decides what it may touch — grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`. A data table with no `permissions` block behaves exactly as before. Everything that opens a connection on someone's behalf goes through one chokepoint, `get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when there is none. The role logs in as itself — never `SET ROLE`, which a script could `RESET ROLE` its way out of. A fork's data table entry becomes a pointer at the workspace that governs it rather than a copy of it. The settings clone used to hand a fork a byte-identical entry naming the parent's database, which a fork admin could edit to grant themselves `admin` there; a pointer has nothing local to edit, and its tenants are evaluated as a member of the governing workspace, by email. `permissions` is stripped from the workspace export and ignored on import: tenants name principals of one workspace, and a settings push is not where an access decision should be made. Operations that see the whole database whatever the roles grant stay with the governing workspace's admins: editing the roles, a migration that declares none, and opening a replication stream for a Postgres trigger or capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): gate the paths that reach a whole database as admin Auditing what still resolved through the unchecked resolver turned up three that act for a caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema export, the full-schema read, database creation, import and the forked-database drop), the connection test, and the schema snapshot a fork clone takes of its parent. On a data table under roles each let any workspace member — or a fork admin who is nobody in the governing workspace — read or copy the whole database whatever its roles grant. All three now require admin reach on the governing workspace. A dump taken under a restricted role would be a silently truncated copy rather than an error, so refusing is the only right answer for the copy paths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin putting a *resource-backed* data table under roles, at which point the executor dialled the host that resource names — one the admin chose — with the role's real cluster password, and `CONNECT` is granted to every registered instance database. Both ends now refuse: the permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials on a non-instance entry rather than trusting the record it read. Two more places reached the governing database without answering to it. The initial-migration generator returned a `pg_dump` of the whole schema to any member. And the migration rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after which the parent re-runs every migration from zero. The remote half is now skipped when the entry resolves into another workspace, which is also just correct: a fork renaming what it calls a data table changes nothing about the data table. Also: revoking a tenant now bounces the replication streams of every workspace holding an entry that resolves here, not only the governing one, so a fork's trigger stops rather than living on inside its open connection; the instance role catalog and the governing workspace's tenant lists are no longer returned to someone who cannot edit them; and the tenant rename dedup collapses non-adjacent duplicates, per role rather than once any role changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): fail loudly where a role or a pointer can be left half-recorded Three ways the feature could end up in a state nobody could see or undo. Creating a role writes the cluster first and the catalog second, but the catalog write was an `UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a live login with a password nobody recorded: invisible to the catalog, un-recreatable because the name is taken, and un-deletable because there is no entry to delete. It now errors, so the operation is retryable once the row is restored. Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy would hand each fork the database outright — so the delete now names the data tables it stranded, and resolving one says which workspace is missing rather than reporting a data table this workspace never had. `InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now hand-written so `{:?}` on the catalog cannot put a live credential in a log line. Adds the two branches the reviews found unpinned: a caller who is not a member of the governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that predates this and sends no job id. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): unbreak two operator messages and two comments that described other code The two strings this branch added for states an operator hits once — the catalog write that matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence. `list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop it, and the comment now says why that is the right place to stay quiet. The non-superadmin check in `edit_datatable_config` was introduced as also covering references, which it does not and need not: `reference` is overwritten from the stored entry for every caller before the check runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): serialize role catalog mutations, and state each helper's authorization contract The catalog is one JSON document, so create, rename, enable and delete are all read-modify-write. Two concurrent creates read the same snapshot, both succeed in the cluster, and the second write drops the first — leaving a live Postgres login with a password nobody recorded, which is the exact state the delete path exists to prevent. Every mutation now runs in one transaction holding an advisory lock across the read, the cluster DDL and the write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL helpers take that transaction rather than the pool, which is what makes the lock cover them. Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely multi-statement SQL, and its future is not `Send`, which an axum handler holding the transaction requires. Each of these is one statement anyway. The new cross-crate surface now says what callers must do. `read_role_catalog` returns plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and `converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's raw config. All of them are superadmin-gated by their current handlers, but nothing said so at the definition, which is where the next caller looks. Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip that did not land; the rename affordance is the design-system `Button`, not a raw one; and `resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing resolves as the data table's default until the database manager grows a picker. Why role passwords stay a plain `String` while the instance user's password beside them is a `StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an operator supplies it and may want it from their own backend, while these are minted here and never entered by anyone, so there is nothing for a ref to point at. Encrypting generated secrets at rest is a separate change that would take the replication password with it. Now said at the field. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): give the role catalog its own row, out of reach of the config machinery Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways. The catalog serializes a generated Postgres password per role, and that row is the operator-facing instance config, so the passwords reached `get_instance_config` and its YAML editor — a live cluster credential in a response body, a UI field and any log of either. Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of that key writes the row back without it and the catalog is gone, while the cluster keeps every login it described. `custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret, written only by the server, never operator-authored, hidden so the config machinery cannot read, rewrite or drop it. The catalog is the same thing, so it now has the same shape: `datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist. No redaction to keep in step with three code paths, and no way for a neighbouring write to take it out. Two races on the same shared documents. `edit_datatable_config` read the stored data tables outside its transaction and then wrote the whole `datatable` document, so a permissions save committing in between was silently rolled back; it now reads under `FOR UPDATE`. And `set_datatable_permissions` validated role ids against the catalog before opening its transaction, so a deletion in between let it write a deleted role back — including as the default, which every later job then fails on; it now holds the catalog lock and the settings row across validation and write. Completes the authorization contracts the previous commit claimed but did not finish: `read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole job is to answer for a workspace the caller may not belong to, and `converge_connect_grants_with`, which had not inherited its wrapper's. Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but `extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so code written from that page passed a keyword-only argument positionally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all three handlers left it green — it pinned Postgres, not the code it was written for. It now drives `create_datatable_role` twice concurrently and asserts the catalog kept both names. Checked the way the last one should have been: removing the lock from the handler makes it fail with "wmtest_a_… is a live cluster login the catalog forgot". The contracts added last commit were stricter than this PR's own callers, which is worse than none — the next reader sees a rule already broken and learns to ignore it. `read_role_catalog` said superadmin-only while two of its four callers are open to any workspace member, and `converge_connect_grants` said superadmin while `set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the rule that actually holds is about the credential never reaching a response, log, audit record or export, not about who may call. They now say that. `read_datatable_entry` gets the same treatment rather than the one the earlier message claimed for it: it is the primitive every resolution goes through, so it is deliberately open, and what must not escape is `permissions` — it names the governing workspace's users, groups and folders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): close the last ways a role or a pointer can be left pointing at nothing The raw settings readers hand back whatever is in the row, so moving the catalog into its own `global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles` and the settings listing returning every live password. Both now filter that one key. The neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it predates this and widening the fix to it is a decision about an operator workflow, not a consequence of this change. Three ways a save could leave something resolving to nothing: A permissioned data table could be moved to a PostgreSQL resource. The block was carried across as a server-owned field, the runtime refuses roles on a resource-backed table, so the save succeeded and every job afterwards failed. Refused instead — turning roles off first is one step, and it keeps discarding an access decision something somebody chose. Renaming a governing data table left every fork pointing at the old name: the data table disappears from their pickers and their jobs stop, with nothing in the renaming workspace to suggest why. The rename now follows into the pointers in the same transaction. Deleting one cannot be followed the same way, so it is reported instead — the response names what it stranded, the way deleting a workspace does, and the fork's own error already says which workspace is gone. Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions handler reaches it as a workspace admin (the same class fixed last commit, one instance missed); the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog` was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its roles only on success — a failing run is exactly the one that creates them without recording them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * refactor(datatables): put the role catalog in its own table, not in global_settings Five findings across three rounds were all the same choice. A set of live Postgres credentials was living in `global_settings`, which has generic read, list, write, config-export and CLI round-trip paths that know nothing about what they carry: the passwords reached the instance config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog, `GET /settings/global/{key}` and the settings listing returned them raw, and this round the redaction that fixed the last two turned `wmill instance push` into something that wipes every password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles` could also empty it outside the lock. The approved plan offered a table or `global_settings`, so this is the other option it already allowed rather than a new design. `datatable_role` is a table: no generic settings path can read it, list it, export it, write it or round-trip it, so none of the five needs a guard. The redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it. One row per role also removes the read-modify-write the concurrency work was about: two concurrent creates are two inserts, and the unique index on `name` is what settles a collision. The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to another transaction until commit, so without it both creates pass their `pg_roles` check. Also from this round: rename mappings are checked against the configuration they claim to describe, since fork pointers are rewritten from them — a caller could otherwise submit `main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning naming forks a delete stranded reached the response but not the screen: both the data table settings save and the workspace delete now show it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): validate a rename against the save it describes, and re-check under the locks Three from the round, all about deciding on state that could already have moved. A permission save resolved the data table and checked it was instance-backed before taking any lock, then wrote under one. A config save committing in between could move the table onto a PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in which case the write targeted a key that no longer existed and reported success having changed nothing. It now re-resolves and re-checks on the locked state. Rename validation checked that the source existed before and the target existed after, which still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows onto a different data table, silently, because it keeps resolving. The rule is now the actual old-to-new key transition — a source may only survive if another rename took its name, and a target may only pre-exist if another rename freed it. That also stops two sources sharing one target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so a swap cannot be done one save at a time, and refusing it was a regression against main. The pointer cascade now runs in two passes through a temporary name, the way the migration cascade one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once from what it named before the save. The tenant mutators say what they are for: they write an access decision for any workspace named, with an arbitrary mutation, and exist for the transaction that frees or renames a principal. Editing a decision on purpose belongs in the permissions endpoint. Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole document, so a workspace whose pointers name something else is not rewritten to a byte-identical value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from the JSON document, one of which would emit `pwd`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call `/workspaces/leave` — a different handler in a different crate with the same name — which deleted the membership and left `u/` in the tenant lists. Leaving and rejoining therefore restored the access the leave was supposed to end, and a later account taking the username would have inherited it. The regression test drives the route the client actually calls; without the fix it fails with "leaving kept the tenant". The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's admin connection, created `_wm_migrations` and read it before reaching the per-migration role check — so with nothing pending, nothing was checked at all. Rollback returned before its check when nothing was applied, and the status endpoint had none. All three now ask, before any connection is opened, whether the caller can reach the data table as any role at all; which role a given migration runs as is still decided per migration, and by the executor after that. Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now runs in the same transaction, so the drop, the row and every tenant list commit together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse to copy a data table that is under roles pg_dump carries no roles and the import runs with --no-privileges, so a copied data table arrives owned by the admin connection with no GRANT for any role. The settings clone brings `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were given, and are denied by Postgres on everything: an entry that reads as configured and answers nothing. Refuse the copy — in the import endpoint before any data moves, and in the fork path the CLI takes. Replaying the source's owners and ACLs into the clone is what lifts this, and is a change of its own. Dropping `permissions` from the copy instead would be the unsafe half, since the copy holds the parent's rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse the clone's database too, not only its data A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only the second refused a data table under roles, so a fork asking to clone one created and registered an empty `wm_fork_…` instance database and then failed — and nothing collects it, since `drop_forked_datatable_databases` only drops entries carrying `forked_from` and no entry names this one. Refuse in both, so the clone stops before a database exists. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * nit worker error msg * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): data table roles in the DB manager and raw apps Co-Authored-By: Claude Opus 5 (1M context) * fix: never add a role to the reference of a data table whose name contains '?' Co-Authored-By: Claude Opus 5 (1M context) * fix: read the roles of a data table whose name contains '?' The generated client leaves a '?' in a path param unencoded, so the lookup 404'd and the raw-app picker blocked Start on such a data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): refuse a reference naming both a legacy data table and a role When a workspace stores both `sales` and a legacy `sales?role=analytics`, the reference resolved to the legacy entry without a role, so browsing `sales` as `analytics` reached another data table. Co-Authored-By: Claude Opus 5 (1M context) * fix: declare the default role in migrations written for a data table whose name contains '?' Such a data table connects as its default role without naming it, so the migrations the manager wrote for it declared no role and ran as admin. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * feat(datatables): put a data table's connection under Postgres roles A data table backed by the instance database resolved to exactly one Postgres connection, `custom_instance_user`, for everyone who could reach it at all. There was no way to say this job reads, that one writes, this one never sees the salaries table. A data table role is now a real Postgres login on the cluster, defined once for the instance by a superadmin and named exactly as they named it. A script that declares `-- role analytics` connects as `analytics`, and Postgres decides what it may touch — grants are ordinary SQL. Windmill answers only "may this caller ask for this role", from the tenant lists on the data table entry: `u/alice`, `g/analysts`, `f/finance` or `*`. A data table with no `permissions` block behaves exactly as before. Everything that opens a connection on someone's behalf goes through one chokepoint, `get_datatable_resource_from_db`, which takes the identity explicitly and fails closed when there is none. The role logs in as itself — never `SET ROLE`, which a script could `RESET ROLE` its way out of. A fork's data table entry becomes a pointer at the workspace that governs it rather than a copy of it. The settings clone used to hand a fork a byte-identical entry naming the parent's database, which a fork admin could edit to grant themselves `admin` there; a pointer has nothing local to edit, and its tenants are evaluated as a member of the governing workspace, by email. `permissions` is stripped from the workspace export and ignored on import: tenants name principals of one workspace, and a settings push is not where an access decision should be made. Operations that see the whole database whatever the roles grant stay with the governing workspace's admins: editing the roles, a migration that declares none, and opening a replication stream for a Postgres trigger or capture. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): gate the paths that reach a whole database as admin Auditing what still resolved through the unchecked resolver turned up three that act for a caller and hand back the admin connection: `resolve_pg_source_checked` (behind schema export, the full-schema read, database creation, import and the forked-database drop), the connection test, and the schema snapshot a fork clone takes of its parent. On a data table under roles each let any workspace member — or a fork admin who is nobody in the governing workspace — read or copy the whole database whatever its roles grant. All three now require admin reach on the governing workspace. A dump taken under a restricted role would be a silently truncated copy rather than an error, so refusing is the only right answer for the copy paths. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): confine roles to the instance database, and stop a fork reaching the parent's bookkeeping A data table role is a login on Windmill's own Postgres. Nothing stopped a workspace admin putting a *resource-backed* data table under roles, at which point the executor dialled the host that resource names — one the admin chose — with the role's real cluster password, and `CONNECT` is granted to every registered instance database. Both ends now refuse: the permissions endpoint rejects the save, and the chokepoint refuses to substitute credentials on a non-instance entry rather than trusting the record it read. Two more places reached the governing database without answering to it. The initial-migration generator returned a `pg_dump` of the whole schema to any member. And the migration rename/delete cascade followed a fork's pointer into the parent, so a fork admin renaming or removing their own local entry relabelled or wiped the parent's `_wm_migrations` — after which the parent re-runs every migration from zero. The remote half is now skipped when the entry resolves into another workspace, which is also just correct: a fork renaming what it calls a data table changes nothing about the data table. Also: revoking a tenant now bounces the replication streams of every workspace holding an entry that resolves here, not only the governing one, so a fork's trigger stops rather than living on inside its open connection; the instance role catalog and the governing workspace's tenant lists are no longer returned to someone who cannot edit them; and the tenant rename dedup collapses non-adjacent duplicates, per role rather than once any role changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): fail loudly where a role or a pointer can be left half-recorded Three ways the feature could end up in a state nobody could see or undo. Creating a role writes the cluster first and the catalog second, but the catalog write was an `UPDATE` that matched nothing when the instance Postgres settings row was absent — leaving a live login with a password nobody recorded: invisible to the catalog, un-recreatable because the name is taken, and un-deletable because there is no entry to delete. It now errors, so the operation is retryable once the row is restored. Deleting a workspace only nulls the fork lineage; the data table entries pointing at it are left resolving to nothing. Sweeping them is not an option — turning a pointer back into a copy would hand each fork the database outright — so the delete now names the data tables it stranded, and resolving one says which workspace is missing rather than reporting a data table this workspace never had. `InstanceDatatableRole` derived `Debug` while holding a Postgres password; it is now hand-written so `{:?}` on the catalog cannot put a live credential in a log line. Adds the two branches the reviews found unpinned: a caller who is not a member of the governing workspace at all, and `NoIdentity` — the compatibility path for an agent worker that predates this and sends no job id. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): unbreak two operator messages and two comments that described other code The two strings this branch added for states an operator hits once — the catalog write that matched nothing, and the delete that stranded a pointer — were collapsed from their multi-line form with the indentation left in, so both rendered with a fourteen-space gap mid-sentence. `list_datatables` claimed to report a chain it cannot follow and then dropped it; it does drop it, and the comment now says why that is the right place to stay quiet. The non-superadmin check in `edit_datatable_config` was introduced as also covering references, which it does not and need not: `reference` is overwritten from the stored entry for every caller before the check runs. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): serialize role catalog mutations, and state each helper's authorization contract The catalog is one JSON document, so create, rename, enable and delete are all read-modify-write. Two concurrent creates read the same snapshot, both succeed in the cluster, and the second write drops the first — leaving a live Postgres login with a password nobody recorded, which is the exact state the delete path exists to prevent. Every mutation now runs in one transaction holding an advisory lock across the read, the cluster DDL and the write, so a lost update cannot happen and a failure rolls the whole thing back. The DDL helpers take that transaction rather than the pool, which is what makes the lock cover them. Their statements moved off `sqlx::raw_sql`: the simple protocol is only needed for genuinely multi-statement SQL, and its future is not `Send`, which an axum handler holding the transaction requires. Each of these is one statement anyway. The new cross-crate surface now says what callers must do. `read_role_catalog` returns plaintext credentials; `create`/`rename`/`set_login`/`drop_instance_role` and `converge_connect_grants` mutate cluster-wide state; `read_datatable_entry` reads a workspace's raw config. All of them are superadmin-gated by their current handlers, but nothing said so at the definition, which is where the next caller looks. Also: the roles table reloads after a failed login toggle instead of leaving it claiming a flip that did not land; the rename affordance is the design-system `Button`, not a raw one; and `resolve_datatable_pg_as_caller` drops a `role` parameter no caller ever filled — browsing resolves as the data table's default until the database manager grows a picker. Why role passwords stay a plain `String` while the instance user's password beside them is a `StringOrSecretRef`, asked three times across reviews: that one is a secret ref because an operator supplies it and may want it from their own backend, while these are minted here and never entered by anyone, so there is nothing for a ref to point at. Encrypting generated secrets at rest is a separate change that would take the replication password with it. Now said at the field. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): give the role catalog its own row, out of reach of the config machinery Putting it inside `custom_instance_pg_databases` was the wrong call, and it cost two ways. The catalog serializes a generated Postgres password per role, and that row is the operator-facing instance config, so the passwords reached `get_instance_config` and its YAML editor — a live cluster credential in a response body, a UI field and any log of either. Worse in the other direction: `to_settings_map` strips the catalog, so a full-row upsert of that key writes the row back without it and the catalog is gone, while the cluster keeps every login it described. `custom_instance_replication_pwd` is the precedent and says exactly why — a generated secret, written only by the server, never operator-authored, hidden so the config machinery cannot read, rewrite or drop it. The catalog is the same thing, so it now has the same shape: `datatable_roles`, in `HIDDEN_SETTINGS`, `PROTECTED_SETTINGS` and the agent-worker denylist. No redaction to keep in step with three code paths, and no way for a neighbouring write to take it out. Two races on the same shared documents. `edit_datatable_config` read the stored data tables outside its transaction and then wrote the whole `datatable` document, so a permissions save committing in between was silently rolled back; it now reads under `FOR UPDATE`. And `set_datatable_permissions` validated role ids against the catalog before opening its transaction, so a deletion in between let it write a deleted role back — including as the default, which every later job then fails on; it now holds the catalog lock and the settings row across validation and write. Completes the authorization contracts the previous commit claimed but did not finish: `read_datatable_entry` (which it named and missed), `resolve_governing_datatable`, whose whole job is to answer for a workspace the caller may not belong to, and `converge_connect_grants_with`, which had not inherited its wrapper's. Also the generic Python SDK reference: `_format_py_params` learned the bare `*` last time, but `extract_py_functions` is a second formatter and still rendered `datatable(name, role)`, so code written from that page passed a keyword-only argument positionally. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): make the concurrency test pin the handlers, and the contracts describe what is enforced The concurrency test reimplemented the read-modify-write inline, so deleting the lock from all three handlers left it green — it pinned Postgres, not the code it was written for. It now drives `create_datatable_role` twice concurrently and asserts the catalog kept both names. Checked the way the last one should have been: removing the lock from the handler makes it fail with "wmtest_a_… is a live cluster login the catalog forgot". The contracts added last commit were stricter than this PR's own callers, which is worse than none — the next reader sees a rule already broken and learns to ignore it. `read_role_catalog` said superadmin-only while two of its four callers are open to any workspace member, and `converge_connect_grants` said superadmin while `set_datatable_permissions` reaches it as a workspace admin. Both were fine on substance: the rule that actually holds is about the credential never reaching a response, log, audit record or export, not about who may call. They now say that. `read_datatable_entry` gets the same treatment rather than the one the earlier message claimed for it: it is the primitive every resolution goes through, so it is deliberately open, and what must not escape is `permissions` — it names the governing workspace's users, groups and folders. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): close the last ways a role or a pointer can be left pointing at nothing The raw settings readers hand back whatever is in the row, so moving the catalog into its own `global_settings` key protected the config machinery and left `GET /settings/global/datatable_roles` and the settings listing returning every live password. Both now filter that one key. The neighbouring `custom_instance_replication_pwd` has the same shape and is not touched here: it predates this and widening the fix to it is a decision about an operator workflow, not a consequence of this change. Three ways a save could leave something resolving to nothing: A permissioned data table could be moved to a PostgreSQL resource. The block was carried across as a server-owned field, the runtime refuses roles on a resource-backed table, so the save succeeded and every job afterwards failed. Refused instead — turning roles off first is one step, and it keeps discarding an access decision something somebody chose. Renaming a governing data table left every fork pointing at the old name: the data table disappears from their pickers and their jobs stop, with nothing in the renaming workspace to suggest why. The rename now follows into the pointers in the same transaction. Deleting one cannot be followed the same way, so it is reported instead — the response names what it stranded, the way deleting a workspace does, and the fork's own error already says which workspace is gone. Also: `ensure_instance_db_grant_options_unchecked` claimed superadmin while the permissions handler reaches it as a workspace admin (the same class fixed last commit, one instance missed); the role entry kept an `instance_config_schema` derive it no longer needs; `write_role_catalog` was the one writer of that table not stamping `updated_at`; and the concurrency test dropped its roles only on success — a failing run is exactly the one that creates them without recording them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * refactor(datatables): put the role catalog in its own table, not in global_settings Five findings across three rounds were all the same choice. A set of live Postgres credentials was living in `global_settings`, which has generic read, list, write, config-export and CLI round-trip paths that know nothing about what they carry: the passwords reached the instance config and its YAML editor, a full-row upsert of a neighbouring key erased the catalog, `GET /settings/global/{key}` and the settings listing returned them raw, and this round the redaction that fixed the last two turned `wmill instance push` into something that wipes every password — a fix breaking the assumption the previous fix made. `POST /settings/global/datatable_roles` could also empty it outside the lock. The approved plan offered a table or `global_settings`, so this is the other option it already allowed rather than a new design. `datatable_role` is a table: no generic settings path can read it, list it, export it, write it or round-trip it, so none of the five needs a guard. The redaction, the hidden/protected/agent-denylist entries and the JSON document all go with it. One row per role also removes the read-modify-write the concurrency work was about: two concurrent creates are two inserts, and the unique index on `name` is what settles a collision. The advisory lock stays for the one window rows do not cover — `CREATE ROLE` is invisible to another transaction until commit, so without it both creates pass their `pg_roles` check. Also from this round: rename mappings are checked against the configuration they claim to describe, since fork pointers are rewritten from them — a caller could otherwise submit `main -> missing` against an unchanged config and repoint every fork of `main` at a name nothing has, and `A -> B` plus `B -> C` moved what pointed at `A` all the way to `C`. And the warning naming forks a delete stranded reached the response but not the screen: both the data table settings save and the workspace delete now show it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): validate a rename against the save it describes, and re-check under the locks Three from the round, all about deciding on state that could already have moved. A permission save resolved the data table and checked it was instance-backed before taking any lock, then wrote under one. A config save committing in between could move the table onto a PostgreSQL resource — recreating exactly what the transition guard refuses — or rename it, in which case the write targeted a key that no longer existed and reported success having changed nothing. It now re-resolves and re-checks on the locked state. Rename validation checked that the source existed before and the target existed after, which still accepts `main -> decoy` against a save that keeps both: every fork of `main` then follows onto a different data table, silently, because it keeps resolving. The rule is now the actual old-to-new key transition — a source may only survive if another rename took its name, and a target may only pre-exist if another rename freed it. That also stops two sources sharing one target, and it admits a swap, which the previous guard refused: `datatables` is keyed by name, so a swap cannot be done one save at a time, and refusing it was a regression against main. The pointer cascade now runs in two passes through a temporary name, the way the migration cascade one layer down already handles the same shape, so `A -> B` with `B -> C` moves each pointer once from what it named before the save. The tenant mutators say what they are for: they write an access decision for any workspace named, with an arbitrary mutation, and exist for the transaction that frees or renames a principal. Editing a decision on purpose belongs in the permissions endpoint. Carried in the same change: the stranded-fork list is a field rather than a phrase to grep out of a success string; the pointer cascade matches with `EXISTS` instead of a `LIKE` over the whole document, so a workspace whose pointers name something else is not rewritten to a byte-identical value under an exclusive lock; and `InstanceDatatableRole` drops the serde derives left over from the JSON document, one of which would emit `pwd`. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): cascade on the leave route that is used, gate migrations before the admin connection, and drop a role atomically The tenant cascade on leaving went onto `/users/leave`. The UI and the generated client call `/workspaces/leave` — a different handler in a different crate with the same name — which deleted the membership and left `u/` in the tenant lists. Leaving and rejoining therefore restored the access the leave was supposed to end, and a later account taking the username would have inherited it. The regression test drives the route the client actually calls; without the fix it fails with "leaving kept the tenant". The migration endpoints authorized too late. `run_datatable_migrations` opened the data table's admin connection, created `_wm_migrations` and read it before reaching the per-migration role check — so with nothing pending, nothing was checked at all. Rollback returned before its check when nothing was applied, and the status endpoint had none. All three now ask, before any connection is opened, whether the caller can reach the data table as any role at all; which role a given migration runs as is still decided per migration, and by the executor after that. Deleting a role committed the cluster drop and the catalog row, then swept the tenant lists in separate transactions. A sweep failing part-way left workspaces naming a role nothing can connect as, while the retry answered `NotFound` because the catalog entry was already gone. The sweep now runs in the same transaction, so the drop, the row and every tenant list commit together. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse to copy a data table that is under roles pg_dump carries no roles and the import runs with --no-privileges, so a copied data table arrives owned by the admin connection with no GRANT for any role. The settings clone brings `permissions` across, so the fork's tenants pass Windmill's check, connect as the role they were given, and are denied by Postgres on everything: an entry that reads as configured and answers nothing. Refuse the copy — in the import endpoint before any data moves, and in the fork path the CLI takes. Replaying the source's owners and ACLs into the clone is what lifts this, and is a change of its own. Dropping `permissions` from the copy instead would be the unsafe half, since the copy holds the parent's rows. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse the clone's database too, not only its data A clone is two endpoints: `create_pg_database` then `import_pg_database`. Only the second refused a data table under roles, so a fork asking to clone one created and registered an empty `wm_fork_…` instance database and then failed — and nothing collects it, since `drop_forked_datatable_databases` only drops entries carrying `forked_from` and no entry names this one. Refuse in both, so the clone stops before a database exists. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * nit worker error msg * fix pg_dump stuck on version 17 on nix * fix(datatables): refuse a malformed role annotation instead of ignoring it `-- Role operator`, `-- role operator;` and `-- role operator -- why` all failed the annotation parser's exact-match rule, so the query fell through to the data table's default role and ran, silently, under a login the author did not choose. Naming a role exists precisely to not do that. A leading comment whose first word is `role` is now an annotation attempt: the keyword matches case-insensitively, one trailing `;` is tolerated, and anything else is an error naming the line. Only callers that already know the target is a `datatable://` reference ever run this, so ordinary SQL keeps its comments. Also bumps the dev shell's postgres client to 18 — it trailed the server the dev database runs, which takes out every data table export, clone and fork-with-data. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): refuse a malformed role query string instead of ignoring it `?Role=analytics`, `?role=` and `?x=1&role=…` all fell through the reference parser's exact-match rule, so the connection resolved to the data table's default role and ran under a login the caller never asked for — the URI half of the same trap as a malformed `-- role` annotation. The key now matches case-insensitively, and anything else in the query string is an error naming it; `role` is the only parameter a reference takes. Callers that only need the entry keep a lenient `datatable_ref_name`, since they never act on the role. The DuckDB `ATTACH` parser propagates it rather than attaching under the default. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012ti5HyeTikPMYyW8YSdiHR * fix(datatables): carry the role annotation into the row_to_json retry The retry rebuilds its SQL from `pruneComments(code)`, so the leading comment block never reached the second attempt — and with it the `-- role ` line that decides which login the query runs as. The retry connected as the data table's default role instead, so a query the first attempt was denied could succeed on the second, reported as "recovered with the row_to_json fix". Carry the leading comment block over. The retry itself is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore(datatables): don't mount the roles UI until the ACL editor lands Enforcement ships first. The permissions drawer is what turns roles on, and the catalog section is what creates them — both are only useful once there is a way to grant a role the privileges it needs, which arrives with the ACL editor. Left mounted they would offer a feature whose other half does not exist. The two components are complete and reviewed; only their call sites here are commented out, with a note pointing the follow-up PRs at them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): honour `-- role: x`, and fix the DuckDB attach test Two review findings, both real. `attach_datatable_parses_name_and_role` never compiled: `parse_attach_datatable` returns `Result>` now and one call site kept a single `unwrap`. Its `?Role=analytics` case also asserted a refusal, contradicting the parser in the same commit, which matches the key case-insensitively. Replaced with the cases that are genuinely malformed, and a positive one for the cased key. `-- role: analytics` fell through to the default role — the silent fallback the strict parser exists to remove, for the spelling most likely to be typed. The keyword now accepts an optional colon, attached or spaced, while a word that merely starts with it (`rolebased`) is still not an attempt. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): clone a fork's pointer instead of failing after the copy Forking a fork with cloning left an orphan database. The preflight resolves the pointer and sees the governing entry, so both endpoints ran and filled the new database; `apply_forked_datatable` then refused the inherited pointer and rolled the fork back, stranding a registered `wm_fork_*` that no entry names and whose name blocks the retry. Refusing earlier would have been the smaller change, but forking a fork and cloning worked before pointers existed, so it would trade an orphan for a regression. Resolve what the pointer names and write the terminal entry the clone needs: the whole `database` object rather than a patch of its `resource_path`, since a pointer has none, and `reference` removed with it. Also accepts `-- role=x` and `-- Role = x`, two more spellings that fell through to the default role. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse to roll back the catalog while roles exist The down migration dropped the table and left every role behind: live Postgres logins whose passwords only that table carried, so after a revert Windmill could neither use, disable nor delete them, and re-applying could not recreate them because the names were taken. Cleaning up here is not possible either — dropping a role means reassigning what it owns in every instance database, and a migration runs in one — so it now refuses while the catalog is non-empty and says to delete the roles through instance settings, which does the cluster work. Also enforces the instance-only invariant the resolved-pointer clone relies on rather than only asserting it in a comment. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * refactor(datatables): settle clonability in one place, before anything is created A clone is three stages a workspace apart — `create_pg_database`, then `import_pg_database`, then `apply_forked_datatable` inside the fork transaction. Only the third can roll back, and `CREATE DATABASE` is not transactional, so any refusal that lives there strands a registered `wm_fork_*` that no entry names and whose name blocks the retry. That orphan has now been fixed three times, most recently reintroduced by a guard added one commit ago. Patching each new refusal into the first endpoint is not the fix; having two places that can refuse is. `ensure_datatable_is_clonable` now answers every reason a copy can be refused and returns what it resolved, and the stage that writes the entry only does the work. Also takes an ACCESS EXCLUSIVE lock before the rollback guard counts, so a role created concurrently cannot slip between the check and the drop. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): let a retried clone reclaim its own leftover database A clone creates its target database one request before it copies into it, and the fork that would name it is written a request after that. Any failure in between — a pg_dump error, a bad restore, a dropped connection, the source's roles changing mid-flow — left a registered `wm_fork_*` that no entry names, and every retry then failed on its name. This predates data table roles. `create_pg_database` now reclaims such a leftover before creating: only a `wm_fork_*` database Windmill registered as a data table database and that no data table or ducklake entry names, in any workspace, archived ones included. The drop never terminates connections, so a clone still copying into it makes the reclaim fail instead of being cut off. It is limited to callers who administer the source — reaching it is not enough, since on a data table without roles every member reaches it — and anyone else gets the refusal an existing database always got. Co-Authored-By: Claude Opus 5 (1M context) * Revert "fix(datatables): let a retried clone reclaim its own leftover database" This reverts commit 7dd3275a10. The reclaim tied the caller to the source they administer, but not to the database it dropped. Between another workspace's import and its final fork request, that workspace's target is full, registered, unnamed and has no open connection, so an admin of any instance data table could name it and have it dropped and recreated empty. The victim's fork would then commit pointing at the empty copy. Safe reclaim needs durable clone ownership and serialization with the request that names the database; until then the leftover stays, as it did before this PR. Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): record the stale clone database as a known limitation A clone is three requests and `CREATE DATABASE` is not transactional, so a failure after the first leaves a registered `wm_fork_*` behind, as it did before data table roles. Accepted for this PR: it is harmless to data and goes away once the clone is a single server-side operation. The comment also records why the obvious fix is wrong: reclaiming the leftover on retry, without durable clone ownership, can drop another workspace's fully copied database between its import and its final fork request. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): bounce the streams reading a data table when it is deleted Deleting a governing data table, or the workspace that holds it, only collected the fork pointers it stranded, for the warning. A Postgres trigger or capture already streaming through one of those pointers kept the replication connection it opened while the pointer still resolved, so it went on dispatching the governing database's rows after the fork lost access — until its connection happened to restart. The governing workspace's own streams on a deleted entry did the same. Both deletion paths now bounce the affected listeners inside their own transaction, through the helper a permission change already uses, so a listener that reconnects re-resolves the entry and finds it gone. The helper is split so a caller can pass the (workspace, local name) pairs it already holds. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): keep the fork schema baseline, and bounce streams on every removal Three fixes from review. `edit_datatable_config` took `forked_from` wholesale from the stored entry, so the fork schema diff's save of an advanced baseline was silently discarded and an applied change was offered again. Whether an entry carries a clone stamp is still carried from the store, since that is what marks its database droppable, but the baseline inside it is now taken from the request. The stranded-pointer warning and the stream bounce ran over the optional `deleted_datatables` hint, which the settings-sync CLI never sends, so removing a governing data table through `wmill` bounced nothing. Removals are now derived from the stored configuration against the saved one. `delete_workspace` read the pointers to bounce before its transaction, so a fork committing a pointer during the deletion was missed. The read now happens inside the transaction, after the workspace row is deleted: a fork's insert key-share locks that row through its parent foreign key, so it is either seen or fails on the missing parent. Co-Authored-By: Claude Opus 5 (1M context) * refactor(datatables): keep Postgres triggers and data table roles apart A replication stream reads every row of every table whatever the data table's roles grant, and its listener checks access only when it connects. Rather than chase every way access can change and bounce the streams each one affects, a data table now carries one or the other: - a Postgres trigger or capture cannot be created on, or connect to, a data table under roles; - roles cannot be turned on while an enabled trigger or a live capture reads the data table, its own or a fork's through its pointer. The refusal names each one to disable. This removes the stream bounces on roles edits and on data table and workspace deletion, and the trigger gate that admitted admins. The fork schema baseline fix from the same review round is kept. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): refuse a Postgres trigger on a data table under roles when it is saved Creating or editing a trigger that points at a data table under roles was accepted, and its listener then retried the refused connection every 30 seconds forever. The save is now refused, and a trigger that reaches such a data table anyway (re-enabled, or cloned into a fork) is disabled by its listener with the reason, as a missing replication slot is. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): disable a data table role before deleting it Deleting a role reassigns and drops what it owns in each registered database on its own connection, and each of those passes commits as it goes. A database failing part-way left the role enabled in the catalog and able to log in, but already stripped in the databases reached before it. The role is now disabled in its own commit first, so a failed delete leaves a disabled role to retry. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with a stream starting Turning roles on looked for enabled triggers and live captures once, without a lock anything starting a stream also took. A trigger enabled in that window could have its listener connect before roles committed, and a healthy listener never checks again. Both transitions now serialize on one advisory lock: roles going on hold it exclusive while they look, and trigger create, edit and enable, and capture setup and ping hold it shared while they commit. Either the look sees the stream, or the listener connects after roles are committed and refuses. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): wait out live listeners, and resolve stored names containing `?` Turning roles on counted a trigger as gone once disabled, and a capture once its client stopped pinging, but the listener keeps its replication connection until its next heartbeat notices. A trigger or capture whose listener pinged in the last 15 seconds, the window a server holds a listener for, now still counts as streaming. Data table names could contain `?` before they were restricted, and such entries are still stored. Splitting `?role=` off a reference misread them: `a?b` became `a` with an unknown parameter, and the clone checks looked at a different entry than the one copied. An entry stored under the whole reference is now looked up first, in the Postgres executor, DuckDB ATTACH and the clone checks. Agent workers cannot read the workspace and keep the strict parse, which refuses such a name rather than misreading it. Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): warn when a settings sync strands fork pointers A settings save reported the fork pointers left resolving to nothing only for the names in `deleted_datatables`, which `wmill sync push` never sends. The save now works out what it removed from the locked entries, and the CLI prints the stranded pointers it returns. Also correct the replication helper's contract: no role or admin check makes a replication connection safe, so a data table under roles is refused outright rather than gated as an admin operation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): refuse a save that drops a data table's roles through an undeclared rename A data table's roles follow its entry only through a declared rename. A settings sync sends the whole map and never declares one, so renaming a data table under roles there read as a delete and a new entry on the same database: the new entry carried no roles, and every caller connected as admin. Such a save is now refused, naming both entries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * fix(datatables): no entry without roles may newly reach a database under roles The previous guard only caught a new name replacing an entry under roles. A whole-map save could also repoint an existing entry without roles at that database, or another workspace could point one there, and every caller of that entry would connect as admin. The rule is now stated on the saved entries: one that carries no roles and newly points at an instance database any entry under roles uses, in this workspace or another, is refused. A declared rename carries its roles and passes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move data table role catalog and resolution to the enterprise edition Roles are an Enterprise Edition feature. The catalog, the Postgres logins, CONNECT convergence, tenant evaluation and the role half of connection resolution move to windmill-ee-private. Every public function keeps its path and signature and forwards through datatable_roles_oss, which re-exports the enterprise implementation or, without it, refuses. Without the enterprise edition a data table under roles, or a caller naming a role, is refused a connection rather than resolved as admin, and the reach and admin-access checks refuse one under roles. A data table not under roles resolves as before in every edition, and an instance database keeps the CONNECT grants it was created with. The catalog lock, the stream lock, the tenant cascades and the permissions stripping stay in OSS: they only restrict. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table permissions endpoints to the enterprise edition The permissions read, save and usable-roles handlers move to windmill-ee-private; the routes stay registered and, without the enterprise edition, answer that data table roles are an Enterprise Edition feature. ensure_governs_datatable and ensure_reaches_datatable keep their paths: the first refuses, the second passes a data table not under roles. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): move the data table role catalog endpoints to the enterprise edition The superadmin list, create, update and delete handlers move to windmill-ee-private. The routes stay registered and, without the enterprise edition, refuse after authentication. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the roles tests on the enterprise edition, refusals without it Each test that exercises roles runs with private and enterprise. Two tests run without them: every roles route answers the Enterprise refusal, and a data table saved under roles, or a named role, is refused a connection while one not under roles resolves as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * feat(datatables): gate the roles UI mount sites on an enterprise license Both mount sites are still commented out; the gate travels with them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * test(datatables): run the tenant matcher test on the enterprise edition The matcher it covers is enterprise code now, so without the enterprise edition the test hit the stub and failed the default windmill-common run. It runs with private and enterprise, and a counterpart without them asserts that no tenant list covers anyone, the wildcard and a workspace admin included. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BjfMkJyKzodxkobqGZ6Lqb * chore: update ee-repo-ref to a1873dbb67f2302b85ff5362f8387b48eccdb607 This commit updates the EE repository reference after PR #783 was merged in windmill-ee-private. Previous ee-repo-ref: 5c853e2c20eca6b748415fc0d6862a6ebfb5fec4 New ee-repo-ref: a1873dbb67f2302b85ff5362f8387b48eccdb607 Automated by sync-ee-ref workflow. * fix(datatables): refuse roles while a same-workspace alias reaches the database Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): let CE migrations connect as an explicitly named admin Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): serialize roles going on with aliases saved from other workspaces Co-Authored-By: Claude Opus 5 (1M context) * docs(datatables): note that legacy names with ? cannot be migrated Co-Authored-By: Claude Opus 5 (1M context) * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix(datatables): drop a DuckDB data table secret once its ATTACH has used it Co-Authored-By: Claude Opus 5 (1M context) * perf(datatables): resolve a workspace's data tables per pointer hop, not per entry Co-Authored-By: Claude Opus 5 (1M context) * fix(datatables): hold the parent's settings while a fork points at its data tables Co-Authored-By: Claude Opus 5 (1M context) * feat(db-manager): add a schema diagram view The database manager could only show one table at a time, so nothing in it answered "how do these tables relate". This adds a Diagram mode beside the data view, for PostgreSQL: the tree's checkboxes pick the tables, and the canvas draws them as cards on a pan/zoom surface. There are deliberately no edges. A schema of any size turns an edge-drawn ERD into a hairball, so the relationships surface on hover instead: hovering a table lights every table it is linked to and the columns tying them together, hovering a column lights only the column it is actually paired with. Only the connected database can be drawn, so another data table's rows in the tree get no checkbox in this mode, and a row of the connected one toggles what is drawn rather than selecting it for a data pane that is not shown. Foreign keys come from a new ALL_FOREIGN_KEYS marker that reads the whole database in one query. The existing FOREIGN_KEYS marker is per table, which would be one job per card. It reads pg_constraint rather than information_schema, whose constraint_column_usage loses which column of a composite key pairs with which. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): discard a superseded relations read The manager is not remounted on every database change and a queued job cannot be recalled, so a slow read could resolve after the next database's and decorate its tables with the previous one's relations: runed stores whatever the fetcher returns, and this one ignored its abort signal. It now throws on a superseded run, the way the per-table foreign-key resource beside it already does, and the result carries the database it was read from so it is only ever shown alongside that one. The database is identified by a key string passed in rather than by the objects read from it: those are replaced whenever the manager re-reads, which says nothing about the database having changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): retry a failed relations read on re-entry A failed read was cached like a successful one, so a transient worker or connection failure left the diagram without relations until the schema was reloaded or the manager remounted. The cache now only answers for a read that succeeded, and leaving the diagram and coming back retries. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): scope the diagram's selection to its database The manager is not remounted on every database change, so the tables checked for one database's diagram carried into the next: same-named tables were drawn from a database they were never chosen in, and differing names left an empty canvas that the first draw no longer filled, having already run. Both the selection and that first draw are now scoped to the database they belong to. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * refactor(db-manager): hold the diagram's state against its database Three separate pieces of diagram state each had to work out on their own which database they were about, and each got it wrong in a different way: the selection carried across a switch, the first-draw latch counted itself done before the schema it needed had arrived, and the relations read could land under the wrong database. They are now one record carrying the database it describes, read through a single derived where a key that no longer matches reads as a fresh diagram. The first draw waits for the schema rather than marking itself done without one, and any table the user checks counts as that draw, so an emptied canvas is never refilled behind them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): draw a schema the database actually has `selected.schemaKey` follows the tree and is only replaced once it is empty, so it can still name a schema of the database browsed before this one. The diagram's first draw took it on trust, found no tables under it and stood down for good, leaving that database on a blank canvas. It is now taken only when this database has that schema, and otherwise falls back to the same default the tree starts from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * feat(db-manager): move the view toggle into the drawer header The Data/Diagram toggle sat above the tree's search box, where it read as a filter over the tree rather than a choice about the pane beside it. It now sits with the manager's other header controls, left of the worker tag. The mode moves with it: the drawer holds it and the manager is told which view to show, clamping it for a database that has no diagram. Holding it outside the key that remounts the manager also means picking another data table or role no longer drops the user back into the data view. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * feat(datatables): add an ACL editor for data table roles Co-Authored-By: Claude Opus 5 (1M context) * fix: take every pooled connection before the ACL apply locks Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: refresh grant options only after the ACL apply validates its plan Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: add only missing grant options before an ACL apply, never default privileges Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: run one data table ACL apply at a time per server before it connects Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: hold the ACL connection to the database that was authorized Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: build the ACL connection from the authorized data table entry Resolving the settings again could land on a resource with the same database name on another server, which the later entry checks never see. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * fix: check ACL read reach against the entry it connects from Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01BRoYE5ZeAVvrDYdfhDAYXb * chore: update ee-repo-ref to 7e338e4dabf91689bfd7fb0333c6534040b17b59 This commit updates the EE repository reference after PR #787 was merged in windmill-ee-private. Previous ee-repo-ref: 0edd40979cf36bfba59323f3f6a0811ae1369cf5 New ee-repo-ref: 7e338e4dabf91689bfd7fb0333c6534040b17b59 Automated by sync-ee-ref workflow. * feat(datatables): clone a data table under roles with its owners and grants (#11120) Claude-Session: https://claude.ai/code/session_01UbrtwiYNfayrmqouBJHwGV Co-authored-by: Claude Opus 5 (1M context) * fix: open the raw app data table drawer when the workspace has none Selecting the first data table of an empty list passed undefined to the name check, which threw instead of opening the drawer on no data table. Co-Authored-By: Claude Opus 5 (1M context) * fix(db-manager): wait for table metadata before reading relations Returning to a cached database with the diagram selected remounts the manager with its schema ready but its metadata not yet loaded. The relations read started anyway, then ran again once the metadata landed, and the first queued job cannot be recalled by an abort, so both full-database queries took a worker and hit PostgreSQL. The read now waits for the metadata, whose arrival already re-runs it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): keep the relations read across a switch to the data view The relations came from a runed `resource`, which abandons a run whenever any of its inputs changes. A queued job cannot be recalled, so switching to the data view while relations loaded threw away a read that still ran to completion, and coming back queued a second full-database job. A plain read replaces it: at most one per database and metadata, started when the diagram needs it, kept across mode changes, and replaced only by another database, reloaded metadata, or a retry after failing. An answer arriving after its read was replaced is ignored, so it can never decorate another database's tables. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): hand down metadata only for the database it was read from A resource keeps its previous value while it refetches, so after a switch of database the manager was handed the previous database's metadata as the new one's. The diagram started its relations read beside it, then again once the right metadata arrived: two full-database jobs, the first unrecallable. It also drew the previous database's columns meanwhile. The metadata now carries the database it was read for, and nothing is handed down while it belongs to another. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): address the schema diagram's review nits - Clear a pinned highlight once its table is unchecked or its column is folded away by collapsing the card; it otherwise left the canvas dimmed around nothing on screen. - Use the design-system button for a card's expand control. - Describe the view-mode counter as what it records — switches between the data and diagram views, not how the manager was opened. - Give the relation index loop its real reason: hovering either table of a pair lights both of its columns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * fix(db-manager): keep schema diagram cards in place and open tables from the header - Cards were draggable, but the canvas is laid out from scratch on every change, so a moved card snapped back at the next one. Dragging is off. - Double-clicking a card anywhere opened its table. Expanding a card moves its rows under the pointer, so a double-click on the expand control landed its second click on a row and left the diagram. Only the header opens the table now; the rows keep hovering and pinning columns. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01MXECAKye6KgN13wznsuRzP * chore: update ee-repo-ref to 39f8896379295728a509c4d642574bb8173368e8 This commit updates the EE repository reference after PR #784 was merged in windmill-ee-private. Previous ee-repo-ref: 720f334ee1843878749e9520567a7b657fc2deda New ee-repo-ref: 39f8896379295728a509c4d642574bb8173368e8 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-common/src/query_builders.rs | 62 ++++++ docs/feature-telemetry.md | 8 +- frontend/src/lib/components/DBManager.svelte | 209 ++++++++++++++++-- .../lib/components/DBManagerContent.svelte | 25 ++- .../src/lib/components/DBManagerDrawer.svelte | 42 +++- .../lib/components/InstanceSettings.svelte | 6 +- frontend/src/lib/components/dbOps.ts | 16 ++ .../src/lib/components/dbRelations.test.ts | 81 +++++++ frontend/src/lib/components/dbRelations.ts | 137 ++++++++++++ .../dbdiagram/DbSchemaDiagram.svelte | 171 ++++++++++++++ .../components/dbdiagram/DbTableNode.svelte | 118 ++++++++++ .../dbdiagram/DiagramAutoFit.svelte | 20 ++ .../dbdiagram/dbDiagramHighlight.svelte.ts | 80 +++++++ .../dbdiagram/dbDiagramModel.test.ts | 45 ++++ .../components/dbdiagram/dbDiagramModel.ts | 131 +++++++++++ 16 files changed, 1126 insertions(+), 27 deletions(-) create mode 100644 frontend/src/lib/components/dbRelations.test.ts create mode 100644 frontend/src/lib/components/dbRelations.ts create mode 100644 frontend/src/lib/components/dbdiagram/DbSchemaDiagram.svelte create mode 100644 frontend/src/lib/components/dbdiagram/DbTableNode.svelte create mode 100644 frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts create mode 100644 frontend/src/lib/components/dbdiagram/dbDiagramModel.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b82e25d93b..24c90028c4 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -559a9ff78ea03439553cf2b412e73765762e9013 +39f8896379295728a509c4d642574bb8173368e8 diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index 9d85278404..546d2d454b 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -333,6 +333,7 @@ pub fn try_expand_internal_db_query( // Metadata queries "LOAD_TABLE_METADATA" => expand_load_table_metadata(json_str, db_type), "FOREIGN_KEYS" => expand_foreign_keys(json_str, db_type).map(ExpandedQuery::sql), + "ALL_FOREIGN_KEYS" => expand_all_foreign_keys(db_type).map(ExpandedQuery::sql), "PRIMARY_KEY_CONSTRAINT" => { expand_primary_key_constraint(json_str, db_type).map(ExpandedQuery::sql) } @@ -2737,6 +2738,47 @@ fn expand_foreign_keys(json_str: &str, db_type: DbType) -> Result Result { + make_all_foreign_keys_query(db_type) +} + +/// Every foreign key of the database, one row per referencing column. +/// +/// Read from `pg_constraint` rather than `information_schema`: the latter's +/// `constraint_column_usage` loses the pairing of a composite key's columns, +/// which the diagram needs to line a source column up with the column it points +/// at. `unnest(conkey, confkey)` keeps them zipped in declaration order. +fn make_all_foreign_keys_query(db_type: DbType) -> Result { + if db_type != DbType::Postgresql { + return Err(format!( + "The schema diagram is only supported for PostgreSQL, not {:?}", + db_type + )); + } + Ok(String::from( + "SELECT + con.conname as fk_constraint_name, + src_ns.nspname as source_schema, + src.relname as source_table, + src_att.attname as source_column, + tgt_ns.nspname as target_schema, + tgt.relname as target_table, + tgt_att.attname as target_column, + k.ord as ordinal +FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class src ON src.oid = con.conrelid + JOIN pg_catalog.pg_namespace src_ns ON src_ns.oid = src.relnamespace + JOIN pg_catalog.pg_class tgt ON tgt.oid = con.confrelid + JOIN pg_catalog.pg_namespace tgt_ns ON tgt_ns.oid = tgt.relnamespace + JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS k(src_attnum, tgt_attnum, ord) ON true + JOIN pg_catalog.pg_attribute src_att ON src_att.attrelid = src.oid AND src_att.attnum = k.src_attnum + JOIN pg_catalog.pg_attribute tgt_att ON tgt_att.attrelid = tgt.oid AND tgt_att.attnum = k.tgt_attnum +WHERE con.contype = 'f' + AND src_ns.nspname NOT IN ('pg_catalog', 'information_schema') +ORDER BY src_ns.nspname, src.relname, con.conname, k.ord;", + )) +} + fn expand_primary_key_constraint(json_str: &str, db_type: DbType) -> Result { let p: PrimaryKeyConstraintPayload = serde_json::from_str(json_str) .map_err(|e| format!("Invalid PRIMARY_KEY_CONSTRAINT payload: {}", e))?; @@ -4721,6 +4763,26 @@ mod tests { assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n")); } + // ----------------------------------------------------------------------- + // ALL_FOREIGN_KEYS + // ----------------------------------------------------------------------- + + #[test] + fn test_expand_all_foreign_keys_postgresql() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let sql = expand_code(marker, &ScriptLang::Postgresql); + assert!(sql.contains("con.contype = 'f'")); + // Columns of a composite key must stay zipped with the ones they point at. + assert!(sql.contains("unnest(con.conkey, con.confkey) WITH ORDINALITY")); + } + + #[test] + fn test_expand_all_foreign_keys_rejects_non_postgres() { + let marker = r#"-- WM_INTERNAL_DB_ALL_FOREIGN_KEYS {}"#; + let result = try_expand_internal_db_query(marker, &ScriptLang::Mysql); + assert!(result.unwrap().is_err()); + } + // ----------------------------------------------------------------------- // PRIMARY_KEY_CONSTRAINT // ----------------------------------------------------------------------- diff --git a/docs/feature-telemetry.md b/docs/feature-telemetry.md index ae62d46b94..9dacfc1cbd 100644 --- a/docs/feature-telemetry.md +++ b/docs/feature-telemetry.md @@ -4,10 +4,10 @@ anonymous usage-stats payload. It answers "does anyone use this, and which variant do they pick" without any identifying data leaving the instance. -It currently carries 51 registered actions across nineteen features (`ai_session`, `ai_chat`, -`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `flow_editor`, `flow_run`, -`flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, `hub_script`, -`usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the +It currently carries 58 registered actions across twenty features (`ai_session`, `ai_chat`, +`ai_fix`, `ai_agent`, `ai_agent_eval`, `app_sandbox`, `datatable`, `db_manager`, `flow_editor`, +`flow_run`, `flow_step`, `home`, `run_form`, `debugger`, `trigger`, `command_script`, +`hub_script`, `usage_meter`, `sso_groups_claim`, `cloud_trial_offer`). Nearly all of the product is uninstrumented, so new user-facing work is the opportunity to change that. ## When to instrument diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 4775409252..16e64dc5a0 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,5 +1,5 @@ + +
+ {#if error} +
+ {error} +
+ {/if} + {#if tables.length === 0} +
+ Check tables in the sidebar to draw them here +
+ {:else} + + highlight.clear()} + > +
+ + + + + + + {/if} + {#if loading} +
+ + Loading relationships +
+ {/if} +
+ + diff --git a/frontend/src/lib/components/dbdiagram/DbTableNode.svelte b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte new file mode 100644 index 0000000000..d17faaa006 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DbTableNode.svelte @@ -0,0 +1,118 @@ + + + +
highlight.hover({ table: table.key })} + onmouseleave={() => highlight.hover(undefined)} + onclick={() => highlight.togglePin({ table: table.key })} + role="button" + tabindex="-1" +> + + +
+ + {table.table} + {table.schema} +
+ + {#each columns as column (column.name)} + {@const columnKey = columnKeyIn(table.key, column.name)} + {@const isLit = highlight.columns?.has(columnKey)} + +
highlight.hover({ table: table.key, column: column.name })} + onmouseleave={() => highlight.hover({ table: table.key })} + onclick={(e) => { + e.stopPropagation() + highlight.togglePin({ table: table.key, column: column.name }) + }} + > + {#if column.isPrimaryKey} + + {:else if data.index.referencingColumns.has(columnKey)} + + {:else} + + {/if} + {column.name} + {column.datatype} +
+ {/each} + + {#if hasExpandToggle(table)} +
+ +
+ {/if} +
diff --git a/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte new file mode 100644 index 0000000000..b271886abd --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/DiagramAutoFit.svelte @@ -0,0 +1,20 @@ + diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts new file mode 100644 index 0000000000..54f52b85ca --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramHighlight.svelte.ts @@ -0,0 +1,80 @@ +import { getContext, setContext } from 'svelte' +import { columnKeyIn, type RelationIndex } from '../dbRelations' + +/** What the pointer is on: a whole table, or one of its columns. */ +export type HighlightTarget = { table: string; column?: string } + +const EMPTY: ReadonlySet = new Set() + +/** + * The diagram draws no edges, so the relationships only exist while something is + * highlighted. This owns that: what is hovered (or pinned by a click), and which + * tables and columns it lights up. + */ +export class DbDiagramHighlight { + #getIndex: () => RelationIndex + + hovered = $state(undefined) + /** Set by clicking, so the relations can be read without holding the pointer + * still. A hover takes precedence while it lasts. */ + pinned = $state(undefined) + + target = $derived(this.hovered ?? this.pinned) + + /** Tables that stay lit. Undefined means nothing is highlighted and every + * table renders at full strength. */ + tables = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + if (target.column) { + const columnTable = this.#getIndex().columnTable + const tables = new Set([target.table]) + for (const column of this.columns ?? EMPTY) { + const table = columnTable.get(column) + if (table) tables.add(table) + } + return tables + } + return new Set([target.table, ...(this.#getIndex().relatedTables.get(target.table) ?? EMPTY)]) + }) + + /** Columns that stay lit inside the highlighted tables. */ + columns = $derived.by | undefined>(() => { + const target = this.target + if (!target) return undefined + const index = this.#getIndex() + if (target.column) { + const key = columnKeyIn(target.table, target.column) + return new Set([key, ...(index.linkedColumns.get(key) ?? EMPTY)]) + } + return index.relatedColumns.get(target.table) ?? EMPTY + }) + + constructor(getIndex: () => RelationIndex) { + this.#getIndex = getIndex + } + + hover(target: HighlightTarget | undefined) { + this.hovered = target + } + + /** Clicking the already-pinned target unpins it. */ + togglePin(target: HighlightTarget) { + const current = this.pinned + this.pinned = + current && current.table === target.table && current.column === target.column + ? undefined + : target + } + + clear() { + this.hovered = undefined + this.pinned = undefined + } +} + +const contextKey = 'DbDiagramHighlight' + +export const setDbDiagramHighlight = (highlight: DbDiagramHighlight) => + setContext(contextKey, highlight) +export const getDbDiagramHighlight = () => getContext(contextKey) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts new file mode 100644 index 0000000000..8b6b584860 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest' +import { CARD_WIDTH, cardHeight, layoutTables, type DiagramTable } from './dbDiagramModel' + +function table(name: string, columnCount: number): DiagramTable { + return { + key: `shop.${name}`, + schema: 'shop', + table: name, + columns: Array.from({ length: columnCount }, (_, i) => ({ + name: `c${i}`, + datatype: 'text', + isPrimaryKey: i === 0 + })) + } +} + +describe('layoutTables', () => { + // Runs again on every checkbox click, so cards must not land on top of each + // other for any mix of card heights. + it('never overlaps two cards', () => { + const tables = Array.from({ length: 40 }, (_, i) => table(`t${i}`, 1 + (i % 17))) + const expanded = new Set(['shop.t3', 'shop.t20']) + const positions = layoutTables(tables, expanded) + + const boxes = tables.map((t) => ({ + x: positions[t.key].x, + y: positions[t.key].y, + h: cardHeight(t, expanded.has(t.key)) + })) + for (let i = 0; i < boxes.length; i++) { + for (let j = i + 1; j < boxes.length; j++) { + const a = boxes[i] + const b = boxes[j] + const apart = + a.x + CARD_WIDTH <= b.x || b.x + CARD_WIDTH <= a.x || a.y + a.h <= b.y || b.y + b.h <= a.y + expect(apart, `${tables[i].key} overlaps ${tables[j].key}`).toBe(true) + } + } + }) + + it('places the same tables in the same spots every time', () => { + const tables = Array.from({ length: 12 }, (_, i) => table(`t${i}`, 3 + i)) + expect(layoutTables(tables, new Set())).toEqual(layoutTables(tables, new Set())) + }) +}) diff --git a/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts new file mode 100644 index 0000000000..ccd32cdb22 --- /dev/null +++ b/frontend/src/lib/components/dbdiagram/dbDiagramModel.ts @@ -0,0 +1,131 @@ +import type { DBSchema } from '$lib/stores' +import type { ColumnDef } from '../apps/components/display/dbtable/utils' +import type { SelectedTable } from '../DBManager.svelte' +import { tableKeyOf } from '../dbRelations' + +export type DiagramColumn = { + name: string + datatype: string + isPrimaryKey: boolean +} + +export type DiagramTable = { + /** `schema.table`, the key relations and highlights are addressed by. */ + key: string + schema: string + table: string + columns: DiagramColumn[] +} + +/** Card geometry. The node component must render exactly the rows the layout + * budgeted for, or cards overlap. */ +export const CARD_WIDTH = 248 +export const HEADER_HEIGHT = 34 +export const ROW_HEIGHT = 22 +const CARD_PADDING = 6 +const GAP_X = 32 +const GAP_Y = 20 +/** Rows shown before a card collapses into a "+N more" row. */ +export const COLLAPSED_ROW_LIMIT = 12 +/** Cards are packed towards this width/height ratio, so the canvas opens on + * something screen-shaped rather than one very long column. */ +const TARGET_ASPECT = 1.7 + +export function visibleColumns(table: DiagramTable, expanded: boolean): DiagramColumn[] { + return expanded ? table.columns : table.columns.slice(0, COLLAPSED_ROW_LIMIT) +} + +export function hiddenColumnCount(table: DiagramTable, expanded: boolean): number { + return table.columns.length - visibleColumns(table, expanded).length +} + +/** Whether the card carries a row to expand or collapse its column list. */ +export function hasExpandToggle(table: DiagramTable): boolean { + return table.columns.length > COLLAPSED_ROW_LIMIT +} + +export function cardHeight(table: DiagramTable, expanded: boolean): number { + const rows = visibleColumns(table, expanded).length + (hasExpandToggle(table) ? 1 : 0) + return HEADER_HEIGHT + rows * ROW_HEIGHT + CARD_PADDING +} + +/** The tables to draw, in the order they are laid out. */ +export function buildDiagramTables( + dbSchema: DBSchema, + colDefs: Record | undefined, + selected: SelectedTable[] +): DiagramTable[] { + if (dbSchema.lang === 'graphql') return [] + const tables: DiagramTable[] = [] + + for (const { schema, table } of selected) { + const key = tableKeyOf({ schema, table }) + const fromColDefs = colDefs?.[key] + let columns: DiagramColumn[] + if (fromColDefs?.length) { + columns = fromColDefs.map((col) => ({ + name: col.field ?? '', + datatype: col.datatype ?? '', + isPrimaryKey: !!col.isprimarykey + })) + } else { + // The cached schema has no primary-key flag; it only stands in until the + // column metadata query lands. + const schemaColumns = dbSchema.schema[schema]?.[table] + if (!schemaColumns) continue + columns = Object.entries(schemaColumns).map(([name, col]) => ({ + name, + datatype: col.type, + isPrimaryKey: false + })) + } + tables.push({ key, schema, table, columns }) + } + + tables.sort((a, b) => a.schema.localeCompare(b.schema) || a.table.localeCompare(b.table)) + return tables +} + +/** Packs the cards into balanced columns, in reading order. Runs again on every + * change to the selection, so it must be deterministic: the same tables always + * land in the same place. */ +export function layoutTables( + tables: DiagramTable[], + expanded: Set +): Record { + const heights = tables.map((t) => cardHeight(t, expanded.has(t.key))) + const stacked = heights.reduce((a, b) => a + b, 0) + GAP_Y * Math.max(0, tables.length - 1) + + let columnCount = 1 + let bestScore = Infinity + for (let c = 1; c <= tables.length; c++) { + const width = c * CARD_WIDTH + (c - 1) * GAP_X + const height = stacked / c + // Compared in log space so twice-too-wide and twice-too-tall score alike. + const score = Math.abs(Math.log(width / height / TARGET_ASPECT)) + if (score < bestScore) { + bestScore = score + columnCount = c + } + } + + const target = stacked / columnCount + const positions: Record = {} + let column = 0 + let columnHeight = 0 + + tables.forEach((table, i) => { + const height = heights[i] + // Overshooting the target by more than stopping short of it costs less in + // the next column. + const overshoot = columnHeight + height - target + if (column < columnCount - 1 && columnHeight > 0 && overshoot > target - columnHeight) { + column++ + columnHeight = 0 + } + positions[table.key] = { x: column * (CARD_WIDTH + GAP_X), y: columnHeight } + columnHeight += height + GAP_Y + }) + + return positions +} From 592ef736104ff63dd36978a69353a2585636da87 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 17:28:52 +0200 Subject: [PATCH 21/34] fix: keep a watched job's code when its completion update drops it (#11290) Co-authored-by: Claude Opus 5 --- frontend/src/lib/components/JobLoader.svelte | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index 324bdea6c7..0994905121 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -444,6 +444,8 @@ streamedResult.length >= completedResult.length ? streamedResult : completedResult + njob.raw_code ??= job?.raw_code + njob.raw_flow = job?.raw_flow ?? njob.raw_flow job = njob await onJobCompleted(testId, job, callbacks) } @@ -870,6 +872,11 @@ const njob = previewJobUpdates.job as Job & { result_stream?: string } njob.logs = pickMoreCompleteLogs(job?.logs, njob.logs) njob.result_stream = job?.result_stream ?? '' + // The server sends the completed job without its code to keep the + // update small (no raw_code, "..." for a deployed flow's inline + // steps), so keep the code fetched with the initial job. + njob.raw_code ??= job?.raw_code + njob.raw_flow = job?.raw_flow ?? njob.raw_flow job = njob onJobCompleted(id, job, callbacks) } From ea0b947644d4094304ecea26406fa6aac15dcfac Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 22 Sep 2026 17:29:20 +0200 Subject: [PATCH 22/34] fix: stop the chat repeating run results the card already shows (#11293) * fix: tell the chat not to repeat run results the card already shows * fix: add the run result note on completion, background runs included * fix: skip the run result note on failed runs * fix: keep the run result note off background completions --- .../components/copilot/chat/shared.test.ts | 44 +++++++++++++++++++ .../src/lib/components/copilot/chat/shared.ts | 16 ++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 768b6d1b65..005a7666db 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1510,6 +1510,50 @@ describe('pollJobCompletion detach', () => { }) }) +describe('executeTestRun result note', () => { + // A failed run's card lands on the error, with the logs a tab away: the model must stay + // free to quote them, so only a success says the user already sees the run. + it('tells the model not to repeat a successful result, and not a failed one', async () => { + vi.useFakeTimers() + try { + const { executeTestRun } = await import('./shared') + const { JobService } = await import('$lib/gen') + const getJobUpdates = vi.mocked(JobService.getJobUpdates) + getJobUpdates.mockReset() + getJobUpdates.mockResolvedValue({ completed: true, running: false } as any) + const run = async (success: boolean) => { + vi.mocked(JobService.getJob).mockReset() + vi.mocked(JobService.getJob).mockResolvedValue({ + type: 'CompletedJob', + success, + result: [1, 2], + logs: 'ran' + } as any) + const promise = executeTestRun({ + jobStarter: async () => 'job1', + workspace: 'w', + toolId: 'tool1', + startMessage: 'Starting...', + contextName: 'script', + // The job hooks are what mark the global/sessions chat. + toolCallbacks: { + setToolStatus: vi.fn(), + removeToolStatus: vi.fn(), + onJobStatus: vi.fn(), + onJobStarted: vi.fn() + } as any + }) + await vi.advanceTimersByTimeAsync(1000) + return promise + } + expect(await run(true)).toContain('Do not repeat') + expect(await run(false)).not.toContain('Do not repeat') + } finally { + vi.useRealTimers() + } + }) +}) + describe('deriveChatJobStatus', () => { // CompletedJob is discriminated by the presence of a `success` key; the branch // order deliberately mirrors JobStatusIcon so the badge and scalar never drift. diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index bd347224f9..f58ed3b6d3 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -1871,6 +1871,15 @@ export async function buildTestRunArgs( return parsedArgs } +// Said with the result rather than in the system prompt: there, models still copied a +// result's table or JSON into the reply, under a card that already renders it. Only for +// a successful run in the generic card completedJobToolStatus fills: a failed run's card +// lands on the error with the logs a tab away, and a formatCompletion card shows its own. +// Never on a background completion: its card can be far up the chat by then, so the +// reply may be the only place the user reads the result. +const RESULT_SHOWN_NOTE = + "The user already sees this run's result in its card. Do not repeat the result or logs in your reply: say what it shows, quoting only the values your conclusion rests on." + // The string handed back to the model when a job is backgrounded. It carries the // job id so the model can pull status/args/result/logs on demand (get_run / list_runs), // and tells it the completion will be reported later (notify-only wake). @@ -2015,7 +2024,12 @@ export async function executeTestRun(config: TestRunConfig): Promise { ...(job.success ? {} : { error: getErrorMessage(job.result) }) }) - const summary = formatResultSummary(job.result, job.logs, job.success) + // detachEnabled marks the global/sessions chat; the in-editor chats read the summary + // alone. The card opening on the result is up to each tool: a run card does, a + // generic one only with `autoCollapseDetails: false` at its registration. + const summary = + (detachEnabled && job.success ? `${RESULT_SHOWN_NOTE}\n` : '') + + formatResultSummary(job.result, job.logs, job.success) // get_run only exists in the global/sessions chat (the same hosts that wire // the job hooks) — don't advertise it to in-editor chats. if (detachEnabled && config.contextName === 'flow' && !job.success) { From 42655ff5ef3e4eae8ff90c623683bb321ae7d06b Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:07:21 +0200 Subject: [PATCH 23/34] fix: memoize the resolved authed so one request resolves identity once (#11299) * fix: memoize the resolved authed so one request resolves identity once Co-Authored-By: Claude Opus 5 (1M context) * test: pin that the memo preserves job token provenance Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/windmill-api-auth/src/auth.rs | 81 +++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 5 deletions(-) diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 6e5aa6b1da..869f40e14c 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -1073,7 +1073,8 @@ fn no_auth_admin_authed() -> ApiAuthed { } /// Resolves OptJobAuthed from request parts. -/// Takes ownership of Parts and returns them back. +/// Takes ownership of Parts and returns them back, memoized into the extensions so a +/// second extraction in the same request is free. #[allow(unreachable_code, unused_mut)] pub async fn resolve_opt_job_authed( mut parts: Parts, @@ -1089,10 +1090,11 @@ pub async fn resolve_opt_job_authed( )); } - let already_authed = parts.extensions.get::().cloned(); - - if let Some(authed) = already_authed { - return Ok((authed, parts)); + // Safe only while what the route checks below read is fixed before the first + // extraction: path and method always are, `workspace_id` only while every + // `GatewayWorkspaceId` writer stays layered outside auth extraction. + if let Some(opt_job_authed) = parts.extensions.get::().cloned() { + return Ok((opt_job_authed, parts)); } let already_tokened = parts.extensions.get::().cloned(); @@ -1185,6 +1187,9 @@ pub async fn resolve_opt_job_authed( if let Some(workspace_id) = workspace_id { Span::current().record("workspace_id", &workspace_id); } + // Separate from the ApiAuthed insert above, which handlers taking + // `Extension` read. + parts.extensions.insert(opt_job_authed.clone()); return Ok((opt_job_authed, parts)); } } @@ -1311,3 +1316,69 @@ pub async fn list_tokens_internal( Ok(Json(tokens)) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Dropping the `AUTH_CACHE` entry between the two resolutions is what makes a memo + /// hit observable: only a working memo can answer the second one. + #[tokio::test] + async fn resolve_opt_job_authed_is_memoized() { + // Never connected: an AUTH_CACHE hit resolves without a query, and AuthCache + // only needs the handle to exist. Without the timeout a regression — which + // falls through to a real lookup — spends sqlx's 30s default failing. + let db = sqlx::postgres::PgPoolOptions::new() + .acquire_timeout(std::time::Duration::from_millis(50)) + .connect_lazy("postgres://memo@127.0.0.1:1/memo") + .expect("lazy pool"); + + let token = "memo_guard_token"; + // Workspace-scoped: a job token on a workspace-less route is refused by + // check_job_token_for_global_route before the memo is ever taken. + let key = ("memo_workspace".to_string(), token.to_string()); + let job_id = uuid::Uuid::from_u128(0x6d656d6f); + AUTH_CACHE.insert( + key.clone(), + ExpiringAuthCache { + authed: ApiAuthed { + email: "memo@windmill.dev".to_string(), + username: "memo".to_string(), + ..Default::default() + }, + expiry: chrono::Utc::now() + chrono::Duration::hours(1), + job_id: Some(job_id), + }, + ); + + #[cfg(feature = "enterprise")] + let cache = AuthCache::new(db, None, None); + #[cfg(not(feature = "enterprise"))] + let cache = AuthCache::new(db, None); + + let mut parts = http::Request::builder() + .uri("/api/w/memo_workspace/jobs/list") + .header(http::header::AUTHORIZATION, format!("Bearer {token}")) + .body(()) + .unwrap() + .into_parts() + .0; + parts.extensions.insert(Arc::new(cache)); + + let (_, parts) = resolve_opt_job_authed(parts) + .await + .map_err(|(e, _)| e) + .expect("first resolution"); + + AUTH_CACHE.remove(&key); + + let (second, _) = resolve_opt_job_authed(parts) + .await + .map_err(|(e, _)| e) + .expect("second resolution must hit the memo, not re-resolve"); + assert_eq!(second.authed.username, "memo"); + // The wrapper must survive whole: memoizing only the inner ApiAuthed and + // rebuilding around it drops the provenance forbid_elevated_job_token gates on. + assert_eq!(second.job_id, Some(job_id)); + } +} From 89323fb0258bf13413217d6867885765f37b4678 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:07:31 +0200 Subject: [PATCH 24/34] refactor: separate the tools the chat sends from the ones it has (#11295) * refactor: split the chat's tool list into sent-now and available-in-session views Co-Authored-By: Claude Opus 5 (1M context) * refactor: drop the forward-looking ticket reference from the tool-base comment Co-Authored-By: Claude Opus 5 (1M context) * refactor: make the readonly tool seam enforceable and the comments accurate Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../copilot/chat/AIChatManager.svelte.ts | 32 +++++++++----- .../copilot/chat/AIChatManager.test.ts | 42 +++++++++++++++++++ .../chat/AssistantSettingsModal.svelte | 6 +-- .../components/copilot/chat/chatViewHost.ts | 5 ++- .../copilot/chat/planModeController.svelte.ts | 6 +++ 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 27f9bfc43f..74a8b4d627 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -717,7 +717,17 @@ export class AIChatManager implements ChatViewHost { role: 'system', content: '' }) - tools = $state[]>([]) + /** The sources the current mode assembled, concatenated. Private, because neither view + * over it is this list: a consumer reading it would advertise a tool set no request ever + * carries. */ + #assembledTools = $state[]>([]) + /** What the request carries: the assembled tools plus the plan-mode transition the current + * posture offers. Read by the request path and by the YOLO disclosure. */ + tools: Tool[] = $derived([...this.#assembledTools, ...this.planMode.tools]) + /** What the assistant can call in this session, posture-independent: both plan-mode + * transitions, whichever one is offered right now. A reference answer, so flipping the + * autonomy picker must not change it. */ + availableTools: Tool[] = $derived([...this.#assembledTools, ...this.planMode.availableTools]) helpers = $state(undefined) scriptEditorOptions = $state(undefined) @@ -1764,7 +1774,7 @@ export class AIChatManager implements ChatViewHost { try { this.apiTools = await loadApiTools() if (this.mode === AIMode.API) { - this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] + this.#assembledTools = [searchDocsTool, readDocsPageTool, ...this.apiTools] } } catch (err) { console.error('Error loading api tools', err) @@ -2346,7 +2356,7 @@ export class AIChatManager implements ChatViewHost { customPrompt ) this.systemMessage.content = this.systemMessage.content - this.tools = [...prepareScriptTools(currentModel, lang, context)] + this.#assembledTools = [...prepareScriptTools(currentModel, lang, context)] this.helpers = { getScriptOptions: () => { return { @@ -2378,7 +2388,7 @@ export class AIChatManager implements ChatViewHost { this.systemMessage = prepareFlowSystemMessage(customPrompt) this.systemMessage.content = this.systemMessage.content this.appendFlowAiAgentProviders(this.systemMessage) - this.tools = [...flowTools] + this.#assembledTools = [...flowTools] this.helpers = { ...(this.flowAiChatHelpers ?? {}), getWorkspaceMutationTarget: this.getFlowWorkspaceMutationTarget @@ -2386,17 +2396,17 @@ export class AIChatManager implements ChatViewHost { } else if (mode === AIMode.NAVIGATOR) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareNavigatorSystemMessage(customPrompt) - this.tools = [this.changeModeTool, ...navigatorTools] + this.#assembledTools = [this.changeModeTool, ...navigatorTools] this.helpers = {} } else if (mode === AIMode.ASK) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareAskSystemMessage(customPrompt) - this.tools = [...askTools] + this.#assembledTools = [...askTools] this.helpers = {} } else if (mode === AIMode.API) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareApiSystemMessage(customPrompt) - this.tools = [searchDocsTool, readDocsPageTool, ...this.apiTools] + this.#assembledTools = [searchDocsTool, readDocsPageTool, ...this.apiTools] this.helpers = {} } else if (mode === AIMode.GLOBAL) { this.configureGlobalMode() @@ -2406,7 +2416,7 @@ export class AIChatManager implements ChatViewHost { } else if (mode === AIMode.APP) { const customPrompt = getCombinedCustomPrompt(mode) this.systemMessage = prepareAppSystemMessage(customPrompt) - this.tools = [...getAppTools()] + this.#assembledTools = [...getAppTools()] this.helpers = this.appAiChatHelpers } } @@ -2445,7 +2455,7 @@ export class AIChatManager implements ChatViewHost { this.rebuildGlobalSystemMessage() } } - this.tools = assembleGlobalTools(opts) + this.#assembledTools = assembleGlobalTools(opts) this.helpers = pipeline ? { ...baseHelpers, pipeline } : baseHelpers this.systemMessage = assembleGlobalSystemMessage(getCustomPromptParts(AIMode.GLOBAL), opts) this.syncArtifactsSession() @@ -2941,7 +2951,7 @@ export class AIChatManager implements ChatViewHost { try { // Use JS getters so runChatLoop re-reads tools/helpers/systemMessage/modelProvider // on each iteration. This is critical for changeModeTool (Navigator → Script/Flow) - // which reassigns this.tools, this.helpers, this.systemMessage mid-loop. + // which reassigns #assembledTools, this.helpers, this.systemMessage mid-loop. const self = this // Pinned for the whole turn, like the `workspace` the loop routes through: // the global chat's operating workspace follows workspaceStore, so a switch @@ -2966,7 +2976,7 @@ export class AIChatManager implements ChatViewHost { return base }, get tools() { - return [...self.tools, ...self.planMode.tools] + return self.tools }, get helpers() { return self.helpers diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index ef15df4e8d..2962925844 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -4559,3 +4559,45 @@ describe('AIChatManager cross-tab run seams', () => { expect(manager.queuedMessage).toBe('') }) }) + +describe('AIChatManager tool views', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.getCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + mocks.tryGetCurrentModel.mockReturnValue({ provider: 'openai', model: 'gpt-4o' }) + }) + + // Every mode assigns the private base separately. A missed site leaves it empty on a fresh + // manager, which is what these catch — silently, since nothing throws on an empty tool set. + it.each([ + AIMode.SCRIPT, + AIMode.FLOW, + AIMode.NAVIGATOR, + AIMode.ASK, + AIMode.API, + AIMode.GLOBAL, + AIMode.APP + ])('assembles tools for %s', async (mode) => { + const manager = new AIChatManager() + await manager.changeMode(mode) + expect(manager.tools.length).toBeGreaterThan(0) + expect(manager.availableTools.length).toBeGreaterThan(0) + }) + + // Which transition each posture offers is planModeController.test.ts's; what this pins is + // that only `tools` follows the picker. `isSessionChat` is what offers plan mode at all. + it('holds availableTools steady across the autonomy picker', async () => { + const manager = new AIChatManager() + manager.isSessionChat = true + await manager.changeMode(AIMode.GLOBAL) + + const names = (tools: { def: { function: { name: string } } }[]) => + tools.map((t) => t.def.function.name) + const before = names(manager.availableTools) + expect(before).toEqual(expect.arrayContaining(['enter_plan_mode', 'exit_plan_mode'])) + + manager.setAutonomyMode(AIAutonomyMode.YOLO) + expect(names(manager.tools)).not.toContain('enter_plan_mode') + expect(names(manager.availableTools)).toEqual(before) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte index c9def4f9e6..1591b3f1b5 100644 --- a/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte +++ b/frontend/src/lib/components/copilot/chat/AssistantSettingsModal.svelte @@ -94,9 +94,9 @@ callers that already know which section they mean, such as the "+" menu's Manage return aiChatManager.operatingWorkspace ?? active ?? '' }) - // Exactly what the chat loop sends — plan mode's transition tool is registered - // alongside `tools` there, so listing only `tools` would under-report. - let tools = $derived(summarizeTools([...aiChatManager.tools, ...aiChatManager.planMode.tools])) + // The session's capability surface, not this turn's: the heading answers what the + // assistant can call here, so it must not shift when the autonomy picker does. + let tools = $derived(summarizeTools(aiChatManager.availableTools)) // `SidebarNavigation`'s item shape, which is what the instance and workspace settings // navs are built from too. diff --git a/frontend/src/lib/components/copilot/chat/chatViewHost.ts b/frontend/src/lib/components/copilot/chat/chatViewHost.ts index 5c4dceb7f1..bdacb57615 100644 --- a/frontend/src/lib/components/copilot/chat/chatViewHost.ts +++ b/frontend/src/lib/components/copilot/chat/chatViewHost.ts @@ -147,7 +147,10 @@ export interface ChatViewHost { * True where the bytes are forwarded somewhere — object storage — rather than read * in the browser. */ attachmentsAsBlobs: boolean - tools: Tool[] + /** What this chat's next request carries, the current posture's plan-mode transition + * included. A consumer narrows it by tool metadata; assembling its own list from the + * sources behind it is how the advertised set and the sent set drift apart. */ + readonly tools: readonly Tool[] autonomyMode: AIAutonomyMode setAutonomyMode: (mode: AIAutonomyMode) => void readonly autoAcceptEditsActive: boolean diff --git a/frontend/src/lib/components/copilot/chat/planModeController.svelte.ts b/frontend/src/lib/components/copilot/chat/planModeController.svelte.ts index ab80dc3e88..d76c1502dd 100644 --- a/frontend/src/lib/components/copilot/chat/planModeController.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/planModeController.svelte.ts @@ -106,6 +106,12 @@ export class PlanModeController { return this.#host.active ? [this.exitTool] : [this.enterTool] } + /** Both transitions, whichever posture is selected: what plan mode contributes to this + * chat's capabilities, rather than to the turn it is about to send. */ + get availableTools(): Tool[] { + return this.#host.available ? [this.enterTool, this.exitTool] : [] + } + // This safety tag is what keeps plan mode escapable through its handoff tool. exitTool: Tool = { def: createToolDef(exitPlanModeArgs, EXIT_PLAN_MODE_TOOL, EXIT_PLAN_MODE_TOOL_DESCRIPTION), From d8aaa73573c46a8a23f6c6850d38438ad395752c Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 22 Sep 2026 19:07:43 +0200 Subject: [PATCH 25/34] feat: chat with a saved agent from the agent editor (#11292) * feat: chat with a saved agent from the agent editor * fix: keep agent chats apart from a same-path flow's conversations * fix: point an agent editor chat's model gap at the agent, not a step * fix: match the memory gate's icon to the chat's empty state --- .../flows/content/AgentEditorHost.svelte | 202 +++++++++++++++--- .../flows/content/AgentEditorModal.svelte | 37 +++- .../flows/conversations/FlowChat.svelte | 13 +- .../conversations/FlowChatInterface.svelte | 11 +- .../flows/conversations/agentChatInputs.ts | 12 +- .../conversations/agentEditorChat.test.ts | 76 +++++++ .../flows/conversations/agentEditorChat.ts | 101 +++++++++ 7 files changed, 406 insertions(+), 46 deletions(-) create mode 100644 frontend/src/lib/components/flows/conversations/agentEditorChat.test.ts create mode 100644 frontend/src/lib/components/flows/conversations/agentEditorChat.ts diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index b6a98160be..4f69d3a1a1 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -1,7 +1,16 @@ {#if draft.refusal} @@ -359,7 +434,7 @@ - +
- - - - -
- + +
+ +
+ + +
+ +
+ + + + + +
+ {#if chatMounted} +
+ {#if chatGap?.memory} +
+ +

Chat needs managed memory

+

+ {chatGap.memoryCanTurnOn + ? 'Without it, every message would be answered without the ones before it.' + : 'This agent replays a fixed list of messages. Switch its memory to managed to chat with it.'} +

+ {#if chatGap.memoryCanTurnOn && !readOnly} + + {/if} +
+ {:else if chatGap?.noStream} +
+ + + {chatGap.noStream === 'image' + ? 'Image answers do not stream: each one shows once its run ends.' + : 'Streaming is off: each answer shows once its run ends.'} + + {#if chatGap.noStream === 'off' && !readOnly} + + {/if} +
+ {/if} + +
+ +
- - - - - + {/if} +
diff --git a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte index a6e113f39b..ea45761bf9 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorModal.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorModal.svelte @@ -1,5 +1,7 @@ @@ -81,25 +91,29 @@ {#if entry.kind === 'assistant'}
- {#if entry.sources} -
- -
- {/if}
{:else if entry.kind === 'search'} - + {}} - /> + label="Searched the web" + expanded={Boolean(entry.sources) && !collapsedSearches.has(index)} + toggleable={Boolean(entry.sources)} + onToggle={() => { + if (!collapsedSearches.delete(index)) collapsedSearches.add(index) + }} + > + {#if entry.sources} + + {/if} + {:else} {@const job = jobOf(entry.jobId)} + {@const failed = job?.type === 'CompletedJob' && !job.success} + {@const search = webSearchResultOf(entry.result)} toggle(index, entry)} contentClass="space-y-3" @@ -110,11 +124,15 @@ {#if job?.logs} {/if} - + {#if search && !failed} + + {:else} + + {/if} {#if entry.resourcePath}
diff --git a/frontend/src/lib/components/agentTrace.test.ts b/frontend/src/lib/components/agentTrace.test.ts index 3297a99d82..7d1dcf3429 100644 --- a/frontend/src/lib/components/agentTrace.test.ts +++ b/frontend/src/lib/components/agentTrace.test.ts @@ -45,7 +45,7 @@ describe('buildAgentTrace', () => { result: '{"eu-central-1":0.184}', jobId: '0199-job' }, - { kind: 'assistant', content: 'eu-central-1 is down.', sources: undefined } + { kind: 'assistant', content: 'eu-central-1 is down.' } ]) }) @@ -76,8 +76,8 @@ describe('buildAgentTrace', () => { // The worker splits a search the same way: a `tool` message tagged web_search // carrying a constant sentence, then the assistant turn that carries the - // citations. The search row therefore has nothing of its own to show. - it('records the search and puts its citations on the turn that follows', () => { + // citations, which move onto the search so it shows what it found. + it('puts the citations of the turn that follows on the search', () => { const entries = buildAgentTrace([ { role: 'tool', @@ -92,12 +92,11 @@ describe('buildAgentTrace', () => { } ]) expect(entries).toEqual([ - { kind: 'search' }, { - kind: 'assistant', - content: 'Postgres 17 changed the default.', + kind: 'search', sources: [{ url: 'https://postgresql.org/docs', title: 'Release notes' }] - } + }, + { kind: 'assistant', content: 'Postgres 17 changed the default.' } ]) }) @@ -118,7 +117,7 @@ describe('buildAgentTrace', () => { }) describe('splitFinalAnswer', () => { - it('moves the answering turn out of the trace, with its citations', () => { + it('moves the answering turn out of the trace, leaving its citations on the search', () => { const entries = buildAgentTrace([ { role: 'tool', content: 'Used websearch tool', agent_action: { type: 'web_search' } }, { @@ -128,10 +127,12 @@ describe('splitFinalAnswer', () => { agent_action: { type: 'message' } } ]) - expect(splitFinalAnswer(entries, 'Postgres 17 changed the default.')).toEqual({ - trace: [{ kind: 'search' }], - sources: [{ url: 'https://postgresql.org/docs', title: 'Release notes' }] - }) + expect(splitFinalAnswer(entries, 'Postgres 17 changed the default.')).toEqual([ + { + kind: 'search', + sources: [{ url: 'https://postgresql.org/docs', title: 'Release notes' }] + } + ]) }) // A run whose last turn returned a tool call and no text leaves its answer @@ -152,12 +153,12 @@ describe('splitFinalAnswer', () => { } } ]) - expect(splitFinalAnswer(entries, 'Let me check.').trace).toEqual([entries[1]]) + expect(splitFinalAnswer(entries, 'Let me check.')).toEqual([entries[1]]) }) it('leaves the trace whole when the output is not a turn of its own', () => { const entries = buildAgentTrace(messages) - expect(splitFinalAnswer(entries, { rows: 3 })).toEqual({ trace: entries }) + expect(splitFinalAnswer(entries, { rows: 3 })).toEqual(entries) }) }) diff --git a/frontend/src/lib/components/agentTrace.ts b/frontend/src/lib/components/agentTrace.ts index ba0cb78d4e..392e8b00c9 100644 --- a/frontend/src/lib/components/agentTrace.ts +++ b/frontend/src/lib/components/agentTrace.ts @@ -8,10 +8,10 @@ import type { AgentMessage } from './aiAgentResult' * are shown as inputs, while the trace is what the agent did with them. */ export type AgentTraceEntry = - | { kind: 'assistant'; content: string; sources?: WebSearchSource[] } - /** A search records only that one happened: the worker writes a constant - * sentence, and the citations ride on the assistant turn that follows. */ - | { kind: 'search' } + | { kind: 'assistant'; content: string } + /** The worker records only that a search happened. Its citations ride on the assistant + * turn that follows, and are moved here so the search shows what it found. */ + | { kind: 'search'; sources?: WebSearchSource[] } | { kind: 'tool' name: string @@ -99,8 +99,14 @@ export function buildAgentTrace(messages: AgentMessage[]): AgentTraceEntry[] { continue } const content = contentText(message.content) + // The worker writes a provider call's search right before that call's answer, and + // only a call that searched carries annotations. + const previous = entries[entries.length - 1] + if (message.role === 'assistant' && previous?.kind === 'search' && !previous.sources) { + previous.sources = sourcesOf(message) + } if (message.role === 'assistant' && content !== '') { - entries.push({ kind: 'assistant', content, sources: sourcesOf(message) }) + entries.push({ kind: 'assistant', content }) } } return entries @@ -108,27 +114,21 @@ export function buildAgentTrace(messages: AgentMessage[]): AgentTraceEntry[] { /** * Separates the turn that produced the output from the rest of the trace, so the - * answer is rendered once with the citations that belong to it. Found by content - * and searched from the end: a run whose last turn returned a tool call leaves its - * answer mid-trace, where inspecting only the final entry prints it twice. + * answer is rendered once. Found by content and searched from the end: a run whose + * last turn returned a tool call leaves its answer mid-trace, where inspecting only + * the final entry prints it twice. */ -export function splitFinalAnswer( - entries: AgentTraceEntry[], - output: unknown -): { trace: AgentTraceEntry[]; sources?: WebSearchSource[] } { +export function splitFinalAnswer(entries: AgentTraceEntry[], output: unknown): AgentTraceEntry[] { if (typeof output !== 'string') { // A schema-shaped output is rendered by the result viewer itself and matches // no turn, so the whole trace stands. - return { trace: entries } + return entries } for (let i = entries.length - 1; i >= 0; i--) { const entry = entries[i] if (entry.kind === 'assistant' && entry.content === output) { - // The citations are an annotation on that turn, so moving the turn without - // them would leave a run that shows a web search ran and no source for - // what it answered. - return { trace: [...entries.slice(0, i), ...entries.slice(i + 1)], sources: entry.sources } + return [...entries.slice(0, i), ...entries.slice(i + 1)] } } - return { trace: entries } + return entries } diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 5405a6ea25..8d945f94d1 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -26,7 +26,7 @@ import { getChatViewHost } from './chatViewHost' const chatHost = getChatViewHost() - import { isActiveUserQuestion, type ToolDisplayMessage } from './shared' + import { isActiveUserQuestion, webSearchResultOf, type ToolDisplayMessage } from './shared' import ChatCollapsibleCard from './ChatCollapsibleCard.svelte' import { twMerge } from 'tailwind-merge' import { slide } from 'svelte/transition' @@ -140,6 +140,18 @@ message.previewCard && !message.isLoading && !message.error && !message.needsConfirmation ) ) + + // A provider-side search sets `webSearchSources` and words its own header; any other tool + // gets the same card by returning the web search result shape. + const searchResult = $derived( + message.webSearchSources || message.error ? undefined : webSearchResultOf(message.result) + ) + const sources = $derived(message.webSearchSources ?? searchResult?.sources) + const label = $derived( + searchResult?.query !== undefined + ? `${message.content} · "${searchResult.query}"` + : message.content + ) {#if activeUserQuestion} @@ -281,7 +293,7 @@ weight alone: queued calls (waiting their turn behind the executing tool) are faded, the running one sweeps, a settled one is plain. --> (isExpanded = !isExpanded)} toggleable={detailsAvailable || message.isStreamingArguments === true} @@ -343,8 +355,8 @@ {#if visibleActions.length > 0} - {:else if message.webSearchSources?.length && !message.error} - + {:else if sources?.length && !message.error} + {:else} import { Globe } from 'lucide-svelte' import { SvelteSet } from 'svelte/reactivity' - import type { WebSearchSource } from './shared' + import { isRenderableSourceUrl, type WebSearchSource } from './shared' import { isOfflineReplay } from '$lib/components/recording/offlineReplay.svelte' interface Props { sources: WebSearchSource[] + /** Whether a hostname may be sent to the favicon service. False for the sources a + * workspace's own tool returned: those can name internal services, and disclosing + * them to a third party is the tool author's call to make, not this card's. */ + favicons?: boolean } - let { sources }: Props = $props() + let { sources, favicons = true }: Props = $props() - // The URLs come from the provider's response: only render absolute http(s) - // ones — anything else (javascript:, data:, relative) must not become an - // href. Also dedupes; providers can surface the same page several times. + // The URLs come from the provider's response or from a tool's own result: only render + // absolute http(s) ones — anything else (javascript:, data:, relative) must not become + // an href. Also dedupes; providers can surface the same page several times. const uniqueSources = $derived( Array.from( - new Map( - sources - .filter((s) => { - try { - return ['http:', 'https:'].includes(new URL(s.url).protocol) - } catch { - return false - } - }) - .map((s) => [s.url, s]) - ).values() + new Map(sources.filter((s) => isRenderableSourceUrl(s.url)).map((s) => [s.url, s])).values() ) ) @@ -43,7 +37,7 @@ // to issue none — a recording comes from an arbitrary origin, so its cited // hostnames must not leak from a viewer's browser either. Degrades to the same // Globe the blocked/failed case already uses. - const noFavicons = $derived(isOfflineReplay()) + const noFavicons = $derived(!favicons || isOfflineReplay()) // Favicons come from Google's public favicon service, which discloses each // consulted hostname to a third party from the user's browser — an accepted @@ -62,6 +56,9 @@
{#each uniqueSources as source (source.url)} {@const hostname = hostnameOf(source.url)} + + {@const title = source.title?.trim() || undefined} failedFavicons.add(hostname)} /> {/if} - {source.title ?? hostname} - {#if source.title} + {title ?? hostname} + {#if title} {hostname} {/if} diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 005a7666db..fb61f3a912 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { z } from 'zod' import type { DisplayMessage, ToolDisplayMessage } from './shared' -import { openItemPreviewAction } from './shared' +import { openItemPreviewAction, webSearchResultOf } from './shared' vi.mock('monaco-editor', () => ({ editor: {} @@ -1869,3 +1869,38 @@ describe('processToolCall confirmation hooks', () => { expect(tool.fn).toHaveBeenCalled() }) }) + +// Any tool gets the web search card by returning this shape. The card renders urls and +// titles only, so a result carrying anything more must keep its JSON pane. +describe('webSearchResultOf', () => { + it('reads the shape, from the value or its JSON text', () => { + const result = { sources: [{ url: 'https://a.dev', title: 'A' }], query: 'a' } + expect(webSearchResultOf(result)).toEqual(result) + expect(webSearchResultOf(JSON.stringify(result))).toEqual(result) + }) + + // A Python tool serializes an absent optional as null, and the card has nothing to tell + // its author why an almost-right result fell back to JSON. + it('reads null on an optional field as absent', () => { + expect( + webSearchResultOf({ sources: [{ url: 'https://a.dev', title: null }], query: null }) + ).toEqual({ + sources: [{ url: 'https://a.dev', title: undefined }], + query: undefined + }) + }) + + it.each([ + ['an extra key', { sources: [{ url: 'https://a.dev' }], summary: 'x' }], + ['an extra source field', { sources: [{ url: 'https://a.dev', snippet: 'x' }] }], + ['a source without url', { sources: [{ title: 'A' }] }], + // The card renders no relative or javascript: link, so a result whose urls it would + // drop keeps its own JSON rather than showing an empty source list. + ['a relative url', { sources: [{ url: '/docs/pg17', title: 'PG 17' }] }], + ['a javascript: url', { sources: [{ url: 'javascript:alert(1)' }] }], + ['no sources', { sources: [] }], + ['a bare list of links', [{ url: 'https://a.dev' }]] + ])('rejects %s', (_, result) => { + expect(webSearchResultOf(result)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index f58ed3b6d3..772284865d 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -610,6 +610,77 @@ export type WebSearchSource = { title?: string } +/** The result shape any tool returns to have it rendered as a web search card. */ +export type WebSearchResult = { + sources: WebSearchSource[] + query?: string +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +/** A link the card can render: the source list drops everything else, so a result whose + * urls are relative or `javascript:` would show as an empty list in place of its own output. */ +export function isRenderableSourceUrl(url: string): boolean { + try { + return ['http:', 'https:'].includes(new URL(url).protocol) + } catch { + return false + } +} + +/** An absent optional field reaches JSON as `null` from a Python tool and as nothing from a + * TypeScript one, so both read as absent. */ +function isOptionalString(value: unknown): boolean { + return value === null || typeof value === 'string' +} + +function isWebSearchSource(value: unknown): value is { url: string; title?: unknown } { + return ( + isRecord(value) && + typeof value.url === 'string' && + isRenderableSourceUrl(value.url) && + Object.entries(value).every( + ([key, v]) => key === 'url' || (key === 'title' && isOptionalString(v)) + ) + ) +} + +/** + * A tool result as a web search, or undefined when it is anything else. The keys must be + * exactly `sources` and an optional `query`: the card renders only urls and titles, so a + * result carrying more would lose it. Accepts the result as its JSON text too. + */ +export function webSearchResultOf(result: unknown): WebSearchResult | undefined { + if (typeof result === 'string') { + try { + result = JSON.parse(result) + } catch { + return undefined + } + } + if (!isRecord(result) || !Array.isArray(result.sources) || result.sources.length === 0) { + return undefined + } + const sources = result.sources + if ( + !sources.every(isWebSearchSource) || + !Object.entries(result).every( + ([key, v]) => key === 'sources' || (key === 'query' && isOptionalString(v)) + ) + ) { + return undefined + } + return { + sources: sources.map((source) => ({ + url: source.url, + title: typeof source.title === 'string' ? source.title : undefined + })), + query: typeof result.query === 'string' ? result.query : undefined + } +} + export type ToolDisplayMessage = { role: 'tool' tool_call_id: string diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts index 6ef18b7212..077090aba2 100644 --- a/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.svelte.ts @@ -3,7 +3,7 @@ import type { ChatSendRequestOptions, ChatViewHost } from '$lib/components/copilot/chat/chatViewHost' -import type { DisplayMessage } from '$lib/components/copilot/chat/shared' +import type { DisplayMessage, WebSearchResult } from '$lib/components/copilot/chat/shared' import type { AIAutonomyMode } from '$lib/components/copilot/chat/AIChatManager.svelte' import { isPlanCardTool } from '$lib/components/copilot/chat/planMode' import { AttachedFilesStore } from '$lib/components/copilot/chat/files/attachedFiles.svelte' @@ -125,6 +125,33 @@ function toolErrorText(result: unknown): string | undefined { return undefined } +/** + * The provider-side search's row as the web search result shape, or undefined for any other + * row. The worker words it like a tool named `websearch`, which an agent tool may also be + * called; what tells them apart is that every agent tool's row stores its arguments, and the + * search stores none, only the citations it produced. + */ +function builtInWebSearch(message: ChatMessage): WebSearchResult | undefined { + const tool = message.tool + if (tool?.name !== 'websearch' || tool.arguments !== undefined || message.pending) { + return undefined + } + const citations = parseToolPayload(tool.result) + if (citations === undefined) return { sources: [] } + if ( + !Array.isArray(citations) || + !citations.every((c) => c && typeof c === 'object' && typeof c.url === 'string') + ) { + return undefined + } + return { + sources: citations.map((c: { url: string; title?: unknown }) => ({ + url: c.url, + title: typeof c.title === 'string' ? c.title : undefined + })) + } +} + /** The name the worker gives the structured-output tool: suffixed when an agent tool already has it. */ const STRUCTURED_OUTPUT_CALL = /^structured_output(_\d+)?$/ @@ -168,6 +195,25 @@ export function toDisplayMessages( ] } case 'tool': { + const search = builtInWebSearch(message) + if (search) { + const found = search.sources.length > 0 + // `autoCollapseDetails: false` opens it on its sources, as the session chat + // shows a search. A tool that returns the same shape stays collapsed like + // every other tool row, however many times the agent called it. + return [ + { + role: 'tool', + tool_call_id: message.id, + content: 'Searched the web', + toolName: 'web_search', + webSearchSources: found ? search.sources : undefined, + showDetails: found, + autoCollapseDetails: false, + jobId: message.jobId + } + ] + } // The model's call and what the tool sent back, as the stream carried them or the // worker stored them on the row. A row stored without them shows its name and job. const toolName = message.tool?.name diff --git a/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts index 53fd7b6ab9..a0ae4b8092 100644 --- a/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts +++ b/frontend/src/lib/components/flows/conversations/flowChatViewHost.test.ts @@ -125,6 +125,42 @@ describe('toDisplayMessages', () => { }) }) + // The worker words the provider-side search like a tool named `websearch`, which an agent + // tool may also be called. The search stores no arguments, and an agent tool always does. + it('renders the provider-side search as a web search card, not an agent tool of that name', () => { + const citations = JSON.stringify([ + { start_index: 0, end_index: 4, url: 'https://postgresql.org/docs', title: 'Release notes' } + ]) + const [search, agentTool, notCitations] = toDisplayMessages([ + message({ + role: 'tool', + content: 'Used websearch tool', + tool: { name: 'websearch', status: 'success', result: citations } + }), + message({ + role: 'tool', + content: 'Used websearch tool', + tool: { name: 'websearch', status: 'success', arguments: '{}', result: citations } + }), + // The safety valve of a name-based rule: a row that is not the provider's search + // keeps its result, whatever the tool is called. + message({ + role: 'tool', + content: 'Used websearch tool', + tool: { name: 'websearch', status: 'success', result: '{"hits":3}' } + }) + ]) + expect(search).toMatchObject({ + content: 'Searched the web', + webSearchSources: [{ url: 'https://postgresql.org/docs', title: 'Release notes' }], + showDetails: true, + autoCollapseDetails: false + }) + expect(agentTool).toMatchObject({ content: 'Used websearch tool', toolName: 'websearch' }) + expect(agentTool).not.toHaveProperty('webSearchSources') + expect(notCitations).toMatchObject({ toolName: 'websearch', result: { hits: 3 } }) + }) + it('shows a call the turn finished without as an error, not as still running', () => { const display = toDisplayMessages([ message({ role: 'user', content: 'hi' }), From 1de54eeea4c4820e7291d9648a44ef3c86979c5a Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 23 Sep 2026 15:44:40 +0200 Subject: [PATCH 29/34] feat(ai-chat): show native script edit diffs (#11278) * feat(ai-chat): show native script edit diffs * fix: render streamed edit diffs * fix: render in-editor script edit diffs * style: soften diff context separators * fix: render full script update diffs * fix: support empty script diffs * fix: bound streamed tool diffs * fix: preserve trailing diff context * fix: defer collapsed tool diffs * fix: retain full-code streaming previews * fix: preserve created script diffs * fix: disambiguate collapsed diff keys * fix: render final newline diffs * fix: preserve multiline diff highlighting * fix: highlight diff sides as complete sources * fix: bound completed tool diffs * fix: diff large completed tool edits Co-Authored-By: Claude Opus 5.5 (1M context) * fix: bound tool diff rendering Co-Authored-By: Claude Opus 5.5 (1M context) * fix: bound tool diff highlighting and expand omitted rows Co-Authored-By: Claude Opus 5.5 (1M context) * fix: reveal omitted diff rows in chunks and bound long-line highlighting Co-Authored-By: Claude Opus 5.5 (1M context) * fix: collapse written scripts, skip empty diff bodies and fit line numbers Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep whole-file tool diffs collapsed while running Co-Authored-By: Claude Opus 5.5 (1M context) * fix: classify whole-file tool calls by argument shape Co-Authored-By: Claude Opus 5.5 (1M context) * fix: reopen failed diff cards and keep empty streamed replacements Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../copilot/chat/ToolCodeDiffView.svelte | 170 ++++++++ .../copilot/chat/ToolDiffCard.svelte | 103 +++++ .../copilot/chat/ToolExecutionDisplay.svelte | 5 + .../copilot/chat/global/core.test.ts | 38 +- .../components/copilot/chat/global/core.ts | 35 +- .../copilot/chat/script/core.test.ts | 50 +++ .../components/copilot/chat/script/core.ts | 15 +- .../src/lib/components/copilot/chat/shared.ts | 10 + .../copilot/chat/toolCodeDiff.test.ts | 411 ++++++++++++++++++ .../components/copilot/chat/toolCodeDiff.ts | 381 ++++++++++++++++ .../copilot/chat/toolCodeDiffHighlight.ts | 57 +++ .../copilot/chat/toolCodeDiffLanguage.ts | 43 ++ .../test_dev/diff-comparison/+page.svelte | 139 ++++++ 13 files changed, 1443 insertions(+), 14 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/ToolCodeDiffView.svelte create mode 100644 frontend/src/lib/components/copilot/chat/ToolDiffCard.svelte create mode 100644 frontend/src/lib/components/copilot/chat/script/core.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/toolCodeDiff.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/toolCodeDiff.ts create mode 100644 frontend/src/lib/components/copilot/chat/toolCodeDiffHighlight.ts create mode 100644 frontend/src/lib/components/copilot/chat/toolCodeDiffLanguage.ts create mode 100644 frontend/src/routes/test_dev/diff-comparison/+page.svelte diff --git a/frontend/src/lib/components/copilot/chat/ToolCodeDiffView.svelte b/frontend/src/lib/components/copilot/chat/ToolCodeDiffView.svelte new file mode 100644 index 0000000000..6f74ff7a80 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/ToolCodeDiffView.svelte @@ -0,0 +1,170 @@ + + + + +
+ {#each visibleRows as row, index ('key' in row ? `${row.kind}:${row.key}` : `line:${row.oldLine}:${row.newLine}:${index}`)} + {#if row.kind === 'collapsed' || row.kind === 'omitted'} + + {:else} +
+ {row.oldLine ?? ''} + + {row.newLine ?? ''}{row.kind === 'added' ? '+' : row.kind === 'removed' ? '-' : ''} + + + {#each row.changedRanges ?? [] as range} + + {/each} + {@html highlightedLine(row) || ' '} + +
+ {/if} + {/each} +
+ + diff --git a/frontend/src/lib/components/copilot/chat/ToolDiffCard.svelte b/frontend/src/lib/components/copilot/chat/ToolDiffCard.svelte new file mode 100644 index 0000000000..a2ab9d341c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/ToolDiffCard.svelte @@ -0,0 +1,103 @@ + + +{#snippet headerRight()} +
+ {#if counts && (counts.added > 0 || counts.removed > 0)} + + +{counts.added} + −{counts.removed} + + {/if} + {#if showPreviewChip && message.previewCard} + + {/if} +
+{/snippet} + + (toggled = { id: toggleKey, open: !expanded })} + toggleable={hasBody} + shimmer={isRunning} + class={message.isQueued && !message.error + ? 'opacity-60 hover:opacity-100 transition-opacity' + : ''} + labelClass="truncate" + contentClass="p-0 overflow-hidden space-y-0" + {headerRight} +> + {#if diff && hasDiff && expanded} + + {/if} + {#if message.error} +
+ {message.error} +
+ {/if} + diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 8d945f94d1..d5fee5a4d9 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -40,6 +40,8 @@ import ToolPreviewCard from './ToolPreviewCard.svelte' import AskUserQuestionDisplay from './AskUserQuestionDisplay.svelte' import RunScriptCard from './RunScriptCard.svelte' + import ToolDiffCard from './ToolDiffCard.svelte' + import { hasToolCodeDiff } from './toolCodeDiff' import WebSearchSourcesDisplay from './WebSearchSourcesDisplay.svelte' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' import McpServerIcon from '$lib/components/mcp/McpServerIcon.svelte' @@ -132,6 +134,7 @@ // A call that inspected a run rather than starting one gets the same card, bound to // the job it named — what happened in a run reads the same either way. const isRunCard = $derived(Boolean(message.runForm || message.inspectedRun)) + const isDiffCard = $derived(Boolean(message.codeDiff) || hasToolCodeDiff(message.toolName)) // The preview chip sits on the header row (to the right of the tool-call text); // shown once the tool settled, never while loading/erroring/awaiting confirmation. @@ -170,6 +173,8 @@
{:else if isRunCard} +{:else if isDiffCard} + {:else if planState} diff --git a/frontend/src/lib/components/copilot/chat/global/core.test.ts b/frontend/src/lib/components/copilot/chat/global/core.test.ts index 7a86e0ba76..a1755bcaa9 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.test.ts @@ -2024,6 +2024,24 @@ describe('global AI tools', () => { }) }) + it('records a creation diff for a new script draft', async () => { + const statuses: any[] = [] + + await callGlobalTool( + 'write_script', + { + path: 'f/scripts/new-script', + language: 'python3', + content: 'print("hello")' + }, + { ...toolCallbacks, setToolStatus: (_toolId, status) => statuses.push(status) } + ) + + expect(statuses).toContainEqual({ + codeDiff: { before: '', after: 'print("hello")', lang: 'python' } + }) + }) + it('tells a code app from a drag-and-drop app', async () => { vi.mocked(AppService.listApps).mockResolvedValueOnce([ { path: 'f/apps/code', summary: 'Code app', raw_app: true }, @@ -3080,12 +3098,17 @@ describe('global AI tools', () => { kind: 'script' } as any) - await callGlobalTool('write_script', { - path: 'f/scripts/existing', - summary: 'new summary', - language: 'bun', - content: 'new content' - }) + const statuses: any[] = [] + await callGlobalTool( + 'write_script', + { + path: 'f/scripts/existing', + summary: 'new summary', + language: 'bun', + content: 'new content' + }, + { ...toolCallbacks, setToolStatus: (_toolId, status) => statuses.push(status) } + ) expect( getBackendDraft('script', 'f/scripts/existing', { workspace: WORKSPACE }) @@ -3097,6 +3120,9 @@ describe('global AI tools', () => { content: 'new content', language: 'bun' }) + expect(statuses).toContainEqual({ + codeDiff: { before: 'old deployed content', after: 'new content', lang: 'typescript' } + }) }) describe('stale-draft deploy guard and rebase', () => { diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 5b0490c724..a4581bd173 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -137,8 +137,10 @@ import { type RunFormDisplay, type Tool, type ToolCallbacks, + type ToolCodeDiff, type ToolDisplayAction } from '../shared' +import { scriptLangToEditorLang } from '$lib/scripts' import { searchDocsTool, readDocsPageTool } from '../docs/core' import { createDbSchemaTool } from '../script/core' import type { ContextElement } from '../context' @@ -5029,10 +5031,12 @@ function finishAppDraftWrite( function finishDraftWrite( result: DraftPersistResult, existed: boolean, - ctx: WriteDraftCtx + ctx: WriteDraftCtx, + codeDiff?: ToolCodeDiff ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + if (codeDiff) ctx.toolCallbacks.setToolStatus(ctx.toolId, { codeDiff }) ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) maybeAttachPreviewCard(ctx, result.itemKind, result.item.path) const stored = result.item @@ -5078,7 +5082,11 @@ async function writeDraft( path: string, args: A, ctx: WriteDraftCtx, - opts: { triggerKind?: TriggerKind; override?: boolean } = {} + opts: { + triggerKind?: TriggerKind + override?: boolean + codeDiff?: ToolCodeDiff | ((base: T | undefined, draft: T) => ToolCodeDiff | undefined) + } = {} ): Promise { const { workspace } = ctx startDraftWrite(ctx, type, path) @@ -5097,7 +5105,8 @@ async function writeDraft( triggerKind: opts.triggerKind, force: opts.override }) - return finishDraftWrite(result, existed, ctx) + const codeDiff = typeof opts.codeDiff === 'function' ? opts.codeDiff(base, draft) : opts.codeDiff + return finishDraftWrite(result, existed, ctx, codeDiff) } type ScriptDraftArgs = { @@ -5150,8 +5159,21 @@ const SCRIPT_SPEC: WriteSpec = { } } -function writeScriptDraft(args: ScriptDraftArgs, ctx: WriteDraftCtx): Promise { - return writeDraft(SCRIPT_SPEC, 'script', args.path, args, ctx, { override: args.override }) +function writeScriptDraft( + args: ScriptDraftArgs, + ctx: WriteDraftCtx, + codeDiff?: ToolCodeDiff +): Promise { + return writeDraft(SCRIPT_SPEC, 'script', args.path, args, ctx, { + override: args.override, + codeDiff: + codeDiff ?? + ((base, draft) => ({ + before: base?.content ?? '', + after: draft.content, + lang: scriptLangToEditorLang(draft.language) + })) + }) } type FlowDraftArgs = { @@ -5368,7 +5390,8 @@ async function editScript( language: base.language, content: updated }, - ctx + ctx, + { before: base.content, after: updated, lang: scriptLangToEditorLang(base.language) } ) } diff --git a/frontend/src/lib/components/copilot/chat/script/core.test.ts b/frontend/src/lib/components/copilot/chat/script/core.test.ts new file mode 100644 index 0000000000..f235ae3796 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/script/core.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('monaco-editor', () => ({ + editor: {}, + languages: {}, + KeyCode: {}, + Uri: { parse: (value: string) => ({ toString: () => value }) }, + MarkerSeverity: { Error: 8, Warning: 4, Info: 2, Hint: 1 } +})) +vi.mock('@codingame/monaco-vscode-standalone-typescript-language-features', () => ({ + getTypeScriptWorker: async () => async () => ({}), + typescriptVersion: 'test' +})) +vi.mock('@codingame/monaco-vscode-languages-service-override', () => ({ default: () => ({}) })) +vi.mock('$lib/components/vscode', () => ({})) + +import { editCodeToolWithDiff } from './core' + +describe('editCodeToolWithDiff', () => { + it('records the complete script diff after applying replacements', async () => { + const statuses: unknown[] = [] + const applyCode = vi.fn(async () => {}) + + await editCodeToolWithDiff.fn({ + args: { diffs: [{ old_string: 'return 1', new_string: 'return 2' }] }, + helpers: { + getScriptOptions: () => ({ + code: 'export function main() {\n\treturn 1\n}', + lang: 'bun', + path: 'f/example', + args: {} + }), + applyCode + }, + toolCallbacks: { + setToolStatus: (_toolId, status) => statuses.push(status) + }, + toolId: 'edit-1' + }) + + expect(applyCode).toHaveBeenCalledTimes(2) + expect(statuses.at(-1)).toMatchObject({ + codeDiff: { + before: 'export function main() {\n\treturn 1\n}', + after: 'export function main() {\n\treturn 2\n}', + lang: 'typescript' + } + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 622f182eae..0c8b3f9a0e 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -27,6 +27,7 @@ import type { ReviewChangesOpts } from '../monaco-adapter' import { copilotInfo, getCurrentModel } from '$lib/aiStore' import { get } from 'svelte/store' import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/metadata' +import { scriptLangToEditorLang } from '$lib/scripts' import { getScriptPrompt, getWorkflowAsCodePrompt } from '$system_prompts' // Score threshold for npm packages search filtering @@ -826,7 +827,12 @@ export const editCodeToolWithDiff: Tool = { toolCallbacks.setToolStatus(toolId, { content: `Code changes applied`, - result: 'Success' + result: 'Success', + codeDiff: { + before: oldCode, + after: updatedCode, + lang: scriptLangToEditorLang(scriptOptions.lang) + } }) return `Applied changes to the script editor.` } catch (error) { @@ -878,7 +884,12 @@ export const editCodeTool: Tool = { toolCallbacks.setToolStatus(toolId, { content: 'Code changes applied', - result: 'Success' + result: 'Success', + codeDiff: { + before: oldCode, + after: args.code, + lang: scriptLangToEditorLang(scriptOptions.lang) + } }) return 'Code has been applied to the script editor.' } catch (error) { diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 772284865d..83a51b527c 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -610,6 +610,13 @@ export type WebSearchSource = { title?: string } +export type ToolCodeDiff = { + before: string + after: string + /** Monaco language id. */ + lang: string +} + /** The result shape any tool returns to have it rendered as a web search card. */ export type WebSearchResult = { sources: WebSearchSource[] @@ -725,6 +732,9 @@ export type ToolDisplayMessage = { * always-visible card that opens (or focuses) the item's preview in the * session side panel. Set only for session chats — the side panel is their surface. */ previewCard?: { kind: PreviewCardKind; path: string } + /** Whole text on both sides of an edit this call saved. Kept in full rather than as a + * reference: drafts are overwritten in place, so the sides cannot be fetched back later. */ + codeDiff?: ToolCodeDiff planArtifactId?: string /** The version this card's proposal wrote, so a card scrolled far up still opens the plan * it proposed rather than what the document became. */ diff --git a/frontend/src/lib/components/copilot/chat/toolCodeDiff.test.ts b/frontend/src/lib/components/copilot/chat/toolCodeDiff.test.ts new file mode 100644 index 0000000000..5077585a4f --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCodeDiff.test.ts @@ -0,0 +1,411 @@ +import { describe, expect, it } from 'vitest' +import hljs from 'highlight.js/lib/core' +import { + argumentsCarryWholeFile, + diffLineCounts, + hasToolCodeDiff, + toolCodeDiff, + toolDiffLines, + visibleToolDiffRows +} from './toolCodeDiff' +import { highlightedSourceLines } from './toolCodeDiffHighlight' +import { TOOL_CODE_DIFF_LANGUAGES, toolCodeDiffLanguage } from './toolCodeDiffLanguage' +import type { ToolDisplayMessage } from './shared' + +function message(overrides: Partial): ToolDisplayMessage { + return { + role: 'tool', + tool_call_id: 'call-1', + content: 'Updated script "f/example" as a draft', + ...overrides + } +} + +describe('toolCodeDiff', () => { + it('uses the saved full script sides after an edit succeeds', () => { + const codeDiff = { + before: 'const before = true\n', + after: 'const after = true\n', + lang: 'typescript' + } + + expect( + toolCodeDiff( + message({ + toolName: 'edit_script', + parameters: { old_string: 'before', new_string: 'after' }, + codeDiff + }) + ) + ).toBe(codeDiff) + }) + + it('uses edit arguments until the full script sides are available', () => { + expect( + toolCodeDiff( + message({ + toolName: 'edit_script', + parameters: { old_string: 'before', new_string: 'after' } + }) + ) + ).toEqual({ before: 'before', after: 'after', lang: 'plaintext' }) + }) + + it('renders streamed edit arguments before their JSON object is complete', () => { + expect( + toolCodeDiff( + message({ + toolName: 'edit_script', + parameters: '{"old_string":"before\\nline","new_string":"after' + }) + ) + ).toEqual({ before: 'before\nline', after: 'after', lang: 'plaintext' }) + }) + + it('uses each replacement from an in-editor edit while its arguments stream', () => { + expect( + toolCodeDiff( + message({ + toolName: 'edit_code', + parameters: { + diffs: [ + { old_string: 'first old', new_string: 'first new' }, + { old_string: 'second old', new_string: 'second new' } + ] + } + }) + ) + ).toEqual({ + before: 'first old\nsecond old', + after: 'first new\nsecond new', + lang: 'plaintext' + }) + }) + + it('uses each replacement while in-editor edit arguments are incomplete', () => { + expect( + toolCodeDiff( + message({ + toolName: 'edit_code', + parameters: + '{"diffs":[{"old_string":"first old","new_string":"first new"},{"old_string":"second old","new_string":"second new' + }) + ) + ).toEqual({ + before: 'first old\nsecond old', + after: 'first new\nsecond new', + lang: 'plaintext' + }) + }) + + it('shows a streamed full-code editor update without replacements', () => { + expect( + toolCodeDiff(message({ toolName: 'edit_code', parameters: '{"code":"console.log(1)' })) + ).toEqual({ before: '', after: 'console.log(1)', lang: 'plaintext' }) + }) + + it('shows streamed full-script updates before the saved diff is available', () => { + expect( + toolCodeDiff( + message({ toolName: 'write_script', parameters: '{"content":"print(\\\"hello\\\")' }) + ) + ).toEqual({ before: '', after: 'print("hello")', lang: 'plaintext' }) + }) + + it('only marks tools with a defined argument diff as diff-capable', () => { + expect(hasToolCodeDiff('edit_script')).toBe(true) + expect(hasToolCodeDiff('edit_code')).toBe(true) + expect(hasToolCodeDiff('write_script')).toBe(true) + expect(hasToolCodeDiff('__proto__')).toBe(false) + expect(hasToolCodeDiff('toString')).toBe(false) + expect(hasToolCodeDiff(undefined)).toBe(false) + }) + + it('ignores tool names inherited from Object.prototype', () => { + expect(toolCodeDiff(message({ toolName: 'constructor', parameters: {} }))).toBeUndefined() + }) + + it('counts added and removed lines independently', () => { + expect( + diffLineCounts({ + before: 'one\ntwo\nthree\n', + after: 'one\nnew\nthree\nfour\n', + lang: 'plaintext' + }) + ).toEqual({ + added: 2, + removed: 1 + }) + }) + + it('keeps a final newline as a diffable line', () => { + expect(toolDiffLines({ before: 'one', after: 'one\n', lang: 'plaintext' })).toEqual([ + { kind: 'context', content: 'one', oldLine: 1, newLine: 1 }, + { kind: 'added', content: '', newLine: 2 } + ]) + }) + + it('does not add a final-newline row when creating or deleting a file', () => { + expect(toolDiffLines({ before: '', after: 'one\n', lang: 'plaintext' })).toEqual([ + { kind: 'added', content: 'one', newLine: 1, changedRanges: [{ start: 0, length: 3 }] } + ]) + expect(toolDiffLines({ before: 'one\n', after: '', lang: 'plaintext' })).toEqual([ + { kind: 'removed', content: 'one', oldLine: 1, changedRanges: [{ start: 0, length: 3 }] } + ]) + }) + + it('keeps both gutters correct across removed and added lines', () => { + expect( + toolDiffLines({ + before: 'one\ntwo\nthree\n', + after: 'one\nnew\nthree\nfour\n', + lang: 'plaintext' + }) + ).toEqual([ + { kind: 'context', content: 'one', oldLine: 1, newLine: 1 }, + { + kind: 'removed', + content: 'two', + oldLine: 2, + newLine: undefined, + changedRanges: [{ start: 0, length: 3 }] + }, + { + kind: 'added', + content: 'new', + oldLine: undefined, + newLine: 2, + changedRanges: [{ start: 0, length: 3 }] + }, + { kind: 'context', content: 'three', oldLine: 3, newLine: 3 }, + { + kind: 'added', + content: 'four', + oldLine: undefined, + newLine: 4, + changedRanges: [{ start: 0, length: 4 }] + } + ]) + }) + + it('keeps unchanged trailing lines out of the change counts', () => { + const diff = { before: 'one\ntwo\nthree', after: 'one\nnew\nthree', lang: 'plaintext' } + + expect(diffLineCounts(diff)).toEqual({ added: 1, removed: 1 }) + expect(toolDiffLines(diff).filter((line) => line.content === 'three')).toEqual([ + { kind: 'context', content: 'three', oldLine: 3, newLine: 3 } + ]) + }) + + it('does not report changes for identical files', () => { + const diff = { before: 'one\ntwo\nthree', after: 'one\ntwo\nthree', lang: 'plaintext' } + + expect(diffLineCounts(diff)).toEqual({ added: 0, removed: 0 }) + expect(toolDiffLines(diff)).toEqual([ + { kind: 'context', content: 'one', oldLine: 1, newLine: 1 }, + { kind: 'context', content: 'two', oldLine: 2, newLine: 2 }, + { kind: 'context', content: 'three', oldLine: 3, newLine: 3 } + ]) + }) + + it('renders a newly created file', () => { + expect(toolDiffLines({ before: '', after: 'print("hello")', lang: 'python' })).toEqual([ + { + kind: 'added', + content: 'print("hello")', + oldLine: undefined, + newLine: 1, + changedRanges: [{ start: 0, length: 14 }] + } + ]) + }) + + it('renders a deleted file', () => { + expect(toolDiffLines({ before: 'print("goodbye")', after: '', lang: 'python' })).toEqual([ + { + kind: 'removed', + content: 'print("goodbye")', + oldLine: 1, + newLine: undefined, + changedRanges: [{ start: 0, length: 16 }] + } + ]) + }) + + it('uses Monaco line alignment for reordered lines', () => { + expect( + toolDiffLines({ + before: 'const a = 1\nconst b = 2', + after: 'const b = 2\nconst a = 1', + lang: 'typescript' + }) + ).toEqual([ + { + kind: 'added', + content: 'const b = 2', + newLine: 1, + changedRanges: [{ start: 0, length: 11, extendsToEnd: true }] + }, + { kind: 'context', content: 'const a = 1', oldLine: 1, newLine: 2 }, + { + kind: 'removed', + content: 'const b = 2', + oldLine: 2, + changedRanges: [{ start: 0, length: 11 }] + } + ]) + }) + + it('avoids expensive alignment while arguments stream', () => { + const before = Array.from({ length: 1_000 }, (_, index) => `before ${index}`).join('\n') + const after = Array.from({ length: 1_000 }, (_, index) => `after ${index}`).join('\n') + + expect(toolDiffLines({ before, after, lang: 'plaintext' }, true)).toHaveLength(400) + expect(diffLineCounts({ before, after, lang: 'plaintext' }, true)).toEqual({ + added: 1_000, + removed: 1_000 + }) + }) + + it('diffs a small edit in a large file', () => { + const beforeLines = Array.from({ length: 600 }, (_, index) => `line ${index}`) + const afterLines = beforeLines.map((line, index) => (index === 400 ? 'changed' : line)) + const diff = { before: beforeLines.join('\n'), after: afterLines.join('\n'), lang: 'plaintext' } + const lines = toolDiffLines(diff) + + expect(lines.filter((line) => line.kind !== 'context')).toMatchObject([ + { kind: 'removed', content: 'line 400', oldLine: 401 }, + { kind: 'added', content: 'changed', newLine: 401 } + ]) + expect(lines[399]).toEqual({ kind: 'context', content: 'line 399', oldLine: 400, newLine: 400 }) + expect(diffLineCounts(diff)).toEqual({ added: 1, removed: 1 }) + }) + + it('reports a large rewrite as a whole-file change', () => { + const before = Array.from({ length: 1_000 }, (_, index) => `before ${index}`).join('\n') + const after = Array.from({ length: 1_000 }, (_, index) => `after ${index}`).join('\n') + + expect(diffLineCounts({ before, after, lang: 'plaintext' })).toEqual({ + added: 1_000, + removed: 1_000 + }) + }) + + it('keeps an empty streamed replacement paired with its own edit', () => { + const parameters = + '{"diffs":[{"old_string":"a","new_string":""},{"old_string":"b","new_string":"c"}]}' + + expect(toolCodeDiff(message({ toolName: 'edit_code', parameters }))).toEqual({ + before: 'a\nb', + after: '\nc', + lang: 'plaintext' + }) + }) + + it('recognizes calls whose arguments are a whole file', () => { + const call = ( + toolName: string, + parameters: unknown, + codeDiff?: ToolDisplayMessage['codeDiff'] + ) => argumentsCarryWholeFile(message({ toolName, parameters, codeDiff })) + + expect(call('write_script', { path: 'f/a', content: '' })).toBe(true) + expect(call('edit_code', '{"code":"x')).toBe(true) + expect( + call('edit_code', { code: 'x' }, { before: 'old', after: 'x', lang: 'typescript' }) + ).toBe(true) + expect(call('edit_code', { diffs: [{ old_string: 'a', new_string: 'b' }] })).toBe(false) + expect(call('edit_script', { old_string: 'a', new_string: 'b' })).toBe(false) + }) + + it('shows each row of a small edit once', () => { + const lines = toolDiffLines({ before: 'a\nb\nc\n', after: 'a\nB\nc\n', lang: 'plaintext' }) + + expect(visibleToolDiffRows(lines, new Map())).toEqual(lines) + expect(lines.map((line) => line.kind)).toEqual(['context', 'removed', 'added', 'context']) + }) + + it('bounds the rows of a large rewrite', () => { + const before = Array.from({ length: 1_000 }, (_, index) => `before ${index}`).join('\n') + const after = Array.from({ length: 1_000 }, (_, index) => `after ${index}`).join('\n') + const lines = toolDiffLines({ before, after, lang: 'plaintext' }) + const rows = visibleToolDiffRows(lines, new Map()) + + expect(rows).toHaveLength(802) + expect(rows[400]).toEqual({ kind: 'omitted', key: 'removed:0', count: 600 }) + expect(rows[401]).toMatchObject({ kind: 'added', newLine: 1 }) + expect(rows[801]).toEqual({ kind: 'omitted', key: 'added:1000', count: 600 }) + + // One click on a run reveals 400 more rows; the overall cap still holds until `end` is clicked. + const expanded = visibleToolDiffRows(lines, new Map([['removed:0', 1]])) + expect(expanded).toHaveLength(1_001) + expect(expanded[800]).toEqual({ kind: 'omitted', key: 'removed:0', count: 200 }) + expect(expanded[1_000]).toEqual({ kind: 'omitted', key: 'end', count: 801 }) + expect( + visibleToolDiffRows( + lines, + new Map([ + ['removed:0', 1], + ['end', 1] + ]) + ) + ).toHaveLength(1_202) + }) + + it('highlights each line within the scope of the whole source', () => { + const typescript = toolCodeDiffLanguage('typescript') + hljs.registerLanguage(typescript.name, typescript.register) + const code = 'const s = `a\nb`\nconst n = 1\n' + + const lines = highlightedSourceLines(code, typescript.name) + expect(lines).toHaveLength(3) + expect(lines[0]).toBe( + 'const s = `a' + ) + expect(lines[1]).toMatch(/^b`<\/span>/) + expect(highlightedSourceLines(code, typescript.name, 2)).toEqual(lines.slice(0, 2)) + expect(highlightedSourceLines('a\nb', typescript.name)).toEqual(['a', 'b']) + expect(highlightedSourceLines('a\nb\n', typescript.name)).toEqual(['a', 'b']) + expect(highlightedSourceLines('', typescript.name)).toEqual([]) + + const large = Array.from({ length: 2_001 }, () => 'const a = 1 < 2').join('\n') + '\n' + const plain = highlightedSourceLines(large, typescript.name) + expect(plain).toHaveLength(2_001) + expect(plain[0]).toBe('const a = 1 < 2') + + const minified = highlightedSourceLines('x'.repeat(100_001), typescript.name) + expect(minified).toEqual(['x'.repeat(10_000) + '…']) + }) + + it('has a highlighter for every supported editor language', () => { + expect(TOOL_CODE_DIFF_LANGUAGES).toEqual( + expect.arrayContaining([ + 'typescript', + 'javascript', + 'python', + 'json', + 'yaml', + 'sql', + 'shell', + 'powershell', + 'php', + 'rust', + 'graphql', + 'csharp', + 'nu', + 'java', + 'r', + 'go', + 'ruby', + 'plaintext' + ]) + ) + for (const language of TOOL_CODE_DIFF_LANGUAGES) { + const highlighter = toolCodeDiffLanguage(language) + hljs.registerLanguage(highlighter.name, highlighter.register) + expect(() => + hljs.highlight('const changed = true', { language: highlighter.name }) + ).not.toThrow() + } + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/toolCodeDiff.ts b/frontend/src/lib/components/copilot/chat/toolCodeDiff.ts new file mode 100644 index 0000000000..2e5d8bf13c --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCodeDiff.ts @@ -0,0 +1,381 @@ +// This is Monaco's advanced diff engine; it supplies the same multi-line inner ranges. +import { DefaultLinesDiffComputer } from '@codingame/monaco-vscode-api/vscode/vs/editor/common/diff/defaultLinesDiffComputer/defaultLinesDiffComputer' +import type { ToolCodeDiff, ToolDisplayMessage } from './shared' + +// Builds a diff from a call's own arguments, for the moments the saved `codeDiff` does not +// exist: while the arguments stream, when the call failed, and for transcripts saved before +// calls recorded one. Argument diffs have no language or surrounding lines. +const ARGS_DIFF_BY_TOOL: Record ToolCodeDiff | undefined> = { + edit_script: (params) => argumentDiff(streamingEditArguments(params)), + write_script: (params) => fullContentArgumentDiff(params, 'content'), + edit_code: (params) => { + if ( + params && + typeof params === 'object' && + Array.isArray((params as { diffs?: unknown }).diffs) + ) { + const diffs = (params as { diffs: unknown[] }).diffs + return argumentDiffs(diffs.map(streamingEditArguments)) + } + if (typeof params === 'string') { + return ( + argumentDiffs(streamingEditArgumentsArray(params)) ?? + fullContentArgumentDiff(params, 'code') + ) + } + return argumentDiff(streamingEditArguments(params)) ?? fullContentArgumentDiff(params, 'code') + } +} + +function fullContentArgumentDiff(params: unknown, key: string): ToolCodeDiff | undefined { + const content = + params && typeof params === 'object' + ? (params as Record)[key] + : typeof params === 'string' + ? partialJsonString(params, key) + : undefined + return typeof content === 'string' ? { before: '', after: content, lang: 'plaintext' } : undefined +} + +function argumentDiff( + edit: { old_string?: string; new_string?: string } | undefined +): ToolCodeDiff | undefined { + return typeof edit?.old_string === 'string' || typeof edit?.new_string === 'string' + ? { + before: typeof edit.old_string === 'string' ? edit.old_string : '', + after: typeof edit.new_string === 'string' ? edit.new_string : '', + lang: 'plaintext' + } + : undefined +} + +function argumentDiffs( + edits: ({ old_string?: string; new_string?: string } | undefined)[] +): ToolCodeDiff | undefined { + const diffs = edits.map(argumentDiff).filter((diff): diff is ToolCodeDiff => diff !== undefined) + if (diffs.length === 0) return undefined + return { + before: diffs.map((diff) => diff.before).join('\n'), + after: diffs.map((diff) => diff.after).join('\n'), + lang: 'plaintext' + } +} + +function streamingEditArguments( + params: unknown +): { old_string?: string; new_string?: string } | undefined { + if (params && typeof params === 'object') + return params as { old_string?: string; new_string?: string } + if (typeof params !== 'string') return undefined + + return { + old_string: partialJsonString(params, 'old_string'), + new_string: partialJsonString(params, 'new_string') + } +} + +function streamingEditArgumentsArray( + partialJson: string +): { old_string?: string; new_string?: string }[] { + const oldStrings = partialJsonStrings(partialJson, 'old_string') + const newStrings = partialJsonStrings(partialJson, 'new_string') + return Array.from({ length: Math.max(oldStrings.length, newStrings.length) }, (_, index) => ({ + old_string: oldStrings[index], + new_string: newStrings[index] + })) +} + +function partialJsonString(partialJson: string, key: string): string | undefined { + return partialJsonStrings(partialJson, key)[0] +} + +function partialJsonStrings(partialJson: string, key: string): string[] { + const matches = partialJson.matchAll(new RegExp(`"${key}"\\s*:\\s*"((?:[^"\\\\]|\\\\.)*)`, 'g')) + const result: string[] = [] + // An empty string is a real value (a deletion): skipping it would pair the next edit's + // `new_string` with this edit's `old_string`. + for (const match of matches) { + try { + result.push(JSON.parse(`"${match[1]}"`)) + } catch { + continue + } + } + return result +} + +export function hasToolCodeDiff(toolName: string | undefined): boolean { + return toolName !== undefined && Object.hasOwn(ARGS_DIFF_BY_TOOL, toolName) +} + +export function toolCodeDiff(message: ToolDisplayMessage): ToolCodeDiff | undefined { + if (message.codeDiff) return message.codeDiff + if (!message.toolName) return undefined + return Object.hasOwn(ARGS_DIFF_BY_TOOL, message.toolName) + ? ARGS_DIFF_BY_TOOL[message.toolName](message.parameters) + : undefined +} + +// Decided by tool and argument shape, never by the saved `codeDiff`: an overwrite's saved diff +// has a before side, and clearing a script leaves both argument sides empty, yet both calls sent +// a whole file. +export function argumentsCarryWholeFile(message: ToolDisplayMessage): boolean { + if (message.toolName === 'write_script') return true + if (message.toolName !== 'edit_code') return false + const params = message.parameters + if (typeof params === 'string') return /"code"\s*:/.test(params) + return Boolean(params && typeof params === 'object' && 'code' in params) +} + +export function diffLineCounts( + diff: ToolCodeDiff, + streaming = false +): { added: number; removed: number } { + if (streaming) { + return { added: lines(diff.after).length, removed: lines(diff.before).length } + } + + return toolDiffLineCounts(toolDiffLines(diff)) +} + +export function toolDiffLineCounts(lines: ToolDiffLine[]): { added: number; removed: number } { + let added = 0 + let removed = 0 + for (const line of lines) { + if (line.kind === 'added') added++ + else if (line.kind === 'removed') removed++ + } + return { added, removed } +} + +export type ToolDiffLine = { + kind: 'context' | 'added' | 'removed' + content: string + oldLine?: number + newLine?: number + changedRanges?: CharacterRange[] +} + +export type CharacterRange = { start: number; length: number; extendsToEnd?: boolean } + +const STREAMING_PREVIEW_LINES = 200 + +function lines(value: string): string[] { + const result = value.split('\n') + if (result.at(-1) === '') result.pop() + return result +} + +function hasFinalNewline(value: string): boolean { + return value.endsWith('\n') +} + +export function toolDiffLines(diff: ToolCodeDiff, streaming = false): ToolDiffLine[] { + const before = lines(diff.before) + const after = lines(diff.after) + if (streaming) return streamingDiffLines(before, after) + + const result: ToolDiffLine[] = [] + const monacoBefore = monacoLines(diff.before) + const monacoAfter = monacoLines(diff.after) + // Runs on the UI thread. Cost follows the amount changed, not file size: only a large + // rewrite reaches the cap, and Monaco then reports the whole file as one change. + const changes = new DefaultLinesDiffComputer().computeDiff(monacoBefore, monacoAfter, { + ignoreTrimWhitespace: false, + maxComputationTimeMs: 200, + computeMoves: false, + extendToSubwords: false + }).changes + const removedRanges = new Map() + const addedRanges = new Map() + for (const change of changes) { + for (const innerChange of change.innerChanges ?? []) { + addCharacterRanges(removedRanges, monacoBefore, innerChange.originalRange) + addCharacterRanges(addedRanges, monacoAfter, innerChange.modifiedRange) + } + } + + let oldIndex = 0 + let newIndex = 0 + for (const change of changes) { + const oldStart = change.original.startLineNumber - 1 + const newStart = change.modified.startLineNumber - 1 + appendContextLines( + result, + before, + after, + oldIndex, + newIndex, + Math.min(oldStart - oldIndex, newStart - newIndex) + ) + appendChangedLines( + result, + 'removed', + before, + oldStart, + change.original.endLineNumberExclusive - 1, + removedRanges + ) + appendChangedLines( + result, + 'added', + after, + newStart, + change.modified.endLineNumberExclusive - 1, + addedRanges + ) + oldIndex = Math.min(change.original.endLineNumberExclusive - 1, before.length) + newIndex = Math.min(change.modified.endLineNumberExclusive - 1, after.length) + } + const trailingContext = Math.max(0, Math.min(before.length - oldIndex, after.length - newIndex)) + appendContextLines(result, before, after, oldIndex, newIndex, trailingContext) + oldIndex += trailingContext + newIndex += trailingContext + appendChangedLines(result, 'removed', before, oldIndex, before.length, removedRanges) + appendChangedLines(result, 'added', after, newIndex, after.length, addedRanges) + if ( + diff.before !== '' && + diff.after !== '' && + hasFinalNewline(diff.before) !== hasFinalNewline(diff.after) + ) { + result.push( + hasFinalNewline(diff.before) + ? { kind: 'removed', content: '', oldLine: before.length + 1 } + : { kind: 'added', content: '', newLine: after.length + 1 } + ) + } + + return result +} + +export type VisibleToolDiffRow = + | ToolDiffLine + | { kind: 'collapsed'; key: string; count: number } + | { kind: 'omitted'; key: string; count: number } + +const CONTEXT_LINES = 3 +// Every visible row is a DOM row with no virtualization, and only context collapses: a whole-file +// rewrite would otherwise render every line of both sides. Two capped runs fit under the overall +// cap, so a rewrite still shows its added side. Each expansion of an omitted row reveals one more +// cap's worth, so no single click renders an unbounded number of rows. +const MAX_CHANGED_RUN_ROWS = 400 +const MAX_VISIBLE_ROWS = 1_000 + +// `expansions` counts the clicks on each collapsed or omitted row, by its key. +export function visibleToolDiffRows( + lines: ToolDiffLine[], + expansions: ReadonlyMap +): VisibleToolDiffRow[] { + const result: VisibleToolDiffRow[] = [] + for (let index = 0; index < lines.length; ) { + const start = index + const kind = lines[start].kind + while (index < lines.length && lines[index].kind === kind) index++ + const count = index - start + + if (kind !== 'context') { + const key = `${kind}:${start}` + const shown = MAX_CHANGED_RUN_ROWS * (1 + (expansions.get(key) ?? 0)) + result.push(...lines.slice(start, Math.min(index, start + shown))) + if (count > shown) result.push({ kind: 'omitted', key, count: count - shown }) + continue + } + + const key = `${lines[start].oldLine}:${lines[start].newLine}:${count}` + if (count <= CONTEXT_LINES * 2 + 1 || expansions.has(key)) { + result.push(...lines.slice(start, index)) + continue + } + result.push(...lines.slice(start, start + CONTEXT_LINES)) + result.push({ kind: 'collapsed', key, count: count - CONTEXT_LINES * 2 }) + result.push(...lines.slice(index - CONTEXT_LINES, index)) + } + + const maxRows = MAX_VISIBLE_ROWS * (1 + (expansions.get('end') ?? 0)) + if (result.length <= maxRows) return result + const hidden = result + .slice(maxRows) + .reduce((total, row) => total + ('count' in row ? row.count : 1), 0) + return [...result.slice(0, maxRows), { kind: 'omitted', key: 'end', count: hidden }] +} + +function streamingDiffLines(before: string[], after: string[]): ToolDiffLine[] { + return [ + ...before + .slice(0, STREAMING_PREVIEW_LINES) + .map((content, index) => ({ kind: 'removed' as const, content, oldLine: index + 1 })), + ...after + .slice(0, STREAMING_PREVIEW_LINES) + .map((content, index) => ({ kind: 'added' as const, content, newLine: index + 1 })) + ] +} + +function monacoLines(value: string): string[] { + const result = lines(value) + return result.length ? result : [''] +} + +function appendContextLines( + result: ToolDiffLine[], + before: string[], + after: string[], + oldStart: number, + newStart: number, + count: number +): void { + for (let index = 0; index < count; index++) { + result.push({ + kind: 'context', + content: before[oldStart + index], + oldLine: oldStart + index + 1, + newLine: newStart + index + 1 + }) + } +} + +function appendChangedLines( + result: ToolDiffLine[], + kind: 'added' | 'removed', + source: string[], + start: number, + end: number, + ranges: Map +): void { + for (let index = start; index < Math.min(end, source.length); index++) { + const changedRanges = ranges.get(index + 1) + result.push({ + kind, + content: source[index], + ...(kind === 'added' ? { newLine: index + 1 } : { oldLine: index + 1 }), + ...(changedRanges?.length ? { changedRanges } : {}) + }) + } +} + +function addCharacterRanges( + ranges: Map, + lines: string[], + range: { + startLineNumber: number + startColumn: number + endLineNumber: number + endColumn: number + } +): void { + for ( + let lineNumber = range.startLineNumber; + lineNumber <= Math.min(range.endLineNumber, lines.length); + lineNumber++ + ) { + const content = lines[lineNumber - 1] + const start = lineNumber === range.startLineNumber ? range.startColumn - 1 : 0 + const extendsToEnd = lineNumber < range.endLineNumber + const end = extendsToEnd ? content.length : range.endColumn - 1 + if (end > start) { + ranges.set(lineNumber, [ + ...(ranges.get(lineNumber) ?? []), + { start, length: end - start, ...(extendsToEnd ? { extendsToEnd } : {}) } + ]) + } + } +} diff --git a/frontend/src/lib/components/copilot/chat/toolCodeDiffHighlight.ts b/frontend/src/lib/components/copilot/chat/toolCodeDiffHighlight.ts new file mode 100644 index 0000000000..b63d002e9e --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCodeDiffHighlight.ts @@ -0,0 +1,57 @@ +import hljs from 'highlight.js/lib/core' +import { escapeHtml } from '$lib/utils' + +// highlight.js runs on the UI thread at roughly 15 ms per 1,000 lines (50 KB), and must see the +// whole source for multi-line strings and comments to scope correctly. Larger sides, including +// minified one-line sources, render as plain text with each line cut at MAX_PLAIN_LINE_CHARS. +const MAX_HIGHLIGHTED_LINES = 2_000 +const MAX_HIGHLIGHTED_CHARS = 100_000 +const MAX_PLAIN_LINE_CHARS = 10_000 + +// Highlights the whole source before splitting it, so a line inside a multi-line string or +// comment keeps that scope. Only the first `lineLimit` lines are highlighted: a streamed source +// grows on every delta while the preview shows a bounded prefix. +export function highlightedSourceLines( + code: string, + languageName: string, + lineLimit = Infinity +): string[] { + if (code === '' || lineLimit <= 0) return [] + const sourceLines = code.split('\n') + const truncated = sourceLines.length > lineLimit + const shownLines = truncated ? sourceLines.slice(0, lineLimit) : sourceLines + const source = truncated ? shownLines.join('\n') : code + if (shownLines.length > MAX_HIGHLIGHTED_LINES || source.length > MAX_HIGHLIGHTED_CHARS) { + const plain = shownLines.map((line) => + escapeHtml( + line.length > MAX_PLAIN_LINE_CHARS ? line.slice(0, MAX_PLAIN_LINE_CHARS) + '…' : line + ) + ) + if (!truncated && code.endsWith('\n')) plain.pop() + return plain + } + const highlighted = hljs.highlight(source, { language: languageName }).value + return splitHighlightedLines(highlighted, !truncated && code.endsWith('\n')) +} + +// Returns one entry per source line, closing the spans still open at each line end and +// reopening them on the next line so every entry is balanced HTML. +export function splitHighlightedLines(highlighted: string, hasFinalNewline: boolean): string[] { + const result: string[] = [] + const openTags: string[] = [] + let line = '' + + for (const token of highlighted.split(/(<[^>]+>|\n)/)) { + if (token === '\n') { + result.push(line + ''.repeat(openTags.length)) + line = openTags.join('') + } else { + line += token + if (token.startsWith('') openTags.pop() + } + } + + if (!hasFinalNewline) result.push(line) + return result +} diff --git a/frontend/src/lib/components/copilot/chat/toolCodeDiffLanguage.ts b/frontend/src/lib/components/copilot/chat/toolCodeDiffLanguage.ts new file mode 100644 index 0000000000..ddf2464136 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/toolCodeDiffLanguage.ts @@ -0,0 +1,43 @@ +import { csharp, go, java, json, plaintext, type LanguageType } from 'svelte-highlight/languages' +import graphql from 'svelte-highlight/languages/graphql' +import javascript from 'svelte-highlight/languages/javascript' +import php from 'svelte-highlight/languages/php' +import python from 'svelte-highlight/languages/python' +import powershell from 'svelte-highlight/languages/powershell' +import r from 'svelte-highlight/languages/r' +import ruby from 'svelte-highlight/languages/ruby' +import rust from 'svelte-highlight/languages/rust' +import shell from 'svelte-highlight/languages/shell' +import sql from 'svelte-highlight/languages/sql' +import typescript from 'svelte-highlight/languages/typescript' +import yaml from 'svelte-highlight/languages/yaml' + +// Keyed by the editor language `scriptLangToEditorLang` returns, not the script language. +// for related places search: ADD_NEW_LANG +const LANGUAGE_BY_MONACO: Record> = { + typescript, + javascript, + python, + json, + yaml, + sql, + shell, + powershell, + php, + rust, + graphql, + csharp, + nu: python, + java, + r, + go, + ruby, + text: plaintext, + plaintext +} + +export const TOOL_CODE_DIFF_LANGUAGES = Object.keys(LANGUAGE_BY_MONACO) + +export function toolCodeDiffLanguage(language: string): LanguageType { + return LANGUAGE_BY_MONACO[language] ?? plaintext +} diff --git a/frontend/src/routes/test_dev/diff-comparison/+page.svelte b/frontend/src/routes/test_dev/diff-comparison/+page.svelte new file mode 100644 index 0000000000..c473dbcb64 --- /dev/null +++ b/frontend/src/routes/test_dev/diff-comparison/+page.svelte @@ -0,0 +1,139 @@ + + + + Diff comparison + + +
+
+ {#each cases as item} + + {/each} +
+ +
+
+

Monaco

+
+ {#key activeCase.id} + + {/key} +
+
+ +
+

Native

+
+ {#key activeCase.id} + + {/key} +
+
+
+
From 7593597617110fbd276f3f32ca2be7064547a080 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 23 Sep 2026 15:50:32 +0200 Subject: [PATCH 30/34] fix: validate app and trigger paths, refuse traversal in workspace export (#11311) * fix: enforce proper_id paths on apps and triggers, refuse traversal in export Co-Authored-By: Claude Opus 5.5 (1M context) * fix: skip proper_id on tables already holding non-conforming paths Co-Authored-By: Claude Opus 5.5 (1M context) * test: pin archive entry path traversal guard Co-Authored-By: Claude Opus 5.5 (1M context) * fix: refuse windows-normalized traversal in export, check raw app path early Co-Authored-By: Claude Opus 5.5 (1M context) * fix: only treat a colon in the first archive segment as a drive prefix Co-Authored-By: Claude Opus 5.5 (1M context) * test: accept a colon past the first archive segment Co-Authored-By: Claude Opus 5.5 (1M context) * revert: drop proper_id migration, keep path validation in the API Co-Authored-By: Claude Opus 5.5 (1M context) * fix: validate paths in bulk http trigger creation Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-api/src/apps.rs | 7 ++- backend/windmill-api/src/workspaces_export.rs | 60 ++++++++++++++++++- backend/windmill-common/src/utils.rs | 3 + backend/windmill-trigger-http/src/handler.rs | 2 + backend/windmill-trigger/src/handler.rs | 6 +- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index edd2040143..54886e2568 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -69,7 +69,7 @@ use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, paginate_optional, + check_proper_path, http_get_from_hub, not_found_if_none, paginate, paginate_optional, query_elems_from_hub, require_admin, strip_json_nul, Pagination, RunnableKind, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, @@ -2506,6 +2506,7 @@ async fn create_app_internal<'a>( // inside process_app_multipart!, so checking after this call would leave a // denied app committed in the DB. check_scopes(&authed, || format!("apps:write:{}", &app.path))?; + check_proper_path(&app.path)?; validate_frontend_sdk_scopes(&app.policy)?; if raw_app { validate_raw_app_path_keys(&app.value.0)?; @@ -3252,6 +3253,7 @@ async fn create_app_raw_source( // Before the compile, which costs a job on a worker: it must not run for a // path already taken, nor for one the caller can't write. `create_app_internal` // rejects both, but only after the sources have been built. + check_proper_path(&path)?; if app_exists(&db, &w_id, &path).await? { return Err(Error::BadRequest(format!("App {path} already exists"))); } @@ -3462,6 +3464,9 @@ async fn update_app_internal<'a>( // the token's write scope, not just the source path. if let Some(npath) = ns.path.as_deref() { check_scopes(&authed, || format!("apps:write:{}", npath))?; + if npath != path { + check_proper_path(npath)?; + } } if raw_app { diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index e29113ecd2..ea79aaa572 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -281,8 +281,29 @@ enum ArchiveImpl { Tar(tokio_tar::Builder), } +/// Entry paths come from item paths stored in the workspace; an absolute or +/// drive-prefixed path, or a parent segment, would make extraction write outside +/// the target directory. Win32 strips trailing dots and spaces from a segment, +/// so `.. ` resolves to `..` there: any segment made only of those is refused. +/// A colon matters only in the first segment (a drive prefix); later segments, +/// such as a data table name, may carry one. +fn check_archive_entry_path(path: &str) -> Result<()> { + let is_dot_segment = |seg: &str| !seg.is_empty() && seg.chars().all(|c| c == '.' || c == ' '); + let first = path.split(['/', '\\']).next().unwrap_or_default(); + if path.starts_with(['/', '\\']) + || first.contains(':') + || path.split(['/', '\\']).any(is_dot_segment) + { + return Err(Error::internal_err(format!( + "refusing to write archive entry with path traversal: {path}" + ))); + } + Ok(()) +} + impl ArchiveImpl { async fn write_to_archive(&mut self, content: &str, path: &str) -> Result<()> { + check_archive_entry_path(path)?; match self { ArchiveImpl::Tar(t) => { let bytes = content.as_bytes(); @@ -1647,7 +1668,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), - datatable: windmill_common::workspaces::strip_datatable_permissions(row.datatable.clone()), + datatable: windmill_common::workspaces::strip_datatable_permissions( + row.datatable.clone(), + ), slack_team_id: row.slack_team_id.clone(), slack_name: row.slack_name.clone(), slack_command_script: row.slack_command_script.clone(), @@ -1792,6 +1815,41 @@ pub(crate) async fn tarball_workspace( Ok((headers, body)) } +#[cfg(test)] +mod archive_entry_path_tests { + use super::check_archive_entry_path; + + #[test] + fn rejects_traversal_and_absolute_paths() { + for ok in [ + "u/admin/app.app.json", + "f/x/a..b.script.json", + "settings.yaml", + "migrations/datatable/foo:bar/20260617120000_name.up.sql", + ] { + assert!( + check_archive_entry_path(ok).is_ok(), + "{ok} should be accepted" + ); + } + for bad in [ + "f/x/../../evil.app.json", + "../evil", + "/etc/evil", + "f\\..\\evil", + "f/x/.. /.. /evil.app.json", + "f/x/.../evil", + "C:/evil", + "\\evil", + ] { + assert!( + check_archive_entry_path(bad).is_err(), + "{bad} should be rejected" + ); + } + } +} + #[cfg(test)] mod fork_export_tests { use super::*; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 29fc9e99f3..b247c66532 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -245,6 +245,9 @@ lazy_static::lazy_static! { /// Reject a path the `proper_id` constraint would reject anyway, so the caller /// gets a plain 400 instead of the raw Postgres constraint-violation string, /// which names the table and constraint and echoes the input back. +/// +/// `app` and the trigger tables have no such constraint: for them this check is +/// the only authority, so loosening it lets malformed paths be stored. pub fn check_proper_path(path: &str) -> Result<()> { // The column is varchar(255); without this an over-long but well-formed path // still reaches Postgres and leaks the same kind of message back. diff --git a/backend/windmill-trigger-http/src/handler.rs b/backend/windmill-trigger-http/src/handler.rs index 01ddfd4a81..539c8e43d2 100644 --- a/backend/windmill-trigger-http/src/handler.rs +++ b/backend/windmill-trigger-http/src/handler.rs @@ -13,6 +13,7 @@ use windmill_common::global_settings::{validate_allowed_origins, HTTP_ROUTE_WORK use windmill_common::{ db::UserDB, error::{Error, Result}, + utils::check_proper_path, worker::CLOUD_HOSTED, DB, }; @@ -269,6 +270,7 @@ pub async fn create_many_http_triggers( check_scopes(&authed, || { format!("http_triggers:write:{}", &new_http_trigger.base.path) })?; + check_proper_path(&new_http_trigger.base.path)?; // This route inserts directly, bypassing the shared create handler. // `error_wrapper` would turn the rejection into a 500. diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 66d6770354..ba40ed2c48 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -22,7 +22,7 @@ use windmill_common::{ fetch_draft_only_list_rows, overlay_or_draft_only, UserDraftItemKind, WithDraftOverlay, WithDraftQuery, }, - utils::{paginate, Pagination, StripPath}, + utils::{check_proper_path, paginate, Pagination, StripPath}, worker::CLOUD_HOSTED, DB, }; @@ -547,6 +547,7 @@ async fn create_trigger( &new_trigger.base.path ) })?; + check_proper_path(&new_trigger.base.path)?; if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { return Err(Error::BadRequest(format!( @@ -817,6 +818,9 @@ async fn update_trigger( &edit_trigger.base.path ) })?; + if edit_trigger.base.path != path { + check_proper_path(&edit_trigger.base.path)?; + } edit_trigger.error_handling.validate()?; From ceb17223dcb5f66532017036916e91a5323279fb Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 23 Sep 2026 16:36:23 +0200 Subject: [PATCH 31/34] fix: bill from a single service account telemetry count (#11301) * [ee] fix: bill from a single service account telemetry count Co-Authored-By: Claude Opus 5 * [ee] fix: count service accounts held in forks Co-Authored-By: Claude Opus 5 * chore: update ee-repo-ref to ed5a367b4def5280c3a5e1090e7abfa099a01042 This commit updates the EE repository reference after PR #824 was merged in windmill-ee-private. Previous ee-repo-ref: bc1fe61364c2a731ec26186eb04596254886a74c New ee-repo-ref: ed5a367b4def5280c3a5e1090e7abfa099a01042 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 Co-authored-by: windmill-internal-app[bot] --- ...5ab0199084b35664cbb0a586917bfa0da075.json} | 4 ++-- ...6aed1162fe85cb31c0414bc7e9b93a0684b43.json | 20 ------------------- backend/ee-repo-ref.txt | 2 +- 3 files changed, 3 insertions(+), 23 deletions(-) rename backend/.sqlx/{query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json => query-3190736818379ad22628aae3c5a25ab0199084b35664cbb0a586917bfa0da075.json} (58%) delete mode 100644 backend/.sqlx/query-e60b837837f9ca9ec9587c6f7b66aed1162fe85cb31c0414bc7e9b93a0684b43.json diff --git a/backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json b/backend/.sqlx/query-3190736818379ad22628aae3c5a25ab0199084b35664cbb0a586917bfa0da075.json similarity index 58% rename from backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json rename to backend/.sqlx/query-3190736818379ad22628aae3c5a25ab0199084b35664cbb0a586917bfa0da075.json index 5661c59faf..90566827e7 100644 --- a/backend/.sqlx/query-a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf.json +++ b/backend/.sqlx/query-3190736818379ad22628aae3c5a25ab0199084b35664cbb0a586917bfa0da075.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM usr WHERE is_service_account = true", + "query": "SELECT COUNT(DISTINCT email) FROM usr WHERE is_service_account AND NOT disabled", "describe": { "columns": [ { @@ -16,5 +16,5 @@ null ] }, - "hash": "a37c2c4d5656d4b44433de84c454046f7586e36b7bd6a4679d70c359d4aacfcf" + "hash": "3190736818379ad22628aae3c5a25ab0199084b35664cbb0a586917bfa0da075" } diff --git a/backend/.sqlx/query-e60b837837f9ca9ec9587c6f7b66aed1162fe85cb31c0414bc7e9b93a0684b43.json b/backend/.sqlx/query-e60b837837f9ca9ec9587c6f7b66aed1162fe85cb31c0414bc7e9b93a0684b43.json deleted file mode 100644 index accf6b0a49..0000000000 --- a/backend/.sqlx/query-e60b837837f9ca9ec9587c6f7b66aed1162fe85cb31c0414bc7e9b93a0684b43.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(DISTINCT email) FROM usr\n WHERE is_service_account AND NOT disabled AND workspace_id NOT LIKE 'wm-fork%'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "e60b837837f9ca9ec9587c6f7b66aed1162fe85cb31c0414bc7e9b93a0684b43" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 80c21f89c1..211848b65a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -0914cece0eb508150898022a0d78da9caa0a1c6d +ed5a367b4def5280c3a5e1090e7abfa099a01042 From 5b03c1732e905673480d0960b95577d47ec919b3 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:36:47 +0200 Subject: [PATCH 32/34] fix: build ag grid link cells as DOM nodes with a vetted href (#11316) Co-authored-by: Claude Opus 5.5 (1M context) --- .../display/table/utils.dom.test.ts | 29 +++++++++++++++++++ .../apps/components/display/table/utils.ts | 23 +++++++++++---- .../copilot/chat/LinkRenderer.svelte | 2 +- .../copilot/chat => utils}/safeHref.test.ts | 0 .../copilot/chat => utils}/safeHref.ts | 6 ++-- 5 files changed, 49 insertions(+), 11 deletions(-) create mode 100644 frontend/src/lib/components/apps/components/display/table/utils.dom.test.ts rename frontend/src/lib/{components/copilot/chat => utils}/safeHref.test.ts (100%) rename frontend/src/lib/{components/copilot/chat => utils}/safeHref.ts (57%) diff --git a/frontend/src/lib/components/apps/components/display/table/utils.dom.test.ts b/frontend/src/lib/components/apps/components/display/table/utils.dom.test.ts new file mode 100644 index 0000000000..0e039139b0 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/table/utils.dom.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import type { ICellRendererParams } from 'ag-grid-community' +import { defaultCellRenderer } from './utils' + +function render(value: unknown) { + const renderer = defaultCellRenderer('link')! + return renderer({ value } as ICellRendererParams) as HTMLElement +} + +// AG Grid writes a renderer's string output to innerHTML, and link cells hold run results. +describe('link cell renderer', () => { + it('renders a link without parsing its label or href as markup', () => { + const el = render({ href: 'https://x.test/">', label: 'hi' }) + expect(el.tagName).toBe('A') + expect(el.textContent).toBe('hi') + expect(el.getAttribute('href')).toBe('https://x.test/">') + expect(el.children).toHaveLength(0) + }) + + it.each([ + { href: 'javascript:alert(1)', label: 'click' }, + 'javascript:alert(1)', + { href: 'data:text/html,', label: 'click' } + ])('renders a non-navigating href %j as plain text', (value) => { + const el = render(value) + expect(el.tagName).toBe('SPAN') + expect(el.hasAttribute('href')).toBe(false) + }) +}) diff --git a/frontend/src/lib/components/apps/components/display/table/utils.ts b/frontend/src/lib/components/apps/components/display/table/utils.ts index 015cdc54db..8b85a5894c 100644 --- a/frontend/src/lib/components/apps/components/display/table/utils.ts +++ b/frontend/src/lib/components/apps/components/display/table/utils.ts @@ -8,6 +8,7 @@ import type { TableAction } from '$lib/components/apps/editor/component' import { mount, unmount } from 'svelte' import { Button } from '$lib/components/common' import { Trash2 } from 'lucide-svelte' +import { safeHref } from '$lib/utils/safeHref' export type WindmillColumnDef = ColDef & { _isActionsColumn?: boolean @@ -110,15 +111,25 @@ export function isLinkObject(value: any): value is LinkObject { export function defaultCellRenderer(cellRendererType: string) { if (cellRendererType === 'link') { + // AG Grid assigns a string returned by a cell renderer to innerHTML, and cell values are + // run results, so the link is built as a DOM node that never parses them as markup. return (params: ICellRendererParams) => { - if (isLinkObject(params.value)) { - const value = params.value - return `${value.label}` - } else if (params.value) { - return `${params.value}` - } else { + if (!params.value) { return params.value } + const { href, label } = isLinkObject(params.value) + ? params.value + : { href: params.value, label: params.value } + const allowedHref = safeHref(String(href), window.location.href) + const el = document.createElement(allowedHref ? 'a' : 'span') + el.textContent = String(label) + if (allowedHref) { + el.setAttribute('href', allowedHref) + el.setAttribute('target', '_blank') + el.setAttribute('rel', 'noopener noreferrer') + el.className = 'underline' + } + return el } } else { return undefined diff --git a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte index adc3fe6184..bce53625bb 100644 --- a/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte +++ b/frontend/src/lib/components/copilot/chat/LinkRenderer.svelte @@ -13,7 +13,7 @@ type WindmillItemKind, type WorkspaceItemTargetKind } from './workspaceItems.svelte' - import { safeHref } from './safeHref' + import { safeHref } from '$lib/utils/safeHref' type Props = { href?: string diff --git a/frontend/src/lib/components/copilot/chat/safeHref.test.ts b/frontend/src/lib/utils/safeHref.test.ts similarity index 100% rename from frontend/src/lib/components/copilot/chat/safeHref.test.ts rename to frontend/src/lib/utils/safeHref.test.ts diff --git a/frontend/src/lib/components/copilot/chat/safeHref.ts b/frontend/src/lib/utils/safeHref.ts similarity index 57% rename from frontend/src/lib/components/copilot/chat/safeHref.ts rename to frontend/src/lib/utils/safeHref.ts index 3acdabb464..1d583fa2f7 100644 --- a/frontend/src/lib/components/copilot/chat/safeHref.ts +++ b/frontend/src/lib/utils/safeHref.ts @@ -1,10 +1,8 @@ const SAFE_PROTOCOLS = ['http:', 'https:', 'mailto:'] /** - * The href a rendered markdown link may carry, or undefined for one that must be dropped. - * - * Markdown reaches the chat renderers from a model, and from another member for a shared - * artifact, and `svelte-exmarkdown` passes `javascript:` and `data:` hrefs through untouched. + * The href a link built from untrusted content may carry, or undefined for one that must be + * dropped: `javascript:`, `data:` and any other non-navigating scheme. * Relative links resolve against `base` (the page), so they stay. */ export function safeHref(href: string | undefined, base: string): string | undefined { From 62d0088de5463d44d00a8c9204e03d20649acd07 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 23 Sep 2026 17:37:41 +0200 Subject: [PATCH 33/34] feat: show restart from failed step in run page top bar (#11317) * feat: show restart from failed step in run page top bar Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep failed-step restart button when selection is not restartable Co-Authored-By: Claude Opus 5.5 (1M context) * feat: add restart link to the failed run status line Co-Authored-By: Claude Opus 5.5 (1M context) * fix: use the input error color for the progress bar error state Co-Authored-By: Claude Opus 5.5 (1M context) * feat: mention restarting on a fixed flow version in the error line Co-Authored-By: Claude Opus 5.5 (1M context) * feat: show a chevron on the restart button when it opens a picker Co-Authored-By: Claude Opus 5.5 (1M context) * fix: shrink run page top bar buttons so the bar fits on one line Co-Authored-By: Claude Opus 5.5 (1M context) * fix: target the step that failed the run, not a tolerated failure Co-Authored-By: Claude Opus 5.5 (1M context) * fix: address review nits on the restart link Co-Authored-By: Claude Opus 5.5 (1M context) * fix: skip canceled runs and match the restart link label to its text Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../lib/components/FlowRestartButton.svelte | 33 ++++- .../common/actionRow/ActionRow.svelte | 2 +- .../components/flows/FlowProgressBar.svelte | 6 +- .../components/progressBar/ProgressBar.svelte | 11 +- .../(root)/(logged)/run/[...run]/+page.svelte | 126 ++++++++++++------ 5 files changed, 133 insertions(+), 45 deletions(-) diff --git a/frontend/src/lib/components/FlowRestartButton.svelte b/frontend/src/lib/components/FlowRestartButton.svelte index 11f5cfd28a..2c7f323649 100644 --- a/frontend/src/lib/components/FlowRestartButton.svelte +++ b/frontend/src/lib/components/FlowRestartButton.svelte @@ -1,7 +1,7 @@