From 34a392fed3452b7701b913d1d8699d2282d81787 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:55:47 +0100 Subject: [PATCH 01/48] add AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS env var (#8482) * feat: add AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS env var Add workspace restrictions by Azure account name, similar to the existing S3_BUCKETS_WORKSPACE_RESTRICTIONS for bucket names. Refactored parsing into a shared parse_restrictions_from_str function. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to a997285e976d0642b72584e1966a70a79d84e7dc This commit updates the EE repository reference after PR #472 was merged in windmill-ee-private. Previous ee-repo-ref: 5718dc7deca18ad52ffb413813e97b8ca75805b8 New ee-repo-ref: a997285e976d0642b72584e1966a70a79d84e7dc Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-object-store/src/lib.rs | 192 +++++++++++++---------- backend/windmill-worker/src/common.rs | 8 + 3 files changed, 115 insertions(+), 87 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 27fc8090fd..820e457354 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -563877bf1c8b4184f638bab51be89b1c0aec6dad +a997285e976d0642b72584e1966a70a79d84e7dc diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 1449412a83..4caf67f348 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -65,8 +65,8 @@ pub mod object_store_reexports { pub use object_store::memory::InMemory; pub use object_store::path::Path; pub use object_store::{ - Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, - PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart, + Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, PutMultipartOpts, + PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart, }; } @@ -530,10 +530,7 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result error::Result> { let store = object_store::local::LocalFileSystem::new_with_prefix(root_path).map_err(|e| { - error::Error::internal_err(format!( - "Error building filesystem object store: {:?}", - e - )) + error::Error::internal_err(format!("Error building filesystem object store: {:?}", e)) })?; Ok(Arc::new(store)) } @@ -644,36 +641,45 @@ lazy_static::lazy_static! { static ref S3_BUCKET_RESTRICTIONS: Option>> = { parse_bucket_restrictions() }; + static ref AZ_ACCOUNT_NAME_RESTRICTIONS: Option>> = { + parse_az_account_name_restrictions() + }; } fn parse_bucket_restrictions() -> Option>> { let env_var = std::env::var("S3_BUCKETS_WORKSPACE_RESTRICTIONS").ok()?; - parse_bucket_restrictions_from_str(&env_var) + parse_restrictions_from_str(&env_var, "S3 bucket") } -fn parse_bucket_restrictions_from_str(input: &str) -> Option>> { +fn parse_az_account_name_restrictions() -> Option>> { + let env_var = std::env::var("AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS").ok()?; + parse_restrictions_from_str(&env_var, "Azure account name") +} + +fn parse_restrictions_from_str(input: &str, label: &str) -> Option>> { if input.trim().is_empty() { return None; } let mut restrictions = HashMap::new(); - for bucket_rule in input.split(';') { - let bucket_rule = bucket_rule.trim(); - if bucket_rule.is_empty() { + for rule in input.split(';') { + let rule = rule.trim(); + if rule.is_empty() { continue; } - let parts: Vec<&str> = bucket_rule.splitn(2, ':').collect(); + let parts: Vec<&str> = rule.splitn(2, ':').collect(); if parts.len() != 2 { tracing::warn!( - "Invalid bucket restriction format: '{}'. Expected 'bucket:workspace1,workspace2'", - bucket_rule + "Invalid {} restriction format: '{}'. Expected 'name:workspace1,workspace2'", + label, + rule ); continue; } - let bucket_name = parts[0].trim().to_string(); + let name = parts[0].trim().to_string(); let workspaces: Vec = parts[1] .split(',') .map(|w| w.trim().to_string()) @@ -682,26 +688,33 @@ fn parse_bucket_restrictions_from_str(input: &str) -> Option Option>> { + parse_restrictions_from_str(input, "S3 bucket") +} + pub fn check_bucket_workspace_restriction( bucket_name: &str, workspace_id: &str, @@ -719,6 +732,23 @@ pub fn check_bucket_workspace_restriction( Ok(()) } +pub fn check_az_account_name_workspace_restriction( + account_name: &str, + workspace_id: &str, +) -> error::Result<()> { + if let Some(ref restrictions) = *AZ_ACCOUNT_NAME_RESTRICTIONS { + if let Some(allowed_workspaces) = restrictions.get(account_name) { + if !allowed_workspaces.contains(&workspace_id.to_string()) { + return Err(error::Error::NotAuthorized(format!( + "Workspace '{}' is not authorized to access Azure account '{}'", + workspace_id, account_name + ))); + } + } + } + Ok(()) +} + pub const DEFAULT_STORAGE: &str = "_default_"; pub fn bundle(w_id: &str, hash: &str) -> String { @@ -739,7 +769,8 @@ pub async fn upload_artifact_to_store( #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let object_store: Option<()> = None; Ok( - if &windmill_common::utils::MODE_AND_ADDONS.mode == &windmill_common::utils::Mode::Standalone + if &windmill_common::utils::MODE_AND_ADDONS.mode + == &windmill_common::utils::Mode::Standalone && object_store.is_none() { let path = format!("{}/{}", standalone_dir, path); @@ -821,11 +852,11 @@ pub fn lfs_to_object_store_resource( })?; Ok(ObjectStoreResource::Gcs(gcs_resource)) } - LargeFileStorage::FilesystemStorage(fs) => Ok(ObjectStoreResource::Filesystem( - FilesystemSettings { + LargeFileStorage::FilesystemStorage(fs) => { + Ok(ObjectStoreResource::Filesystem(FilesystemSettings { root_path: fs.root_path.clone(), - }, - )), + })) + } } } @@ -995,13 +1026,9 @@ pub async fn convert_json_line_stream>( drop(file); let ctx = SessionContext::new(); - ctx.register_json( - "my_table", - path_str, - NdJsonReadOptions::default(), - ) - .await - .map_err(to_anyhow)?; + ctx.register_json("my_table", path_str, NdJsonReadOptions::default()) + .await + .map_err(to_anyhow)?; let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?; let schema = df.schema().clone().into(); @@ -1252,8 +1279,7 @@ mod tests { advanced_permissions: None, }); // resource_value is ignored for filesystem - let result = - lfs_to_object_store_resource(&lfs, serde_json::Value::Null).unwrap(); + let result = lfs_to_object_store_resource(&lfs, serde_json::Value::Null).unwrap(); match result { ObjectStoreResource::Filesystem(fs) => { assert_eq!(fs.root_path, "/tmp/mydata"); @@ -1279,10 +1305,18 @@ mod tests { port: None, }; let result = duckdb_connection_settings_internal(s3).unwrap(); - assert!(result.connection_settings_str.contains("SET s3_region='eu-west-1'")); - assert!(result.connection_settings_str.contains("SET s3_access_key_id='AKIA123'")); - assert!(result.connection_settings_str.contains("SET s3_secret_access_key='secret456'")); - assert!(result.connection_settings_str.contains("SET s3_url_style='path'")); + assert!(result + .connection_settings_str + .contains("SET s3_region='eu-west-1'")); + assert!(result + .connection_settings_str + .contains("SET s3_access_key_id='AKIA123'")); + assert!(result + .connection_settings_str + .contains("SET s3_secret_access_key='secret456'")); + assert!(result + .connection_settings_str + .contains("SET s3_url_style='path'")); assert!(!result.connection_settings_str.contains("SET s3_use_ssl=0")); assert_eq!(result.s3_bucket, Some("test-bucket".to_string())); assert!(result.azure_container_path.is_none()); @@ -1304,7 +1338,9 @@ mod tests { }; let result = duckdb_connection_settings_internal(s3).unwrap(); assert!(result.connection_settings_str.contains("SET s3_use_ssl=0")); - assert!(!result.connection_settings_str.contains("SET s3_url_style='path'")); + assert!(!result + .connection_settings_str + .contains("SET s3_url_style='path'")); } #[test] @@ -1320,8 +1356,12 @@ mod tests { federated_token_file: None, }); let result = format_duckdb_connection_settings(resource).unwrap(); - assert!(result.connection_settings_str.contains("AccountName=myaccount")); - assert!(result.connection_settings_str.contains("AccountKey=base64key==")); + assert!(result + .connection_settings_str + .contains("AccountName=myaccount")); + assert!(result + .connection_settings_str + .contains("AccountKey=base64key==")); assert_eq!( result.azure_container_path, Some("az://mycontainer".to_string()) @@ -1337,7 +1377,10 @@ mod tests { }); let result = format_duckdb_connection_settings(resource); assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("GCS is not supported")); + assert!(result + .unwrap_err() + .to_string() + .contains("GCS is not supported")); } #[test] @@ -1361,10 +1404,8 @@ mod tests { use windmill_common::error::Error; // NotFound - let err = object_store::Error::NotFound { - path: "test/path".into(), - source: "missing".into(), - }; + let err = + object_store::Error::NotFound { path: "test/path".into(), source: "missing".into() }; let mapped = object_store_error_to_error(err); assert!(matches!(mapped, Error::NotFound(_))); @@ -1388,10 +1429,8 @@ mod tests { assert!(matches!(mapped, Error::BadRequest(_))); // Unauthenticated - let err = object_store::Error::Unauthenticated { - path: "obj".into(), - source: "no creds".into(), - }; + let err = + object_store::Error::Unauthenticated { path: "obj".into(), source: "no creds".into() }; let mapped = object_store_error_to_error(err); assert!(matches!(mapped, Error::NotAuthorized(_))); } @@ -1533,15 +1572,10 @@ mod tests { .await .unwrap(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root.to_string(), - }); - let s3_obj = S3Object { - s3: "etag.txt".to_string(), - storage: None, - filename: None, - presigned: None, - }; + let resource = + ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root.to_string() }); + let s3_obj = + S3Object { s3: "etag.txt".to_string(), storage: None, filename: None, presigned: None }; let etag = get_etag_or_empty(&resource, s3_obj).await; // LocalFileSystem should return an etag based on file metadata @@ -1554,9 +1588,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root.to_string(), - }); + let resource = + ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root.to_string() }); let s3_obj = S3Object { s3: "nonexistent.txt".to_string(), storage: None, @@ -1575,9 +1608,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap().to_string(); - let settings = ObjectSettings::Filesystem(FilesystemSettings { - root_path: root, - }); + let settings = ObjectSettings::Filesystem(FilesystemSettings { root_path: root }); let expirable = build_object_store_from_settings(settings, None) .await @@ -1585,10 +1616,7 @@ mod tests { let data = bytes::Bytes::from("end to end via settings"); expirable .store - .put( - &Path::from("e2e.txt"), - PutPayload::from(data.clone()), - ) + .put(&Path::from("e2e.txt"), PutPayload::from(data.clone())) .await .unwrap(); let result = expirable @@ -1610,9 +1638,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap().to_string(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root, - }); + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root }); let client = build_object_store_client(&resource).await.unwrap(); let data = bytes::Bytes::from("end to end via resource"); @@ -1634,7 +1660,10 @@ mod tests { #[test] fn test_bundle_path_format() { - assert_eq!(bundle("my_workspace", "abc123"), "script_bundle/my_workspace/abc123"); + assert_eq!( + bundle("my_workspace", "abc123"), + "script_bundle/my_workspace/abc123" + ); } #[test] @@ -1646,8 +1675,7 @@ mod tests { #[test] fn test_parse_bucket_restrictions_single_bucket() { - let result = - parse_bucket_restrictions_from_str("my-bucket:workspace1,workspace2").unwrap(); + let result = parse_bucket_restrictions_from_str("my-bucket:workspace1,workspace2").unwrap(); assert_eq!(result.len(), 1); assert_eq!( result.get("my-bucket").unwrap(), @@ -1657,19 +1685,13 @@ mod tests { #[test] fn test_parse_bucket_restrictions_multiple_buckets() { - let result = parse_bucket_restrictions_from_str( - "bucket-a:ws1,ws2;bucket-b:ws3", - ) - .unwrap(); + let result = parse_bucket_restrictions_from_str("bucket-a:ws1,ws2;bucket-b:ws3").unwrap(); assert_eq!(result.len(), 2); assert_eq!( result.get("bucket-a").unwrap(), &vec!["ws1".to_string(), "ws2".to_string()] ); - assert_eq!( - result.get("bucket-b").unwrap(), - &vec!["ws3".to_string()] - ); + assert_eq!(result.get("bucket-b").unwrap(), &vec!["ws3".to_string()]); } #[test] @@ -1681,16 +1703,14 @@ mod tests { #[test] fn test_parse_bucket_restrictions_invalid_format_skipped() { // "no-colon" is invalid, only "valid:ws1" should be parsed - let result = - parse_bucket_restrictions_from_str("no-colon;valid:ws1").unwrap(); + let result = parse_bucket_restrictions_from_str("no-colon;valid:ws1").unwrap(); assert_eq!(result.len(), 1); assert!(result.contains_key("valid")); } #[test] fn test_parse_bucket_restrictions_trailing_semicolons() { - let result = - parse_bucket_restrictions_from_str(";bucket:ws1;;").unwrap(); + let result = parse_bucket_restrictions_from_str(";bucket:ws1;;").unwrap(); assert_eq!(result.len(), 1); assert!(result.contains_key("bucket")); } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bdecb0526f..f62964c60c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1000,6 +1000,14 @@ pub(crate) async fn get_workspace_s3_resource_path( windmill_object_store::check_bucket_workspace_restriction(bucket, workspace_id)?; } + // Check Azure account name workspace restrictions + if let ObjectStoreResource::Azure(azure_resource) = &object_store_resource { + windmill_object_store::check_az_account_name_workspace_restriction( + &azure_resource.account_name, + workspace_id, + )?; + } + Ok(Some(object_store_resource)) } From f329ee7aaefbae0ad344743c40825440a936bd30 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 23 Mar 2026 17:01:01 +0000 Subject: [PATCH 02/48] fix: respect NO_COLOR env variable for stdout log output (#8483) Co-authored-by: Claude Opus 4.6 (1M context) --- backend/windmill-common/src/tracing_init.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index f4892b67c9..3b009a36eb 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -68,7 +68,11 @@ pub fn initialize_tracing( mode: &Mode, environment: &str, ) -> (WorkerGuard, crate::otel_oss::OtelProvider) { - let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); + let style = if std::env::var("NO_COLOR").is_ok() { + "never".into() + } else { + std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()) + }; let rust_log_env = std::env::var("RUST_LOG"); let rust_log_stdout_env = std::env::var("RUST_LOG_STDOUT"); From 010753c73ac85237af50acadf9c08567b1bc993c Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:59:46 +0100 Subject: [PATCH 03/48] fix: skip debounce arg accumulation when batch table is empty (CE) (#8485) On CE (without private feature), v2_job_debounce_batch is never populated because maybe_debounce_post_preprocessing is EE-only. The accumulation query returns zero rows, producing an empty array that replaces the original nodes_to_relock value. This causes flow modules to never get relocked when triggered by relative imports. Fix: only replace the original value when the batch query actually returned entries to accumulate. Co-authored-by: Claude Opus 4.6 --- backend/windmill-queue/src/jobs.rs | 69 ++++++++++++++++-------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1aff43a388..a6ba745c24 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3100,41 +3100,46 @@ impl PulledJobResult { "Accumulated arguments from debounced jobs in batch" ); - let new_value = to_raw_value(&accumulated_arg); + // If the batch query returned no entries (e.g. CE where + // v2_job_debounce_batch is never populated), keep the + // original value unchanged instead of replacing it with []. + if !accumulated_arg.is_empty() { + let new_value = to_raw_value(&accumulated_arg); - let original_value = j - .args - .as_ref() - .and_then(|a| a.get(arg_name_to_accumulate)) - .map(|v| v.get().to_string()) - .unwrap_or_else(|| "null".to_string()); + let original_value = j + .args + .as_ref() + .and_then(|a| a.get(arg_name_to_accumulate)) + .map(|v| v.get().to_string()) + .unwrap_or_else(|| "null".to_string()); - append_logs( - &j_id, - &j.workspace_id, - format!( - "Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n", - &new_value - ), - &(db.into()), - ) - .await; - - j.args - .get_or_insert(Json(Default::default())) - .as_mut() - .insert(arg_name_to_accumulate.to_owned(), new_value); - - // Persist accumulated args to v2_job so that flow steps - // re-reading from the DB (via get_mini_pulled_job) see them - if let Some(ref args) = j.args { - sqlx::query!( - "UPDATE v2_job SET args = $2 WHERE id = $1", - j_id, - args as &Json>>, + append_logs( + &j_id, + &j.workspace_id, + format!( + "Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n", + &new_value + ), + &(db.into()), ) - .execute(db) - .await?; + .await; + + j.args + .get_or_insert(Json(Default::default())) + .as_mut() + .insert(arg_name_to_accumulate.to_owned(), new_value); + + // Persist accumulated args to v2_job so that flow steps + // re-reading from the DB (via get_mini_pulled_job) see them + if let Some(ref args) = j.args { + sqlx::query!( + "UPDATE v2_job SET args = $2 WHERE id = $1", + j_id, + args as &Json>>, + ) + .execute(db) + .await?; + } } } From 9643006f1e90b991b334bb58caf62301bc26d09d Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:20:19 +0100 Subject: [PATCH 04/48] feat(cli): better stale scripts detection #3 (#8480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix Signed-off-by: pyranota * reduce tests Signed-off-by: pyranota * update Signed-off-by: pyranota * fix Signed-off-by: pyranota * update Signed-off-by: pyranota * WIP: stash changes after merge with origin/main * Delete backend/parsers/windmill-parser-wasm/Cargo.lock * reset cargo.toml * feat(cli): integrate dependency tree into generate-metadata command - Add isDirectlyStale field to DependencyNode for staleness tracking - Update addScript to accept itemType, folder, isRawApp, isDirectlyStale - Update propagateStaleness to use isDirectlyStale field instead of parameter - Handlers now determine staleness and pass it to tree.addScript - generate-metadata calls propagateStaleness() and populates staleItems from tree - Pass legacyBehaviour=false and tree to handlers during generation phase 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * fix(cli): store originalPath in tree for correct handler invocation Scripts need the path with extension to be passed to the handler. Added originalPath field to DependencyNode to track this. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * fix parsers Signed-off-by: pyranota * rever sqlx removal * update sqlx * feat: make py-imports parser WASM-compatible and add as separate WASM package Gate heavy deps (sqlx, windmill-common, async-recursion, toml, pep440_rs, tracing) behind cfg(not(wasm32)). Make parse_code_for_imports, parse_relative_imports, NImport, and ImportPin public. Remove duplicate import_parser from parser-py (reset to origin/main). Add py-imports-parser feature to windmill-parser-wasm and py-imports target to build.nu. Co-Authored-By: Claude Opus 4.6 * safer return * update * fix: CLI metadata fixes - folder filter, staleness detection, WASM py-imports setup - Fix lazy_static cfg gating for WASM compatibility (split into separate blocks) - Fix folder argument filter to match specific file paths (not just directories) - Fix staleness detection to use checkHash with conf (includes module hashes) - Convert relative_imports_skip tests from Deno to bun APIs - Add windmill-parser-wasm-py-imports to CLI and build-npm dependencies - Relax module stale test to not require per-module change detail in output Co-Authored-By: Claude Opus 4.6 * fix: restore temp_script_refs parameter in parse_python_imports Re-adds the temp_script_refs parameter that was lost when resetting py-imports crate to origin/main. This enables resolving relative imports from not-yet-deployed scripts during CLI lock generation. * fixes * extend testsuit * update ee repo ref * fix: diff endpoint bytea cast, upload only mismatched scripts - Add POST /scripts/raw_temp/diff endpoint to batch-compare local content hashes against deployed versions using Postgres sha256() - Use convert_to(content, 'UTF8') instead of content::bytea to avoid failure on scripts containing backslash sequences (e.g. \n) - CLI now diffs all scripts against deployed, uploads only mismatched ones - propagateStaleness no longer deletes non-stale nodes (needed for diff) - Suppress verbose log.info messages during metadata generation - Add E2E tests for locally modified and unpushed helper scripts Co-Authored-By: Claude Opus 4.6 * rework * sqlx * fixes * add index * expand tests * fix flows * archive script before executing * disable tests for ci * skip Python-dependent E2E tests on CI Tests requiring the python backend feature are skipped when CI_MINIMAL_FEATURES=true since CI builds with zip-only features. Co-Authored-By: Claude Opus 4.6 * fix: make flow fixture lock optional and reset nonDottedPaths after tests Flow fixtures no longer emit an empty lock file by default. The lockContent parameter controls whether a lock: "!inline ..." line appears in flow.yaml. This prevents flows from appearing "up-to-date" when they should be processed by generate-metadata. Also adds afterAll to reset setNonDottedPaths(false) so global state doesn't leak between test files when run together. Co-Authored-By: Claude Opus 4.6 * debug: add error logging in withTestBackend to diagnose CI failures Co-Authored-By: Claude Opus 4.6 * debug: add --bail 1 to CI test runner to show full error on first failure Co-Authored-By: Claude Opus 4.6 * debug: include CLI stdout/stderr in assertion message for workspace deps test Co-Authored-By: Claude Opus 4.6 * fix: set WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES in test backend The workspace deps feature requires workers to report their version, but in test/CI there are no separate workers (standalone mode). The version check fails because workers haven't had time to ping yet. Setting this env var bypasses the version check. Also reverts --bail 1 from CI workflow now that the root cause is fixed. Co-Authored-By: Claude Opus 4.6 * debug: add --bail 1 to Windows CI and assertion messages for Windows failure diagnosis Co-Authored-By: Claude Opus 4.6 * fix: replace TEMP_SCRIPT_REFS_PLACEHOLDER in bun builder tests The loader.bun.js now includes a TEMP_SCRIPT_REFS_PLACEHOLDER that must be replaced before execution. The builder tests were missing this replacement, causing all 6 bun_builder_tests to fail. Co-Authored-By: Claude Opus 4.6 * fix: use cdirFwd in Windows loader filterLoad regex Raw cdir (with backslashes) interpolated into RegExp causes \r to become carriage return and \w to become word-char, so filterLoad never matches main.ts. This prevents replaceRelativeImports from running, leaving bare relative imports like "./script_b" in the bundled output, which scanImports then misparses as package ".". Co-Authored-By: Claude Opus 4.6 * fix: Windows filterLoad regex + graceful fallback for old backends - Fix filterLoad in loader.bun.windows.js to match both native backslash and forward-slash paths from Bun's resolver by escaping cdir for regex - Wrap uploadScripts in try/catch so generate-metadata degrades gracefully when the backend lacks /raw_temp endpoints (locks use deployed versions) - Add TODO for missing TEMP_SCRIPT_REFS support in Windows loader Co-Authored-By: Claude Opus 4.6 * debug: add loader/builder debug logging for Windows CI diagnosis Temporary console.log statements to understand: - What path Bun passes to onLoad for main.ts - Whether filterLoad regex matches - Whether replaceRelativeImports fires - What the bundled output contains - What imports scanImports extracts Co-Authored-By: Claude Opus 4.6 * chore: trigger CI for cli path Co-Authored-By: Claude Opus 4.6 * chore: trigger CI via workflow file change Co-Authored-By: Claude Opus 4.6 * Add TEMP_SCRIPT_REFS to Windows loader, use .ts extensions in test imports - Add TEMP_SCRIPT_REFS_PLACEHOLDER support to loader.bun.windows.js (mirrors loader.bun.js) so CLI lock generation can resolve imports from locally-modified scripts on Windows - Use .ts extensions in all test relative imports to work around the Windows filterLoad regex bug (replaceRelativeImports doesn't fire on Windows, so extensionless imports fail) - Remove unused uploadSucceeded variable Co-Authored-By: Claude Opus 4.6 * Remove debug logging from loader_builder.bun.js Co-Authored-By: Claude Opus 4.6 * Remove windmill-parser-wasm-py-imports from frontend package.json This dependency is only needed by the CLI, not the frontend. Co-Authored-By: Claude Opus 4.6 * debug: add temp_script_refs logging for Windows CI investigation Co-Authored-By: Claude Opus 4.6 * ci: remove --bail 1 from Windows CLI tests Co-Authored-By: Claude Opus 4.6 * fix: normalize backslashes in folder filter treePath lookup (Windows) On Windows, item.path (originalPath) uses backslashes but tree keys use forward slashes. The isRelevant filter's touchesFolder call passed the unnormalized path to traverseTransitive, which couldn't find the node. This caused cross-folder importers to be excluded from generate-metadata when a folder argument was specified. Also removes debug logging from previous commit. Co-Authored-By: Claude Opus 4.6 * Update cli-tests.yml * fix: normalize backslashes in strict-folder-boundaries warning message (Windows) Co-Authored-By: Claude Opus 4.6 * chore: update ee-repo-ref to fe8f0d1d7448464c98474d994e6492c0a45e8e38 This commit updates the EE repository reference after PR #467 was merged in windmill-ee-private. Previous ee-repo-ref: 03e6eaf950776c96b9581848a583af9ad735be60 New ee-repo-ref: fe8f0d1d7448464c98474d994e6492c0a45e8e38 Automated by sync-ee-ref workflow. * revert cli-tests.yml --------- Signed-off-by: pyranota Co-authored-by: Claude Opus 4.5 Co-authored-by: windmill-internal-app[bot] --- ...32b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json | 12 + ...40af492d4c8a8871cef972980150f319fe6ff.json | 16 + ...7b932800e33cd462a651f7e6716929ee9b6f2.json | 22 + ...a158db101f54f0908551b5a4f5e6655e122b.json} | 4 +- backend/Cargo.lock | 4 +- backend/ee-repo-ref.txt | 2 +- .../20260304000000_raw_script_temp.down.sql | 2 + .../20260304000000_raw_script_temp.up.sql | 11 + .../windmill-parser-py-imports/Cargo.toml | 20 +- .../windmill-parser-py-imports/src/lib.rs | 80 +- .../windmill-parser-py-imports/tests/tests.rs | 3 + .../parsers/windmill-parser-sql/Cargo.toml | 1 - .../parsers/windmill-parser-sql/src/lib.rs | 3 +- backend/parsers/windmill-parser-ts/src/lib.rs | 82 ++ .../parsers/windmill-parser-ts/tests/tests.rs | 84 +- .../parsers/windmill-parser-wasm/Cargo.toml | 2 + backend/parsers/windmill-parser-wasm/build.nu | 6 + backend/parsers/windmill-parser-wasm/dev.nu | 7 +- .../windmill-parser-wasm/publish-pkgs.sh | 3 + .../parsers/windmill-parser-wasm/src/lib.rs | 19 + backend/parsers/windmill-parser/src/lib.rs | 17 + backend/src/main.rs | 1 + backend/tests/bun_jobs.rs | 2 + backend/tests/nativets_dedicated.rs | 1 + backend/windmill-api-scripts/src/scripts.rs | 142 ++ backend/windmill-api/openapi.yaml | 77 + backend/windmill-api/src/jobs.rs | 12 + backend/windmill-common/src/cache.rs | 35 +- backend/windmill-types/Cargo.toml | 1 + backend/windmill-types/src/s3.rs | 17 +- backend/windmill-worker/loader.bun.js | 17 +- backend/windmill-worker/loader.bun.windows.js | 23 +- backend/windmill-worker/src/bun_executor.rs | 26 +- .../windmill-worker/src/python_executor.rs | 1 + .../windmill-worker/src/worker_lockfiles.rs | 48 +- cli/build-npm.ts | 1 + cli/bun.lock | 7 +- cli/package.json | 3 +- cli/src/commands/app/app_metadata.ts | 148 +- cli/src/commands/flow/flow.ts | 6 + cli/src/commands/flow/flow_metadata.ts | 147 +- .../generate-metadata/generate-metadata.ts | 257 ++-- cli/src/commands/script/script.ts | 4 +- cli/src/commands/sync/sync.ts | 10 +- cli/src/utils/dependency_tree.ts | 373 +++++ cli/src/utils/metadata.ts | 104 +- cli/src/utils/relative_imports.ts | 39 + cli/src/utils/resource_folders.ts | 20 + cli/test/cargo_backend.ts | 29 +- cli/test/relative_imports_skip.test.ts | 420 ++++++ cli/test/relative_imports_wasm.test.ts | 1235 +++++++++++++++++ cli/test/resource_folders_unit.test.ts | 34 + cli/test/sync_pull_push.test.ts | 3 +- cli/test/test_backend.ts | 121 +- cli/test/test_fixtures.ts | 99 +- cli/test/unified_generate_metadata.test.ts | 5 +- 56 files changed, 3565 insertions(+), 303 deletions(-) create mode 100644 backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json create mode 100644 backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json create mode 100644 backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json rename backend/.sqlx/{query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json => query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json} (50%) create mode 100644 backend/migrations/20260304000000_raw_script_temp.down.sql create mode 100644 backend/migrations/20260304000000_raw_script_temp.up.sql create mode 100644 cli/src/utils/dependency_tree.ts create mode 100644 cli/src/utils/relative_imports.ts create mode 100644 cli/test/relative_imports_skip.test.ts create mode 100644 cli/test/relative_imports_wasm.test.ts diff --git a/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json new file mode 100644 index 0000000000..3c7f23d1af --- /dev/null +++ b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce" +} diff --git a/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json new file mode 100644 index 0000000000..10ec0a5c49 --- /dev/null +++ b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at)\n VALUES ($1, $2, $3, NOW())\n ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bpchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff" +} diff --git a/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json new file mode 100644 index 0000000000..ed1b03fd77 --- /dev/null +++ b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT content FROM raw_script_temp WHERE hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2" +} diff --git a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json similarity index 50% rename from backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json rename to backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json index 0946fa4006..c910be0b82 100644 --- a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json +++ b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", + "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623" + "hash": "96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 25c4233bbd..e40478cc4b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16898,8 +16898,6 @@ dependencies = [ "async-recursion", "itertools 0.14.0", "lazy_static", - "malachite", - "malachite-bigint", "pep440_rs", "phf 0.11.3", "regex", @@ -16956,7 +16954,6 @@ dependencies = [ "serde", "serde_json", "windmill-parser", - "windmill-types", ] [[package]] @@ -17465,6 +17462,7 @@ dependencies = [ "strum 0.27.2", "tracing", "uuid", + "windmill-parser", ] [[package]] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 820e457354..f30dcc7a1f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a997285e976d0642b72584e1966a70a79d84e7dc +fe8f0d1d7448464c98474d994e6492c0a45e8e38 diff --git a/backend/migrations/20260304000000_raw_script_temp.down.sql b/backend/migrations/20260304000000_raw_script_temp.down.sql new file mode 100644 index 0000000000..05703e4f18 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_raw_script_temp_created_at; +DROP TABLE IF EXISTS raw_script_temp; diff --git a/backend/migrations/20260304000000_raw_script_temp.up.sql b/backend/migrations/20260304000000_raw_script_temp.up.sql new file mode 100644 index 0000000000..c7b5a23754 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.up.sql @@ -0,0 +1,11 @@ +-- Temporary storage for raw script content during CLI lock generation +-- Content is stored with hash as key, cleaned up after 1 week +CREATE TABLE raw_script_temp ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id), + hash CHAR(64) NOT NULL, + content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, hash) +); + +CREATE INDEX IF NOT EXISTS idx_raw_script_temp_created_at ON raw_script_temp (created_at); diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index deea68c2ef..565e1422bb 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -13,21 +13,19 @@ regex-lite.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] regex.workspace = true - -[dependencies] -windmill-parser.workspace = true windmill-common.workspace = true -rustpython-parser.workspace = true -malachite.workspace = true -malachite-bigint.workspace = true -phf.workspace = true -itertools.workspace = true -serde_json.workspace = true -anyhow.workspace = true -lazy_static.workspace = true sqlx.workspace = true async-recursion.workspace = true toml.workspace = true serde.workspace = true pep440_rs.workspace = true tracing.workspace = true + +[dependencies] +windmill-parser.workspace = true +rustpython-parser.workspace = true +phf.workspace = true +itertools.workspace = true +serde_json.workspace = true +anyhow.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 06762dbc9e..de7f51bc79 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -8,10 +8,14 @@ mod mapping; +#[cfg(not(target_arch = "wasm32"))] use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; -use std::{collections::HashMap, str::FromStr}; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; +#[cfg(not(target_arch = "wasm32"))] +use std::collections::HashMap; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -24,7 +28,9 @@ use rustpython_parser::{ text_size::TextRange, Parse, }; +#[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; +#[cfg(not(target_arch = "wasm32"))] use windmill_common::{ error::{self, to_anyhow}, worker::{ @@ -46,10 +52,14 @@ fn replace_full_import(x: &str) -> Option { FULL_IMPORTS_MAP.get(x).map(|x| (*x).to_owned()) } +#[cfg(not(target_arch = "wasm32"))] lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); - static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap(); +} + +lazy_static! { + static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); // Regex to properly match main function definition at line start, // capturing both sync and async variants static ref DEF_MAIN_RE: Regex = Regex::new(r"(?m)^(async\s+)?def\s+main\s*\(").unwrap(); @@ -82,7 +92,7 @@ fn process_import(module: Option, path: &str, level: usize) -> Vec error::Result> { +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { let nimports = parse_code_for_imports(code, path)?; return Ok(nimports .into_iter() @@ -94,7 +104,7 @@ pub fn parse_relative_imports(code: &str, path: &str) -> error::Result error::Result> { +pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -175,7 +187,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> let code_with_fake_main = format!("{}\n\ndef main(): pass", code); let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) + anyhow::anyhow!("Error parsing code for imports: {}", e.to_string()) })?; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main @@ -256,6 +268,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> return Ok(nimports); } +#[cfg(not(target_arch = "wasm32"))] pub async fn parse_python_imports( code: &str, w_id: &str, @@ -264,6 +277,7 @@ pub async fn parse_python_imports( version_specifiers: &mut Vec, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -276,6 +290,7 @@ pub async fn parse_python_imports( &mut None, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -313,6 +328,7 @@ pub async fn parse_python_imports( Ok((imports, compile_error_hint)) } +#[cfg(not(target_arch = "wasm32"))] fn extract_pkg_name(requirement: &str) -> String { PKG_RE .captures(requirement) @@ -320,6 +336,7 @@ fn extract_pkg_name(requirement: &str) -> String { .unwrap_or_default() } +#[cfg(not(target_arch = "wasm32"))] #[async_recursion] async fn parse_python_imports_inner( code: &str, @@ -331,6 +348,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv: &mut Option, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); @@ -494,17 +512,37 @@ async fn parse_python_imports_inner( for n in nimports.into_iter() { let mut nested = match n { NImport::Relative(rpath) => { - let code = sqlx::query_scalar!( - r#" - SELECT content FROM script WHERE path = $1 AND workspace_id = $2 - AND archived = false ORDER BY created_at DESC LIMIT 1 - "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()); + // First try to get content from temp_script_refs cache if available + let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { + tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { + Ok(content) => Some(content), + Err(e) => { + tracing::warn!("temp_script_refs hash '{}' not found in cache: {}, falling back to deployed script", hash, e); + None + } + } + } else { + None + }; + + // Use cached content if available, otherwise fall back to deployed script + let code = match code_from_cache { + Some(content) => content, + None => { + sqlx::query_scalar!( + r#" + SELECT content FROM script WHERE path = $1 AND workspace_id = $2 + AND archived = false ORDER BY created_at DESC LIMIT 1 + "#, + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()) + } + }; if already_visited.contains(&rpath) { vec![] @@ -522,6 +560,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -646,6 +685,7 @@ async fn parse_python_imports_inner( Ok(final_imports) } +#[cfg(not(target_arch = "wasm32"))] fn extract_nimports_from_content( content: &str, hm: &mut HashMap, diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index e61b4cc7db..c853c8e5b4 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -26,6 +26,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -67,6 +68,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -98,6 +100,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index e84888280c..143b02a2cc 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -16,7 +16,6 @@ regex.workspace = true [dependencies] windmill-parser.workspace = true -windmill-types.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 4938a8e194..c8aef3eed5 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -15,7 +15,7 @@ use std::{ iter::Peekable, str::CharIndices, }; -pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ}; +pub use windmill_parser::{s3_mode_extension, Arg, MainArgSignature, ObjectType, S3ModeFormat, Typ}; pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__"; pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__"; @@ -143,7 +143,6 @@ pub fn parse_db_resource(code: &str) -> Option { cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap()) } -pub use windmill_types::s3::{s3_mode_extension, S3ModeFormat}; pub struct S3ModeArgs { pub prefix: Option, pub storage: Option, diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 53ecaf277c..125834a3ac 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -117,6 +117,12 @@ impl Visit for ImportsFinder { } } +/// Parse TypeScript/JavaScript code and extract all import paths as raw strings. +/// +/// Returns import paths exactly as written in the code (e.g., `"./module"`, `"../utils"`, `"lodash"`). +/// Does not resolve relative paths to absolute Windmill paths. +/// +/// See also: [`parse_relative_imports`] for resolved absolute paths. pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result> { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into()); @@ -151,6 +157,82 @@ pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Resul Ok(imports) } +/// Parse TypeScript/JavaScript code and extract relative imports resolved to absolute Windmill paths. +/// +/// Takes the script's Windmill path (e.g., `"f/folder/script"`) and resolves relative imports +/// like `"./module"` or `"../utils"` to absolute paths like `"f/folder/module"` or `"f/utils"`. +/// +/// Only returns relative imports (those starting with `./`, `../`, or `/`). +/// External package imports (e.g., `"lodash"`) are filtered out. +/// +/// See also: [`parse_expr_for_imports`] for raw import strings without resolution. +/// +/// # Arguments +/// * `code` - The TypeScript/JavaScript source code +/// * `path` - The Windmill path of the script (e.g., `"f/folder/script"`) +/// +/// # Returns +/// A sorted, deduplicated list of resolved absolute Windmill paths. +/// +/// # Examples +/// ```ignore +/// // Script at "f/folder/script" with: import { x } from "../utils" +/// // Returns: ["f/utils"] +/// ``` +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { + let imports = parse_expr_for_imports(code, false)?; + let script_dir = path.rsplit_once('/').map(|(dir, _)| dir).unwrap_or(""); + + let mut resolved: Vec = imports + .into_iter() + .filter(|imp| is_relative_import(imp)) + .map(|imp| { + // Remove .ts extension if present + let imp = imp.strip_suffix(".ts").unwrap_or(&imp); + + if imp.starts_with("/") { + // Absolute path (e.g., /f/folder/script) - remove leading slash + imp[1..].to_string() + } else { + // Relative path (e.g., ./script or ../folder/script) + let combined = format!("{}/{}", script_dir, imp); + normalize_path(&combined) + } + }) + .collect(); + + resolved.sort(); + resolved.dedup(); + Ok(resolved) +} + +/// Check if an import path is a relative import (starts with `./`, `../`, or `/`) +fn is_relative_import(import_path: &str) -> bool { + import_path.starts_with("./") + || import_path.starts_with("../") + || import_path.starts_with("/") +} + +/// Normalize a path by resolving `.` and `..` components +fn normalize_path(input_path: &str) -> String { + let parts: Vec<&str> = input_path.split('/').filter(|p| !p.is_empty()).collect(); + let mut result: Vec<&str> = Vec::new(); + + for part in parts { + if part == "." { + continue; + } else if part == ".." { + if !result.is_empty() { + result.pop(); + } + } else { + result.push(part); + } + } + + result.join("/") +} + struct OutputFinder { idents: HashSet<(String, String)>, } diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 7a685fbb77..b50169ca68 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -2,7 +2,7 @@ mod tests { use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; - use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports}; + use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports, parse_relative_imports}; #[test] fn test_imports_basic() { @@ -798,7 +798,7 @@ mod tests { // Test case where there are exports but no preprocessor let code = r#" export { foo, bar } from "./utils"; - + export async function main(param: string) { return param; } @@ -806,4 +806,84 @@ mod tests { let sig = parse_deno_signature(code, false, false, None).unwrap(); assert_eq!(sig.has_preprocessor, Some(false)); } + + // ========================================================================== + // Tests for parse_relative_imports + // ========================================================================== + + #[test] + fn test_relative_imports_dot() { + let code = r#" + import { helper } from "./helper"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_double_dot() { + let code = r#" + import { utils } from "../utils/helper"; + export async function main() { return utils(); } + "#; + let result = parse_relative_imports(code, "f/folder/subfolder/script").unwrap(); + assert_eq!(result, vec!["f/folder/utils/helper"]); + } + + #[test] + fn test_relative_imports_absolute_path() { + let code = r#" + import { shared } from "/f/shared/utils"; + export async function main() { return shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/shared/utils"]); + } + + #[test] + fn test_relative_imports_mixed() { + let code = r#" + import { helper } from "./helper"; + import { utils } from "../utils"; + import { shared } from "/f/shared/lib"; + import lodash from "lodash"; + export async function main() { return helper() + utils() + shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + // Should only include relative imports, not external packages like lodash + assert_eq!(result, vec!["f/folder/helper", "f/shared/lib", "f/utils"]); + } + + #[test] + fn test_relative_imports_with_ts_extension() { + let code = r#" + import { helper } from "./helper.ts"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_external_only() { + let code = r#" + import lodash from "lodash"; + import { something } from "@scope/package"; + export async function main() { return lodash.map([]); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn test_relative_imports_deeply_nested() { + let code = r#" + import { a } from "../../a"; + import { b } from "../../../b"; + export async function main() { return a() + b(); } + "#; + let result = parse_relative_imports(code, "f/one/two/three/script").unwrap(); + assert_eq!(result, vec!["f/b", "f/one/a"]); + } } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 8372b1838c..895a35713e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -40,6 +40,7 @@ java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] wac-parser = [ "dep:windmill-parser-wac"] asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"] +py-imports-parser = [ "dep:windmill-parser-py-imports"] [dependencies] anyhow.workspace = true @@ -61,6 +62,7 @@ windmill-parser-wac = { workspace = true, optional = true } windmill-parser-ts-asset = { workspace = true, optional = true } windmill-parser-py-asset = { workspace = true, optional = true } windmill-parser-sql-asset = { workspace = true, optional = true } +windmill-parser-py-imports = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 4e08ce95d3..212c5f3a37 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -67,6 +67,12 @@ const targets = [ features: "asset-parser", env: "default", }, + { + ident: "py-imports", + desc: "Python imports" + features: "py-imports-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/dev.nu b/backend/parsers/windmill-parser-wasm/dev.nu index f4d7ef6495..ec865f36dc 100755 --- a/backend/parsers/windmill-parser-wasm/dev.nu +++ b/backend/parsers/windmill-parser-wasm/dev.nu @@ -1,6 +1,6 @@ #!/usr/bin/env nu - -# Build in debug mode specified lang parser to wasm + +# Build in debug mode specified lang parser to wasm # and perform installation to frontend def "main" [ lang: string # Example: nu @@ -9,4 +9,7 @@ def "main" [ ( cd ../../../frontend; npm install ../backend/parsers/windmill-parser-wasm/pkg-($lang) ) + ( + cd ../../../cli; bun install ../backend/parsers/windmill-parser-wasm/pkg-($lang) + ) } diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index 80f31650f3..3ac0ceef18 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -36,3 +36,6 @@ popd pushd "pkg-asset" && npm publish ${args} popd + +pushd "pkg-py-imports" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2af0a3bc9a..a2cef1b0ef 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -38,6 +38,8 @@ pub fn parse_outputs(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return raw import strings. +/// See [`parse_ts_relative_imports`] for resolved absolute paths. #[cfg(feature = "ts-parser")] #[wasm_bindgen] pub fn parse_ts_imports(code: &str) -> String { @@ -50,6 +52,15 @@ pub fn parse_ts_imports(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +/// See [`parse_ts_imports`] for raw import strings. +#[cfg(feature = "ts-parser")] +#[wasm_bindgen] +pub fn parse_ts_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_ts::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "bash-parser")] #[wasm_bindgen] pub fn parse_bash(code: &str) -> String { @@ -214,6 +225,14 @@ pub fn parse_assets_py(code: &str) -> String { } } +/// Parse Python imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +#[cfg(feature = "py-imports-parser")] +#[wasm_bindgen] +pub fn parse_py_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_py_imports::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_assets_ansible(code: &str) -> String { diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index dd2eada7cf..80019d044f 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -14,6 +14,23 @@ use serde_json::Value; pub mod asset_parser; +/// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) +#[derive(Clone, Copy, Debug)] +pub enum S3ModeFormat { + Json, + Csv, + Parquet, +} + +/// Returns the file extension for the given S3 mode format +pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { + match format { + S3ModeFormat::Json => "json", + S3ModeFormat::Csv => "csv", + S3ModeFormat::Parquet => "parquet", + } +} + #[derive(Serialize, Debug, PartialEq, Default)] pub struct MainArgSignature { pub star_args: bool, diff --git a/backend/src/main.rs b/backend/src/main.rs index 6d977c2db4..17567167a3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -312,6 +312,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "cache_init", "", &mut None, + &None, ) .await { diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index b2eb89112c..2edfef8989 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -921,6 +921,7 @@ mod dedicated_worker_protocol { "test-workspace", "f/test/script", LoaderMode::Node, + &None, )) .expect("build_loader failed"); @@ -1299,6 +1300,7 @@ mod bun_builder_tests { // Write build.js using the loader and builder constants directly // Parameters are dummy values since tests don't use Windmill relative imports let loader = RELATIVE_BUN_LOADER + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", "{}") .replace("W_ID", "test-workspace") .replace("BASE_INTERNAL_URL", "http://localhost:8000") .replace("TOKEN", "test-token") diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index f13759e5bc..1c6987a480 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -32,6 +32,7 @@ mod prewarmed_isolate_tests { "test-workspace", "f/test/script", LoaderMode::BrowserBundle, + &None, ) .await .expect("build_loader failed"); diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index fcbb159887..56de4d9892 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -239,6 +239,9 @@ pub fn workspaced_service() -> Router { "/history_update/h/:hash/p/*path", post(update_script_history), ) + // Temporary raw script storage for CLI lock generation + .route("/raw_temp/store", post(store_raw_script_temp)) + .route("/raw_temp/diff", post(diff_raw_scripts_with_deployed)) } #[derive(Serialize, FromRow)] @@ -1614,6 +1617,9 @@ struct RawScriptByPathQuery { cache_key: Option, // used specifically for python to cache folders on import success to avoid extra db calls on package fetch cache_folders: Option, + // If provided, load content from raw_script_temp table using this hash instead of deployed script. + // Used by CLI lock generation to resolve imports from not-yet-deployed scripts. + temp_script_hash: Option, } struct StringWithLength(String); @@ -1672,6 +1678,16 @@ async fn raw_script_by_path_internal( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + + // If temp_script_hash is provided, try loading from temp storage first. + // This is used by CLI lock generation to resolve imports from not-yet-deployed scripts. + // Falls back to the normal deployed script lookup if not found in temp storage. + if let Some(hash) = query.temp_script_hash { + if let Ok(content) = windmill_common::cache::raw_script_temp::load(hash, &db).await { + return Ok(content); + } + } + let cache_path = query .cache_key .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); @@ -2463,3 +2479,129 @@ async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { Ok(()) } } + +// ============================================================================ +// Temporary Raw Script Storage for CLI Lock Generation +// ============================================================================ + +/// Store raw script content temporarily for CLI lock generation. +async fn store_raw_script_temp( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(content): Json, +) -> Result> { + check_scopes(&authed, || "scripts:write".to_string())?; + + let hash = windmill_common::cache::raw_script_temp::compute_hash(&w_id, &content); + + // Store to DB + sqlx::query!( + "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + &w_id, + &hash, + &content + ) + .execute(&db) + .await?; + + // Clean up old entries (1 week TTL) + sqlx::query!( + "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'" + ) + .execute(&db) + .await?; + + Ok(Json(hash)) +} + +/// Compare local script content hashes with deployed versions. +/// Receives a map of path → SHA256(content), returns paths where the hash +/// differs from the deployed script (or the script doesn't exist on remote). +/// Hash comparison is done entirely in Postgres to avoid transferring content. +#[derive(Deserialize)] +struct WorkspaceDepDiff { + path: String, + language: ScriptLang, + name: Option, + hash: String, +} + +#[derive(Deserialize)] +struct DiffRequest { + scripts: std::collections::HashMap, + #[serde(default)] + workspace_deps: Vec, +} + +async fn diff_raw_scripts_with_deployed( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result>> { + check_scopes(&authed, || "scripts:read".to_string())?; + + let mut matching_set: std::collections::HashSet = std::collections::HashSet::new(); + let mut all_paths: Vec = Vec::new(); + + // --- Scripts --- + if !req.scripts.is_empty() { + let paths: Vec = req.scripts.keys().cloned().collect(); + let hashes: Vec = paths.iter().map(|p| req.scripts[p].clone()).collect(); + + let matching: Vec = sqlx::query_scalar( + "SELECT local.path FROM \ + unnest($1::text[], $2::text[]) AS local(path, hash) \ + INNER JOIN LATERAL ( \ + SELECT encode(sha256(convert_to(s.content, 'UTF8')), 'hex') AS deployed_hash \ + FROM script s \ + WHERE s.path = local.path AND s.workspace_id = $3 AND s.archived = false \ + ORDER BY s.created_at DESC LIMIT 1 \ + ) deployed ON deployed.deployed_hash = local.hash" + ) + .bind(&paths) + .bind(&hashes) + .bind(&w_id) + .fetch_all(&db) + .await?; + + matching_set.extend(matching); + all_paths.extend(paths); + } + + // --- Workspace dependencies --- + for dep in &req.workspace_deps { + let matching: Option = sqlx::query_scalar( + "SELECT $1::text \ + WHERE EXISTS ( \ + SELECT 1 FROM workspace_dependencies wd \ + WHERE wd.workspace_id = $2 AND wd.archived = false \ + AND wd.language = $3::SCRIPT_LANG \ + AND wd.name IS NOT DISTINCT FROM $4 \ + AND encode(sha256(convert_to(wd.content, 'UTF8')), 'hex') = $5 \ + )" + ) + .bind(&dep.path) + .bind(&w_id) + .bind(dep.language.as_str()) + .bind(&dep.name) + .bind(&dep.hash) + .fetch_optional(&db) + .await?; + + if let Some(path) = matching { + matching_set.insert(path); + } + all_paths.push(dep.path.clone()); + } + + let mismatched: Vec = all_paths + .into_iter() + .filter(|p| !matching_set.contains(p)) + .collect(); + + Ok(Json(mismatched)) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7d15eea003..930cb94949 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6988,6 +6988,83 @@ paths: type: string format: uuid + /w/{workspace}/scripts/raw_temp/store: + post: + summary: store raw script content temporarily for CLI lock generation + operationId: storeRawScriptTemp + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: script content to store + required: true + content: + application/json: + schema: + type: string + responses: + "200": + description: hash of stored content + content: + application/json: + schema: + type: string + + /w/{workspace}/scripts/raw_temp/diff: + post: + summary: diff local script hashes against deployed versions + operationId: diffRawScriptsWithDeployed + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: scripts and workspace deps to diff against deployed versions + required: true + content: + application/json: + schema: + type: object + required: + - scripts + properties: + scripts: + description: map of script path to SHA256 content hash + type: object + additionalProperties: + type: string + workspace_deps: + description: workspace dependencies to diff + type: array + items: + type: object + required: + - path + - language + - hash + properties: + path: + description: CLI path (e.g. dependencies/package.json) + type: string + language: + $ref: "#/components/schemas/ScriptLang" + name: + description: named workspace dependency (null for default) + type: string + hash: + description: SHA256 content hash + type: string + responses: + "200": + description: list of paths that differ from deployed versions + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/jobs/list_selected_job_groups: # We use post because sending a huge array as a query param can produce # URLs that may be too long diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 29b9765480..7f9a3f36db 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -5074,6 +5074,10 @@ pub struct RunDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option, + /// Map of script path -> content hash for resolving imports from temp storage. + /// Used by CLI to provide local script content during lock generation. + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Deserialize, Clone, Debug)] @@ -5133,6 +5137,8 @@ async fn run_dependencies_job( let mut hm = HashMap::new(); req.raw_workspace_dependencies .map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v))); + req.temp_script_refs + .map(|v| hm.insert("temp_script_refs".to_owned(), to_raw_value(&v))); let (uuid, tx) = push( &db, @@ -5183,6 +5189,8 @@ pub struct RunFlowDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option>, + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Serialize)] @@ -5226,6 +5234,10 @@ async fn run_flow_dependencies_job( req.raw_workspace_dependencies .map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v))); + // Add temp_script_refs to args if present (for CLI local import resolution) + req.temp_script_refs + .map(|v| args_map.insert("temp_script_refs".to_string(), to_raw_value(&v))); + let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 8901055889..eae1648657 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -991,6 +991,38 @@ pub mod workspace_dependencies { } } +/// Temporary raw script content cache for CLI lock generation. +pub mod raw_script_temp { + use super::*; + use crate::DB; + + make_static! { + static ref CACHE: { String => String } in "raw_script_temp" <= 10000; + } + + /// Compute hash for raw script content (includes workspace_id for isolation). + pub fn compute_hash(workspace_id: &str, content: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(workspace_id.as_bytes()); + hasher.update(content.as_bytes()); + format!("{:x}", hasher.finalize()) + } + + /// Load content from cache, falling back to DB. + pub fn load(hash: String, db: &DB) -> impl Future> + '_ { + CACHE.get_or_insert_async(hash.clone(), async move { + sqlx::query_scalar!( + "SELECT content FROM raw_script_temp WHERE hash = $1", + &hash + ) + .fetch_optional(db) + .await? + .ok_or_else(|| error::Error::NotFound(format!("raw_script_temp hash: {}", hash))) + }) + } +} + const _: () = { impl Import for RawFlow { fn import(src: &impl Storage) -> error::Result { @@ -1183,7 +1215,8 @@ const _: () = { ((u8, ScriptHash), |x| format!("{:02x}-{:016x}", x.0, x.1.0)), (FlowNodeId, |x| format!("{:016x}", x.0)), (AppScriptId, |x| format!("{:016x}", x.0)), - ((i64, String), |x| format!("{}-{}", x.1, x.0)) + ((i64, String), |x| format!("{}-{}", x.1, x.0)), + (String, |x| x.as_str()) } #[cfg(feature = "scoped_cache")] diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml index 9619a10ea7..2472d6bf02 100644 --- a/backend/windmill-types/Cargo.toml +++ b/backend/windmill-types/Cargo.toml @@ -9,6 +9,7 @@ name = "windmill_types" path = "src/lib.rs" [dependencies] +windmill-parser.workspace = true serde.workspace = true serde_json.workspace = true chrono.workspace = true diff --git a/backend/windmill-types/src/s3.rs b/backend/windmill-types/src/s3.rs index e3cbd1f9a3..0ab1794d7e 100644 --- a/backend/windmill-types/src/s3.rs +++ b/backend/windmill-types/src/s3.rs @@ -346,20 +346,9 @@ pub struct DuckdbConnectionSettingsQueryV2 { pub storage: Option, } -#[derive(Clone, Copy, Debug)] -pub enum S3ModeFormat { - Json, - Csv, - Parquet, -} - -pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { - match format { - S3ModeFormat::Json => "json", - S3ModeFormat::Csv => "csv", - S3ModeFormat::Parquet => "parquet", - } -} +// Re-export from windmill-parser to keep a single type definition +// (windmill-parser is WASM-compatible, windmill-types is not due to sqlx) +pub use windmill_parser::{s3_mode_extension, S3ModeFormat}; #[cfg(test)] mod tests { diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index f64e6d769b..f2a00de0cf 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -1,8 +1,11 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + const p = { name: "windmill-relative-resolver", async setup(build) { const { writeFileSync, readFileSync, mkdirSync } = await import("fs"); - const { dirname, resolve } = await import("node:path"); + const { dirname, resolve, join } = await import("node:path"); const base_internal_url = "BASE_INTERNAL_URL".replace( "localhost", @@ -95,11 +98,17 @@ const p = { : args.importer.replace(cdir + "/", ""); const isRelative = !args.path.startsWith("/"); + const endExt = args.path.endsWith(".ts") ? "" : ".ts"; + const pathNoExt = args.path.replace(/\.ts$/, ""); - let endExt = args.path.endsWith(".ts") ? "" : ".ts"; - const url = isRelative + // Lookup temp script hash + const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); + const hash = TEMP_SCRIPT_REFS?.[normalized]; + + const url = (isRelative ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}`; + : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") : resolve("./" + args.path + ".url"); diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index fedef5fc5a..7d14ffcbc3 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -1,3 +1,6 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + // Windows-specific bun loader that uses a virtual "windmill-url" namespace instead // of writing .url files to disk. This avoids Windows path issues (backslashes in // resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace @@ -70,7 +73,13 @@ const p = { const rawScriptPath = isAbsolute ? `${path}${endExt}` : `${importerPath}/../${path}${endExt}`; - return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + const normalized = normalizePath(rawScriptPath); + // Look up temp script hash (keys are extensionless paths) + const lookupPath = normalized.replace(/\.ts$/, ""); + const hash = TEMP_SCRIPT_REFS?.[lookupPath]; + // Encode hash in the path so onLoad can extract it and append to fetch URL + const resolvedPath = hash ? `${normalized}?temp_script_hash=${hash}` : normalized; + return { path: resolvedPath, namespace: "windmill-url" }; } build.onLoad({ filter: filterLoad }, async (args) => { @@ -80,8 +89,13 @@ const p = { // Load windmill scripts by fetching from the API build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { - const path = args.path.replace(/^windmill-url:/, ""); - const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + // Extract temp_script_hash if embedded in the path by resolveWindmillImport + const [scriptPath, queryString] = args.path.replace(/^windmill-url:/, "").split("?"); + const hashParam = queryString?.startsWith("temp_script_hash=") + ? queryString.replace("temp_script_hash=", "") + : undefined; + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${scriptPath}` + + (hashParam ? `?temp_script_hash=${hashParam}` : ""); const req = await fetch(url, { method: "GET", headers: { @@ -124,7 +138,8 @@ const p = { // Resolve nested imports from within windmill-url modules build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { - const importer = args.importer.replace(/^windmill-url:/, ""); + // Strip any query string from the importer path before resolving + const importer = args.importer.replace(/^windmill-url:/, "").split("?")[0]; return resolveWindmillImport(importer, args.path); }); }, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index c3065be7d2..fb07806fd6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -206,6 +206,7 @@ pub async fn gen_bun_lockfile( workspace_dependencies: &WorkspaceDependenciesPrefetched, npm_mode: bool, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, quiet: bool, ) -> Result> { let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; @@ -216,6 +217,11 @@ pub async fn gen_bun_lockfile( gen_bunfig(job_dir, job_id, w_id, db).await?; write_file(job_dir, "package.json", package_json_content.as_str())?; } else { + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -224,7 +230,8 @@ pub async fn gen_bun_lockfile( "CURRENT_PATH", &crate::common::use_flow_root_path(script_path), ) - .replace("RAW_GET_ENDPOINT", "raw"); + .replace("RAW_GET_ENDPOINT", "raw") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); write_file( &job_dir, @@ -615,9 +622,15 @@ pub async fn build_loader( w_id: &str, current_path: &str, mode: LoaderMode, + temp_script_refs: &Option>, ) -> Result<()> { // Use forward slashes in JS strings to avoid backslash escape issues on Windows let job_dir_js = job_dir.replace('\\', "/"); + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -626,7 +639,8 @@ pub async fn build_loader( "CURRENT_PATH", &crate::common::use_flow_root_path(current_path), ) - .replace("RAW_GET_ENDPOINT", "raw_unpinned"); + .replace("RAW_GET_ENDPOINT", "raw_unpinned") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); if mode == LoaderMode::Node { write_file( @@ -924,6 +938,7 @@ pub async fn prebundle_bun_script( worker_name: &str, token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; @@ -950,6 +965,7 @@ pub async fn prebundle_bun_script( } else { LoaderMode::BunBundle }, + temp_script_refs, ) .await?; @@ -1271,6 +1287,7 @@ pub async fn handle_bun_job( workspace_dependencies, annotation.npm, &mut Some(occupancy_metrics), + &None, wac_replay_info.is_some(), ) .await?; @@ -1639,6 +1656,7 @@ try {{ } else { LoaderMode::BunBundle }, + &None, ) .await?; @@ -1655,6 +1673,7 @@ try {{ } else { LoaderMode::Bun }, + &None, ) .await } else { @@ -3358,6 +3377,7 @@ pub async fn start_worker( w_id, script_path, LoaderMode::BrowserBundle, + &None, ) .await?; generate_bun_bundle( @@ -3470,6 +3490,7 @@ pub async fn start_worker( .await?, annotation.npm, &mut None, + &None, false, ) .await?; @@ -3544,6 +3565,7 @@ pub async fn start_worker( } else { LoaderMode::Bun }, + &None, ) .await?; } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3aa7580220..3b6507a7e0 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1348,6 +1348,7 @@ async fn handle_python_deps( &mut version_specifiers, &mut locked_v, &None, + &None, // temp_script_refs: only used during CLI lock generation )) .await?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index daba455f1c..93b48cf818 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -139,6 +139,13 @@ pub async fn handle_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let content = capture_dependency_job( &job.id, job.script_lang.as_ref().map(|v| Ok(v)).unwrap_or_else(|| { @@ -164,6 +171,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &temp_script_refs, ) .await; @@ -210,6 +218,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &None, ) .await { @@ -368,6 +377,13 @@ pub async fn handle_flow_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let version = if skip_flow_update { None } else { @@ -467,6 +483,7 @@ pub async fn handle_flow_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -681,6 +698,7 @@ async fn lock_flow_value<'c>( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( FlowValue, sqlx::Transaction<'c, sqlx::Postgres>, @@ -711,6 +729,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -742,6 +761,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -779,6 +799,7 @@ async fn lock_flow_value<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -817,6 +838,7 @@ async fn lock_modules<'c>( dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( Vec, sqlx::Transaction<'c, sqlx::Postgres>, @@ -872,6 +894,7 @@ async fn lock_modules<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -911,6 +934,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -942,6 +966,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -978,6 +1003,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1008,6 +1034,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; errors.extend(ninner_errors); @@ -1078,6 +1105,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; @@ -1179,6 +1207,7 @@ async fn lock_modules<'c>( job_path, Some(&e.id), "flow", + &temp_script_refs, ) .await; // @@ -1586,6 +1615,7 @@ async fn lock_modules_app( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result { match value { Value::Object(mut m) => { @@ -1696,6 +1726,7 @@ async fn lock_modules_app( &job.runnable_path(), container_id.as_deref(), "app", + temp_script_refs, ) .await; match new_lock { @@ -1773,6 +1804,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1801,6 +1833,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1852,6 +1885,13 @@ pub async fn handle_app_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + sqlx::query!( "DELETE FROM workspace_runnable_dependencies WHERE app_path = $1 AND workspace_id = $2", job_path, @@ -1899,6 +1939,7 @@ pub async fn handle_app_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -2398,6 +2439,8 @@ async fn capture_dependency_job( base_path: &str, step_id: Option<&str>, runnable_type: &str, // "script", "flow", or "app" + // Map of script path -> content hash for resolving imports from temp storage (CLI). + temp_script_refs: &Option>, ) -> error::Result { // Check if we can skip relocking: // - Must be triggered by relative import @@ -2456,12 +2499,13 @@ async fn capture_dependency_job( let (mut version_specifiers, mut locked_v) = (vec![], None); let reqs = windmill_parser_py_imports::parse_python_imports( job_raw_code, - &w_id, + w_id, script_path, &db, &mut version_specifiers, &mut locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .0 @@ -2585,6 +2629,7 @@ async fn capture_dependency_job( &workspace_dependencies, windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm, &mut Some(occupancy_metrics), + temp_script_refs, false, ) .await? @@ -2602,6 +2647,7 @@ async fn capture_dependency_job( worker_name, &token, &mut Some(occupancy_metrics), + temp_script_refs, ) .await?; } diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 72be8f1ca9..865091104c 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -12,6 +12,7 @@ const parserPackages = [ "windmill-parser-wasm-yaml", "windmill-parser-wasm-csharp", "windmill-parser-wasm-nu", "windmill-parser-wasm-java", "windmill-parser-wasm-ruby", + "windmill-parser-wasm-py-imports", ]; const parserExternals = parserPackages.flatMap(p => ["--external", p]); diff --git a/cli/bun.lock b/cli/bun.lock index e400420a63..6c421141b4 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -24,10 +24,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", @@ -294,13 +295,15 @@ "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="], + "windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.659.1", "", {}, "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="], + "windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.639.0", "", {}, "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="], "windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="], "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="], - "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.623.1", "", {}, "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="], + "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="], "windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="], diff --git a/cli/package.json b/cli/package.json index 6102f815f2..e44a215631 100644 --- a/cli/package.json +++ b/cli/package.json @@ -32,10 +32,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 253dc99f3b..82e2a1ac22 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts"; import { stringify as yamlStringify } from "yaml"; import { GlobalOptions } from "../../types.ts"; import { + readLockfile, checkifMetadataUptodate, blueColor, clearGlobalLock, @@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; import { getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__app_hash"; export const APP_BACKEND_FOLDER = "backend"; @@ -113,7 +116,9 @@ export async function generateAppLocksInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (appFolder.endsWith(SEP)) { appFolder = appFolder.substring(0, appFolder.length - 1); @@ -125,9 +130,6 @@ export async function generateAppLocksInternal( log.info(`Generating locks for app ${appFolder} at ${remote_path}`); } - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - // Read the app file first to filter workspace dependencies const appFilePath = path.join( appFolder, @@ -135,35 +137,80 @@ export async function generateAppLocksInternal( ); const appFile = (await yamlParseFile(appFilePath)) as AppFile; - // Filter workspace dependencies based on inline scripts' languages and annotations const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value; - const filteredDeps = await filterWorkspaceDependenciesForApp( - appValue, - rawWorkspaceDependencies, - appFolder - ); + const folderNormalized = appFolder.replaceAll(SEP, "/"); - let hashes = await generateAppHash( - filteredDeps, - appFolder, - rawApp, - opts.defaultTs - ); + let filteredDeps: Record = {}; + const conf = await readLockfile(); - const conf = await import("../../utils/metadata.ts").then((m) => - m.readLockfile() - ); - if ( - await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH) - ) { - if (!noStaleMessage) { - log.info( - colors.green(`App ${remote_path} metadata is up-to-date, skipping`) - ); + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + // For raw apps in new format, runnables are in separate files under backend/ + let treeAppValue = structuredClone(appValue); + if (rawApp) { + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER); + const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath); + if (Object.keys(runnablesFromFiles).length > 0) { + treeAppValue = runnablesFromFiles; + } + } + + // First pass: add inline scripts as separate nodes, then add app node importing them + const inlineScriptPaths: string[] = []; + await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => { + if (!inlineScript.content || !inlineScript.language) { + return inlineScript; + } + + let content = inlineScript.content; + // Resolve !inline references + if (typeof content === "string" && content.startsWith("!inline ")) { + const filePath = appFolder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + return inlineScript; + } + } + + const treePath = folderNormalized + "/" + context.path.join("/"); + const language = inlineScript.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false); + inlineScriptPaths.push(treePath); + + return inlineScript; + }); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp); + return; + } + // Second pass: get mismatched workspace deps from tree + // TODO: pass raw workspace deps more precisely to every inline script lock generation call + // (currently we pass the union of all mismatched deps filtered for the whole app) + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder); + } else { + // Legacy behaviour + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder); + + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`App ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -179,6 +226,8 @@ export async function generateAppLocksInternal( let updatedScripts: string[] = []; if (!justUpdateMetadataLock) { + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const changedScripts = []; // Find hashes that do not correspond to previous hashes for (const [scriptPath, hash] of Object.entries(hashes)) { @@ -190,7 +239,13 @@ export async function generateAppLocksInternal( } } - if (changedScripts.length > 0) { + // Get temp_script_refs from tree for relative import resolution + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); + + // In tree mode, the tree already verified this app is stale (possibly via dependency change). + // Per-script hashes only detect content changes, not transitive dependency changes, + // so we must regenerate locks for all inline scripts regardless. + if (changedScripts.length > 0 || (tree && !legacyBehaviour)) { if (!noStaleMessage) { log.info( `Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}` @@ -219,7 +274,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); // Note: updateRawAppRunnables now writes each runnable to its own file } else { @@ -236,7 +292,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); normalAppFile.value = result.value; updatedScripts = result.updatedScripts; @@ -252,15 +309,16 @@ export async function generateAppLocksInternal( } } - // Regenerate hashes after updates - hashes = await generateAppHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateAppHash( + depsForHash, appFolder, rawApp, opts.defaultTs ); await clearGlobalLock(appFolder); - for (const [scriptPath, hash] of Object.entries(hashes)) { + for (const [scriptPath, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(appFolder, hash, scriptPath); } if (!noStaleMessage) { @@ -366,7 +424,8 @@ async function updateRawAppRunnables( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise { const updatedRunnables: string[] = []; const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); @@ -446,7 +505,8 @@ async function updateRawAppRunnables( content, language, `${remotePath}/${runnableId}`, - rawDeps + rawDeps, + tempScriptRefs ); // Determine file extension for this language @@ -513,7 +573,8 @@ async function updateAppInlineScripts( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise<{ value: any; updatedScripts: string[] }> { const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }); const updatedScripts: string[] = []; @@ -561,7 +622,8 @@ async function updateAppInlineScripts( content, language, scriptPath, - rawDeps + rawDeps, + tempScriptRefs ); } // Determine file extension for this language (following extractInlineScriptsForApps pattern) @@ -626,7 +688,8 @@ async function generateInlineScriptLock( content: string, language: string, scriptPath: string, - rawWorkspaceDependencies: Record | undefined + rawWorkspaceDependencies: Record | undefined, + tempScriptRefs?: Record ): Promise { // Filter workspace dependencies to only include those matching this script's language and annotations const filteredDeps = rawWorkspaceDependencies @@ -657,6 +720,9 @@ async function generateInlineScriptLock( ? filteredDeps : null, entrypoint: scriptPath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 4d92405d01..58d0777f90 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -437,6 +437,7 @@ async function preview( export async function generateLocks( opts: GlobalOptions & { yes?: boolean; + dryRun?: boolean; } & SyncOptions, folder: string | undefined ) { @@ -487,6 +488,10 @@ export async function generateLocks( } if (hasAny) { + if (opts.dryRun) { + log.info(colors.gray("Dry run complete.")); + return; + } if ( !opts.yes && !(await Confirm.prompt({ @@ -592,6 +597,7 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") + .option("--dry-run", "Perform a dry run without making changes") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 6a9040094f..1f5d86b8ea 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -29,10 +29,9 @@ import { FlowFile } from "./flow.ts"; import { FlowValue } from "../../../gen/types.gen.ts"; import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; import { workspaceDependenciesLanguages } from "../../utils/script_common.ts"; -import { - extractNameFromFolder, - getNonDottedPaths, -} from "../../utils/resource_folders.ts"; +import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__flow_hash"; async function generateFlowHash( @@ -70,7 +69,9 @@ export async function generateFlowLockInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); @@ -80,33 +81,67 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - // Always get out-of-sync workspace dependencies - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - const flowValue = (await yamlParseFile( folder! + SEP + "flow.yaml" )) as FlowFile; - // Filter workspace dependencies based on inline scripts' languages and annotations - const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); - - let hashes = await generateFlowHash( - filteredDeps, - folder, + const folderNormalized = folder.replaceAll(SEP, "/"); + const inlineScriptsForTree = extractInlineScriptsForFlows( + structuredClone(flowValue.value.modules), + {}, + SEP, opts.defaultTs - ); + ).filter(s => !s.is_lock); + let filteredDeps: Record = {}; const conf = await readLockfile(); - if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { - if (!noStaleMessage) { - log.info( - colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) - ); + + if (!legacyBehaviour && tree) { + if (dryRun) { + const inlineScriptPaths: string[] = []; + for (const script of inlineScriptsForTree) { + let content = script.content; + if (content.startsWith("!inline ")) { + const filePath = folder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + continue; + } + } + + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + const language = script.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false); + inlineScriptPaths.push(treePath); + } + + const hashes = await generateFlowHash({}, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale); + return; + } + // Second pass: get mismatched workspace deps from tree + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder); + } else { + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); + + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -122,7 +157,23 @@ export async function generateFlowLockInternal( let changedScripts: string[] = []; + // Build mapping from on-disk file names (hash keys like "a.py") to tree paths + // (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without + // a path assigner, so paths always have .inline_script suffix, but on-disk files + // may not (non-dotted mode). + const fileToTreePath = new Map(); + for (const script of inlineScriptsForTree) { + const c = script.content; + if (c.startsWith("!inline ")) { + const fileName = c.replace("!inline ", ""); + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + fileToTreePath.set(fileName, treePath); + } + } + if (!justUpdateMetadataLock) { + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + //find hashes that do not correspond to previous hashes for (const [path, hash] of Object.entries(hashes)) { if (path == TOP_HASH) { @@ -137,27 +188,39 @@ export async function generateFlowLockInternal( log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`); } const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8"); + // In tree mode, use the tree's staleness info (which includes transitive dependency changes) + // to determine which scripts need relocking, instead of only content-changed ones. + const locksToRemove = (tree && !legacyBehaviour) + ? Object.keys(hashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; await replaceInlineScripts( flowValue.value.modules, fileReader, log, folder + SEP!, SEP, - changedScripts + locksToRemove ); if (flowValue.value.failure_module) { - await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } if (flowValue.value.preprocessor_module) { - await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } //removeChangedLocks + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); flowValue.value = await updateFlow( workspace, flowValue.value, remote_path, - filteredDeps + filteredDeps, + tempScriptRefs ); const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", { @@ -187,13 +250,15 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateFlowHash( + depsForHash, folder, opts.defaultTs ); await clearGlobalLock(folder); - for (const [path, hash] of Object.entries(hashes)) { + for (const [path, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(folder, hash, path); } if (!noStaleMessage) { @@ -201,7 +266,16 @@ export async function generateFlowLockInternal( } // Return the list of updated scripts (extract just the filename from the path) - const updatedScripts = changedScripts.map(p => { + // In tree mode, use the same staleness-aware list we used for lock removal + const relocked = (tree && !legacyBehaviour) + ? Object.keys(finalHashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; + const updatedScripts = relocked.map(p => { const parts = p.split(SEP); return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension }); @@ -239,7 +313,8 @@ export async function updateFlow( workspace: Workspace, flow_value: FlowValue, remotePath: string, - rawWorkspaceDependencies: Record + rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { let rawResponse; @@ -264,6 +339,9 @@ export async function updateFlow( path: remotePath, use_local_lockfiles: true, raw_workspace_dependencies: rawWorkspaceDependencies, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); @@ -282,6 +360,9 @@ export async function updateFlow( body: JSON.stringify({ flow_value, path: remotePath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index c523996cdc..d273f4b631 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -10,6 +10,8 @@ import * as log from "../../core/log.ts"; import { generateScriptMetadataInternal, getRawWorkspaceDependencies, + readLockfile, + checkifMetadataUptodate, } from "../../utils/metadata.ts"; import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts"; import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts"; @@ -19,14 +21,20 @@ import { ignoreF, } from "../sync/sync.ts"; import { exts } from "../script/script.ts"; -import { isFlowPath, isAppPath, isRawAppPath, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; +import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; import { listSyncCodebases } from "../../utils/codebase.ts"; +import { + DoubleLinkedDependencyTree, + uploadScripts, + ItemType, +} from "../../utils/dependency_tree.ts"; interface StaleItem { - type: "script" | "flow" | "app"; + type: ItemType; path: string; folder: string; isRawApp?: boolean; + staleReason?: string; } async function generateMetadata( @@ -38,6 +46,7 @@ async function generateMetadata( skipScripts?: boolean; skipFlows?: boolean; skipApps?: boolean; + strictFolderBoundaries?: boolean; } & SyncOptions, folder?: string ) { @@ -49,12 +58,10 @@ async function generateMetadata( await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false); const codebases = await listSyncCodebases(opts); const ignore = await ignoreF(opts); - const staleItems: StaleItem[] = []; - // --schema-only implies skipping flows and apps (they only have locks, no schemas) const skipScripts = opts.skipScripts ?? false; const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false; @@ -70,7 +77,11 @@ async function generateMetadata( return; } - log.info(colors.gray(`Checking ${checking.join(", ")}...`)); + log.info(`Checking ${checking.join(", ")}...`); + + // Build dependency tree for relative import tracking + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); // === Collect stale scripts === if (!skipScripts) { @@ -81,9 +92,7 @@ async function generateMetadata( return ( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || - isFlowPath(p) || - isAppPath(p) || - isRawAppPath(p) || + isFolderResourcePathAnyFormat(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)) ); }, @@ -92,19 +101,18 @@ async function generateMetadata( ); for (const e of Object.keys(scriptElems)) { - const candidate = await generateScriptMetadataInternal( + await generateScriptMetadataInternal( e, workspace, opts, - true, // dryRun + true, // dryRun - populate tree true, // noStaleMessage rawWorkspaceDependencies, codebases, - false + false, + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "script", path: candidate, folder: e }); - } } } @@ -126,18 +134,17 @@ async function generateMetadata( ) ).map((x) => x.substring(0, x.lastIndexOf(SEP))); - for (const folder of flowElems) { - const candidate = await generateFlowLockInternal( - folder, - true, // dryRun + for (const flowFolder of flowElems) { + await generateFlowLockInternal( + flowFolder, + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "flow", path: candidate, folder }); - } } } @@ -161,33 +168,74 @@ async function generateMetadata( const appFolders = getAppFolders(elems, "app.yaml"); for (const appFolder of rawAppFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, true, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: true }); - } } for (const appFolder of appFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, false, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: false }); - } + } + } + + // === Propagate staleness through imports === + tree.propagateStaleness(); + + // Upload stale scripts to temp storage so the backend can resolve relative imports. + // If this fails (e.g. backend is older and doesn't have /raw_temp endpoints), + // degrade gracefully: locks will be generated using deployed script content only. + try { + await uploadScripts(tree, workspace); + } catch (e) { + log.warn(colors.yellow( + `Failed to upload scripts to temp storage (backend may be too old): ${e}. ` + + `Locks will be generated using deployed script versions only — locally modified ` + + `relative imports may not be reflected.` + )); + } + + // === Populate staleItems from tree === + const staleItems: StaleItem[] = []; + const seenFolders = new Set(); + + for (const p of tree.allPaths()) { + const staleReason = tree.getStaleReason(p); + if (!staleReason) continue; + + const itemType = tree.getItemType(p)!; + const itemFolder = tree.getFolder(p)!; + + if (itemType === "dependencies") { + staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason }); + } else if (itemType === "inline_script") { + // Inline scripts are not listed separately — their parent flow/app is stale via propagation + continue; + } else if (itemType === "script") { + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason }); + } else if (!seenFolders.has(itemFolder)) { + // Flows/Apps: one entry per folder (dedupe multiple inline scripts) + seenFolders.add(itemFolder); + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason }); } } @@ -200,11 +248,54 @@ async function generateMetadata( if (folder.endsWith("/")) { folder = folder.substring(0, folder.length - 1); } - // Normalize item.folder for comparison (Windows file paths use backslashes) - filteredItems = staleItems.filter((item) => { + // Strip file extension if user passed a specific file path (e.g. f/test/script.ts) + const folderNoExt = folder.replace(/\.[^/.]+$/, ""); + // Check if an item is inside the specified folder + const isInsideFolder = (item: StaleItem) => { const normalizedFolder = item.folder.replaceAll("\\", "/"); - return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/"); - }); + const normalizedPath = item.path.replaceAll("\\", "/"); + return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/") + || normalizedPath === folder || normalizedPath === folderNoExt; + }; + const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt; + // Check if a tree path or any of its transitive deps is inside the folder + const touchesFolder = (treePath: string) => { + if (isPathInFolder(treePath)) return true; + let found = false; + tree.traverseTransitive(treePath, (importPath) => { + if (isPathInFolder(importPath)) { + found = true; + return true; // stop early + } + }); + return found; + }; + + const isRelevant = (item: StaleItem) => { + if (isInsideFolder(item)) return true; + if (item.type === "dependencies") return true; + const treePath = (item.type === "script" + ? item.path.replace(/\.[^/.]+$/, "") + : item.folder).replaceAll("\\", "/"); + return touchesFolder(treePath); + }; + + if (opts.strictFolderBoundaries) { + // Strict mode: only items inside the folder + filteredItems = staleItems.filter(isInsideFolder); + + // Warn about stale items outside the folder that would be included by default + const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies"); + for (const item of excludedStale) { + const normalizedPath = item.path.replaceAll("\\", "/"); + log.warn(colors.yellow( + `Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.` + )); + } + } else { + // Default: include items inside the folder and any stale importers that transitively depend on it + filteredItems = staleItems.filter(isRelevant); + } } // === Show stale items and confirm === @@ -217,28 +308,24 @@ async function generateMetadata( const scripts = filteredItems.filter((i) => i.type === "script"); const flows = filteredItems.filter((i) => i.type === "flow"); const apps = filteredItems.filter((i) => i.type === "app"); + const deps = filteredItems.filter((i) => i.type === "dependencies"); log.info(""); - log.info(`Found ${filteredItems.length} item(s) with stale metadata:`); + log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`); - if (scripts.length > 0) { - log.info(colors.gray(` Scripts (${scripts.length}):`)); - for (const item of scripts) { - log.info(colors.yellow(` ${item.path}`)); + const printItems = (label: string, items: StaleItem[]) => { + if (items.length === 0) return; + log.info(` ${label} (${items.length}):`); + for (const item of items) { + const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : ""; + log.info(` ~ ${item.path}` + reason); } - } - if (flows.length > 0) { - log.info(colors.gray(` Flows (${flows.length}):`)); - for (const item of flows) { - log.info(colors.yellow(` ${item.path}`)); - } - } - if (apps.length > 0) { - log.info(colors.gray(` Apps (${apps.length}):`)); - for (const item of apps) { - log.info(colors.yellow(` ${item.path}`)); - } - } + }; + + printItems("Workspace dependencies", deps); + printItems("Scripts", scripts); + printItems("Flows", flows); + printItems("Apps", apps); if (opts.dryRun) { return; @@ -259,28 +346,30 @@ async function generateMetadata( log.info(""); // === Process all stale items with progress counter === - const total = filteredItems.length; + const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps(); + const total = filteredItems.length - deps.length; const maxWidth = `[${total}/${total}]`.length; let current = 0; const formatProgress = (n: number) => { - const bracket = `[${n}/${total}]`; - return colors.gray(bracket.padEnd(maxWidth, " ")); + return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " "))); }; // Process scripts for (const item of scripts) { current++; - log.info(`${formatProgress(current)} script ${colors.cyan(item.path)}`); + log.info(`${formatProgress(current)} script ${item.path}`); await generateScriptMetadataInternal( - item.folder, + item.path, // originalPath with extension workspace, opts, false, // dryRun - true, // noStaleMessage - we handle output - rawWorkspaceDependencies, + true, // noStaleMessage + mismatchedWorkspaceDeps, codebases, - false + false, + false, // legacyBehaviour + tree ); } @@ -288,38 +377,49 @@ async function generateMetadata( for (const item of flows) { current++; const result = await generateFlowLockInternal( - item.folder, + item.folder.replaceAll("/", SEP), false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as FlowLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const flowResult = result as FlowLocksResult | undefined; + const scriptsInfo = flowResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} flow ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`); } + // Process apps for (const item of apps) { current++; const result = await generateAppLocksInternal( - item.folder, + item.folder.replaceAll("/", SEP), item.isRawApp!, // rawApp false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as AppLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const appResult = result as AppLocksResult | undefined; + const scriptsInfo = appResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} app ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`); } + // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped) + const allStaleDeps = staleItems.filter((i) => i.type === "dependencies"); + await tree.persistDepsHashes(allStaleDeps.map((d) => d.path)); + log.info(""); - log.info(colors.green(`Done. Updated ${total} item(s).`)); + log.info(`Done. Updated ${colors.bold(String(total))} item(s).`); } const command = new Command() @@ -332,6 +432,7 @@ const command = new Command() .option("--skip-scripts", "Skip processing scripts") .option("--skip-flows", "Skip processing flows") .option("--skip-apps", "Skip processing apps") + .option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)") .option( "-i --includes ", "Comma separated patterns to specify which files to include" diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 68ff946cee..14f3155ff7 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -130,7 +130,7 @@ async function push(opts: PushOptions, filePath: string) { [], undefined, opts, - await getRawWorkspaceDependencies(), + await getRawWorkspaceDependencies(true), codebases ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); @@ -1161,7 +1161,7 @@ export async function generateMetadata( opts = await mergeConfigWithConfigFile(opts); const codebases = await listSyncCodebases(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); if (scriptPath) { // read script metadata file await generateScriptMetadataInternal( diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index bc0eee2275..d62c39cba9 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2280,7 +2280,7 @@ export async function pull( const tracker: ChangeTracker = await buildTracker(changes); const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); + await getRawWorkspaceDependencies(true); for (const change of tracker.scripts) { await generateScriptMetadataInternal( @@ -2611,7 +2611,7 @@ export async function push( false, // els1 (local) is not the remote source ); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); const tracker: ChangeTracker = await buildTracker(changes); @@ -2657,7 +2657,7 @@ export async function push( true, ); if (stale) { - staleFlows.push(stale); + staleFlows.push(stale as string); } } @@ -2682,7 +2682,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } @@ -2697,7 +2697,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts new file mode 100644 index 0000000000..5e2adf7453 --- /dev/null +++ b/cli/src/utils/dependency_tree.ts @@ -0,0 +1,373 @@ +/** + * Double-linked dependency tree for tracking script imports and propagating staleness. + */ + +import { Workspace } from "../commands/workspace/workspace.ts"; +import * as wmill from "../../gen/services.gen.ts"; +import type { ScriptLang } from "../../gen/types.gen.ts"; +import { ScriptLanguage } from "./script_common.ts"; +import { + filterWorkspaceDependencies, + generateScriptHash, + checkifMetadataUptodate, + workspaceDependenciesPathToLanguageAndFilename, + updateMetadataGlobalLock, +} from "./metadata.ts"; +import { generateHash } from "./utils.ts"; + +/** + * Diff local scripts against deployed versions, upload only those that differ. + * Only uploaded (mismatched) scripts get contentHash set, so flatten() returns + * temp_script_refs only for scripts the backend can't resolve from deployed versions. + */ +export async function uploadScripts( + tree: DoubleLinkedDependencyTree, + workspace: Workspace +): Promise { + // Split into scripts vs workspace deps and compute SHA256(content) for each + const scriptHashes: Record = {}; + const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = []; + + for (const path of tree.allPaths()) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Empty string is valid for workspace deps (means "no deps") — only skip undefined + if (content === undefined) continue; + const info = workspaceDependenciesPathToLanguageAndFilename(path); + if (info) { + const hash = await generateHash(content); + workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash }); + } + } else if (itemType === "script") { + if (!content) continue; + const hash = await generateHash(content); + scriptHashes[path] = hash; + } + // Skip inline_script, flow, app — they don't need temp storage uploads + } + + if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return; + + // Single batch query: find which scripts/deps differ from deployed versions + const mismatched = await wmill.diffRawScriptsWithDeployed({ + workspace: workspace.workspaceId, + requestBody: { + scripts: scriptHashes, + workspace_deps: workspaceDeps, + }, + }); + + // Upload only mismatched scripts to temp storage + for (const path of mismatched) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Workspace deps don't need temp storage — just mark as mismatched. + // Empty string is valid (means the dep file was emptied locally). + if (content !== undefined) { + tree.setContentHash(path, "mismatched"); + } + } else if (content) { + const hash = await wmill.storeRawScriptTemp({ + workspace: workspace.workspaceId, + requestBody: content, + }); + tree.setContentHash(path, hash); + } + } +} + +export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies"; + +interface DependencyNode { + content: string; + stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata) + contentHash?: string; // Hash for temp storage lookup (content only) + language: ScriptLanguage; + metadata: string; + imports: Set; + importedBy: Set; + staleReason?: string; + // Item metadata for generate-metadata command + itemType: ItemType; + folder: string; // Folder path (for flows/apps) or remote path (for scripts) + originalPath: string; // Original path passed to handler (with extension for scripts) + isRawApp?: boolean; // Only set for apps + isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) +} + +export class DoubleLinkedDependencyTree { + private nodes: Map = new Map(); + private workspaceDeps: Record = {}; + + setWorkspaceDeps(deps: Record): void { + this.workspaceDeps = deps; + } + + async addNode( + path: string, + content: string, + language: ScriptLanguage, + metadata: string, + imports: string[], + itemType: ItemType, + folder: string, + originalPath: string, + isDirectlyStale: boolean, + isRawApp?: boolean + ): Promise { + const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script"; + const filteredDeps = hasWorkspaceDeps + ? filterWorkspaceDependencies(this.workspaceDeps, content, language) + : {}; + const stalenessHash = await generateScriptHash({}, content, metadata); + + if (!this.nodes.has(path)) { + this.nodes.set(path, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + const node = this.nodes.get(path)!; + node.content = content; + node.stalenessHash = stalenessHash; + node.language = language; + node.metadata = metadata; + node.itemType = itemType; + node.folder = folder; + node.originalPath = originalPath; + node.isDirectlyStale = isDirectlyStale; + node.isRawApp = isRawApp; + + // Create nodes for referenced workspace deps with content and language. + const filteredDepsPaths = Object.keys(filteredDeps); + for (const depsPath of filteredDepsPaths) { + if (!this.nodes.has(depsPath)) { + const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath); + const contentHash = await generateHash(filteredDeps[depsPath] + depsPath); + const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined); + this.nodes.set(depsPath, { + content: filteredDeps[depsPath], + stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "dependencies", folder: "", originalPath: depsPath, + isDirectlyStale: !isUpToDate, + }); + } + } + + const allImports = [...imports, ...filteredDepsPaths]; + for (const importPath of allImports) { + node.imports.add(importPath); + + if (!this.nodes.has(importPath)) { + this.nodes.set(importPath, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + this.nodes.get(importPath)!.importedBy.add(path); + } + } + + getContent(path: string): string | undefined { + return this.nodes.get(path)?.content; + } + + getStalenessHash(path: string): string | undefined { + return this.nodes.get(path)?.stalenessHash; + } + + getContentHash(path: string): string | undefined { + return this.nodes.get(path)?.contentHash; + } + + setContentHash(path: string, hash: string): void { + const node = this.nodes.get(path); + if (node) { + node.contentHash = hash; + } + } + + getLanguage(path: string): ScriptLanguage | undefined { + return this.nodes.get(path)?.language; + } + + getMetadata(path: string): string | undefined { + return this.nodes.get(path)?.metadata; + } + + getStaleReason(path: string): string | undefined { + return this.nodes.get(path)?.staleReason; + } + + getItemType(path: string): ItemType | undefined { + return this.nodes.get(path)?.itemType; + } + + getFolder(path: string): string | undefined { + return this.nodes.get(path)?.folder; + } + + getIsRawApp(path: string): boolean | undefined { + return this.nodes.get(path)?.isRawApp; + } + + getIsDirectlyStale(path: string): boolean { + return this.nodes.get(path)?.isDirectlyStale ?? false; + } + + getOriginalPath(path: string): string | undefined { + return this.nodes.get(path)?.originalPath; + } + + getImports(path: string): Set | undefined { + return this.nodes.get(path)?.imports; + } + + /** + * Returns true if this node has been marked stale (directly or transitively). + */ + isStale(path: string): boolean { + return this.nodes.get(path)?.staleReason !== undefined; + } + + /** + * Mutates the tree by removing all nodes that are not stale. + * Uses BFS on reverse graph (importedBy) to find all stale scripts. + * Starts from nodes with isDirectlyStale=true. + */ + propagateStaleness(): void { + // Collect directly stale nodes + const directlyStale = new Set(); + for (const [path, node] of this.nodes.entries()) { + if (node.isDirectlyStale) { + directlyStale.add(path); + node.staleReason = "content changed"; + } + } + + const allStale = new Set(directlyStale); + const queue = [...directlyStale]; + const visited = new Set(); + + while (queue.length > 0) { + const scriptPath = queue.shift()!; + if (visited.has(scriptPath)) continue; + visited.add(scriptPath); + + const node = this.nodes.get(scriptPath); + if (!node) continue; + + for (const importer of node.importedBy) { + if (!allStale.has(importer)) { + allStale.add(importer); + queue.push(importer); + // Set reason for transitively stale scripts + const importerNode = this.nodes.get(importer); + if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`; + } + } + } + + } + + /** + * Walks all transitive imports for a node, calling the callback for each. + * Callback may return true to stop traversing that branch. + */ + traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void { + const queue = [scriptPath]; + const visited = new Set(); + + while (queue.length > 0) { + const current = queue.shift()!; + if (visited.has(current)) continue; + visited.add(current); + + const node = this.nodes.get(current); + if (!node) continue; + + for (const importPath of node.imports) { + const importNode = this.nodes.get(importPath); + if (importNode) { + const stop = callback(importPath, importNode); + if (!stop) { + queue.push(importPath); + } + } + } + } + } + + allPaths(): IterableIterator { + return this.nodes.keys(); + } + + /** + * Returns paths of all stale nodes (those with a staleReason). + */ + *stalePaths(): IterableIterator { + for (const [path, node] of this.nodes.entries()) { + if (node.staleReason) { + yield path; + } + } + } + + has(path: string): boolean { + return this.nodes.has(path); + } + + /** + * Returns workspace deps that were uploaded as mismatched with remote. + * These need to be passed as raw_workspace_dependencies in job args + * so the backend uses local content instead of deployed. + */ + getMismatchedWorkspaceDeps(): Record { + const result: Record = {}; + for (const [path, node] of this.nodes.entries()) { + if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) { + result[path] = node.content; + } + } + return result; + } + + /** + * Returns path → contentHash for all transitive imports that have been uploaded. + * Must be called after uploadScripts() has populated contentHash values. + */ + getTempScriptRefs(scriptPath: string): Record { + const result: Record = {}; + this.traverseTransitive(scriptPath, (_path, node) => { + if (node.contentHash) { + result[_path] = node.contentHash; + } + }); + return result; + } + + /** + * Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies + * considers them up-to-date on the next run. + */ + async persistDepsHashes(depsPaths: string[]): Promise { + for (const path of depsPaths) { + const node = this.nodes.get(path); + if (node?.itemType === "dependencies" && node.content !== undefined) { + const hash = await generateHash(node.content + path); + await updateMetadataGlobalLock(path, hash); + } + } + } + + get size(): number { + return this.nodes.size; + } +} diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 0a3340cc6d..0e010a0cc5 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -26,11 +26,13 @@ import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts"; import { getIsWin } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "./dependency_tree.ts"; const _require = createRequire(import.meta.url); const _parserCache = new Map>(); -function loadParser(pkgName: string): Promise { +export function loadParser(pkgName: string): Promise { let p = _parserCache.get(pkgName); if (!p) { p = (async () => { @@ -54,7 +56,7 @@ export class LockfileGenerationError extends Error { } -export async function getRawWorkspaceDependencies(): Promise> { +export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise> { const rawWorkspaceDeps: Record = {}; try { @@ -68,11 +70,13 @@ export async function getRawWorkspaceDependencies(): Promise, codebases: SyncCodebase[], - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { // Detect folder layout: my_script__mod/script.ts const isFolderLayout = isModuleEntryPoint(scriptPath); @@ -222,13 +228,15 @@ export async function generateScriptMetadataInternal( const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory(); - let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent); + // In non-legacy mode, workspace deps are tracked via the tree — exclude from hash + const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies; + let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent); // Compute per-module hashes for stale detection (like flow inline scripts) let moduleHashes: Record = {}; if (hasModules) { moduleHashes = await computeModuleHashes( - moduleFolderPath, opts.defaultTs, rawWorkspaceDependencies, isFolderLayout + moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout ); } const hasModuleHashes = Object.keys(moduleHashes).length > 0; @@ -243,27 +251,43 @@ export async function generateScriptMetadataInternal( } const conf = await readLockfile(); - if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { - if (!noStaleMessage) { - log.info( - colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) - ); + + // Use checkHash (includes module hashes) so module changes are detected as stale + const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)); + + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + // First pass: populate tree with script and its imports + const imports = await extractRelativeImports(scriptContent, remotePath, language); + await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale); + return; } - return; - } else if (dryRun) { - let detail = `${remotePath} (${language})`; - if (hasModuleHashes) { - const changed: string[] = []; - for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { - if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { - changed.push(modulePath); + // Second pass: proceed to generate (caller verified this script is stale via tree) + } else { + // Legacy behaviour: use existing staleness check + if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { + if (!noStaleMessage) { + log.info( + colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + let detail = `${remotePath} (${language})`; + if (hasModuleHashes) { + const changed: string[] = []; + for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { + if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { + changed.push(modulePath); + } + } + if (changed.length > 0) { + detail += ` [changed modules: ${changed.join(", ")}]`; } } - if (changed.length > 0) { - detail += ` [changed modules: ${changed.join(", ")}]`; - } + return detail; } - return detail; } if (!justUpdateMetadataLock && !noStaleMessage) { @@ -288,6 +312,7 @@ export async function generateScriptMetadataInternal( const hasCodebase = findCodebase(scriptPath, codebases) != undefined; if (!hasCodebase) { + const tempScriptRefs = tree?.getTempScriptRefs(remotePath); const lockPathOverride = isFolderLayout ? path.dirname(scriptPath) + "/script.lock" : undefined; @@ -298,6 +323,7 @@ export async function generateScriptMetadataInternal( remotePath, metadataParsedContent, filteredRawWorkspaceDependencies, + tempScriptRefs, lockPathOverride, ); } else { @@ -358,7 +384,7 @@ export async function generateScriptMetadataInternal( const metadataContentUsedForHash = newMetadataContent; hash = await generateScriptHash( - filteredRawWorkspaceDependencies, + depsForHash, scriptContent, metadataContentUsedForHash ); @@ -511,6 +537,7 @@ export async function computeLockCacheKey( scriptContent: string, language: ScriptLanguage, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const annotation = extractWorkspaceDepsAnnotation(scriptContent, language); const annotationStr = annotation @@ -518,7 +545,10 @@ export async function computeLockCacheKey( : "none"; const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort(); const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";"); - return await generateHash(`${language}|${annotationStr}|${depsStr}`); + const tempRefsStr = tempScriptRefs + ? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";") + : ""; + return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`); } const lockCache = new Map(); @@ -533,13 +563,15 @@ async function fetchScriptLock( language: ScriptLanguage, remotePath: string, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0; - const cacheKey = hasRawDeps - ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies) + const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0; + const cacheKey = (hasRawDeps || hasTempRefs) + ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs) : undefined; if (cacheKey && lockCache.has(cacheKey)) { - log.info(`Using cached lockfile for ${remotePath}`); + log.debug(`Using cached lockfile for ${remotePath}`); return lockCache.get(cacheKey)!; } @@ -564,6 +596,8 @@ async function fetchScriptLock( raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 ? rawWorkspaceDependencies : null, entrypoint: remotePath, + temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? tempScriptRefs : null, }), } ); @@ -604,6 +638,7 @@ async function updateScriptLock( remotePath: string, metadataContent: Record, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record, lockPathOverride?: string, ): Promise { if ( @@ -621,7 +656,7 @@ async function updateScriptLock( if (Object.keys(rawWorkspaceDependencies).length > 0) { const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', '); - log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); + log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); } const lock = await fetchScriptLock( @@ -630,6 +665,7 @@ async function updateScriptLock( language, remotePath, rawWorkspaceDependencies, + tempScriptRefs ); const lockPath = lockPathOverride ?? remotePath + ".script.lock"; @@ -692,7 +728,7 @@ async function updateModuleLocks( const moduleContent = readFileSync(fullPath, "utf-8"); const moduleRemotePath = scriptRemotePath + "/" + relPath; - log.info(colors.gray(`Generating lock for module ${relPath}`)); + log.debug(`Generating lock for module ${relPath}`); try { const lock = await fetchScriptLock( diff --git a/cli/src/utils/relative_imports.ts b/cli/src/utils/relative_imports.ts new file mode 100644 index 0000000000..7ea2d582a6 --- /dev/null +++ b/cli/src/utils/relative_imports.ts @@ -0,0 +1,39 @@ +/** + * Relative Imports Utilities for CLI + * + * Provides functions to parse relative imports from TypeScript/Python scripts using WASM. + */ + +import { ScriptLanguage } from "./script_common.ts"; +import { loadParser } from "./metadata.ts"; +import * as log from "../core/log.ts"; + +/** + * Extract relative imports from script content based on language. + * Returns resolved absolute Windmill paths (e.g., "f/folder/helper"). + */ +export async function extractRelativeImports( + code: string, + scriptPath: string, + language: ScriptLanguage +): Promise { + try { + switch (language) { + case "bun": + case "nativets": + case "deno": { + const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts"); + return parse_ts_relative_imports(code, scriptPath); + } + case "python3": { + const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports"); + return parse_py_relative_imports(code, scriptPath); + } + default: + return []; + } + } catch (e) { + log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`); + return []; + } +} diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 713a47ecb1..1531314f04 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean { return isFlowPath(p) || isAppPath(p) || isRawAppPath(p); } +/** + * Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app) + * and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting. + * Use this instead of isFolderResourcePath when the config may not yet be loaded or when + * you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files). + */ +export function isFolderResourcePathAnyFormat(p: string): boolean { + const n = normalizeSep(p); + for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) { + if ( + n.includes(suffixes.flow + "/") || + n.includes(suffixes.app + "/") || + n.includes(suffixes.raw_app + "/") + ) { + return true; + } + } + return false; +} + /** * Detect the resource type from a path, if any */ diff --git a/cli/test/cargo_backend.ts b/cli/test/cargo_backend.ts index 5c9719f6d5..a25a788f7e 100644 --- a/cli/test/cargo_backend.ts +++ b/cli/test/cargo_backend.ts @@ -63,11 +63,11 @@ export class CargoBackend { // Determine default features based on environment // CI mode: minimal features (zip only) - // Local mode with license key: full features (zip, private, enterprise, license) + // Local mode with license key: full features (zip, private, enterprise, license, python) // Local mode without license key: zip only (EE features reject API calls without valid license) const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; const hasLicenseKey = !!process.env["EE_LICENSE_KEY"]; - const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license"] : ["zip"]); + const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license", "python"] : ["zip", "python"]); // Parse additional features from environment variable const envFeatures = process.env["TEST_FEATURES"]?.split(",").filter(f => f.trim()) || []; @@ -328,6 +328,8 @@ export class CargoBackend { SQLX_OFFLINE: "true", // Disable embedding to speed up startup DISABLE_EMBEDDING: "true", + // Skip worker version check for workspace deps (workers need time to report version) + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: "1", // Create default admin user CREATE_SUPERADMIN_IF_NOT_EXISTS: "1", SUPERADMIN_EMAIL: this.config.username, @@ -708,6 +710,7 @@ export class CargoBackend { this.deleteAll("resources"), this.deleteAll("variables"), this.deleteAll("folders"), + this.deleteAllWorkspaceDeps(), ]); console.log("Workspace reset complete"); @@ -735,6 +738,28 @@ export class CargoBackend { // Ignore listing failures } } + + private async deleteAllWorkspaceDeps(): Promise { + try { + const listResponse = await this.apiRequest(`/api/w/${this.config.workspace}/workspace_dependencies/list`); + if (!listResponse.ok) return; + + const items = await listResponse.json() as { language: string; name?: string }[]; + for (const item of items) { + try { + const nameParam = item.name ? `?name=${encodeURIComponent(item.name)}` : ""; + await this.apiRequest( + `/api/w/${this.config.workspace}/workspace_dependencies/delete/${item.language}${nameParam}`, + { method: "POST" } + ); + } catch { + // Ignore individual deletion failures + } + } + } catch { + // Ignore failures + } + } } // Global backend instance diff --git a/cli/test/relative_imports_skip.test.ts b/cli/test/relative_imports_skip.test.ts new file mode 100644 index 0000000000..e43b10aa12 --- /dev/null +++ b/cli/test/relative_imports_skip.test.ts @@ -0,0 +1,420 @@ +/** + * Relative Imports Tests + * + * E2E tests for the `generate-metadata` command with relative imports: + * - Lock files correctly include transitive dependencies + * - Staleness propagates through import chains + * - Various import patterns handled correctly + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile, mkdir } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; + +// TODO: re-enable Python tests on CI if python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +const defaultMetadata = `summary: "Test" +schema: + type: object + properties: {} +lock: "" +`; + +// ============================================================================= +// Test 1: TS basic import with npm dependency propagation +// ============================================================================= + +test("TS: imported script's npm dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code, `generate-metadata failed:\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr}`).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 2: TS chained imports - dependency propagates through chain +// ============================================================================= + +test("TS: chained imports propagate npm deps through entire chain", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { utilB } from "./script_b.ts"; +export async function main() { return utilB(); } +`; + const scriptB = `import { utilC } from "./script_c.ts"; +export function utilB() { return utilC() + " B"; } +`; + const scriptC = `import _ from "lodash"; +export function utilC() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + + expect(lockC).toContain("lodash"); + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 3: TS circular imports - completes without hanging, locks generated +// ============================================================================= + +test("TS: circular imports handled gracefully with correct locks", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Circular: A imports B, B imports A, B has npm dep + const scriptA = `import { funcB } from "./script_b.ts"; +export function funcA() { return "A"; } +export async function main() { return funcA() + funcB(); } +`; + const scriptB = `import { funcA } from "./script_a.ts"; +import _ from "lodash"; +export function funcB() { return _.VERSION + funcA(); } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 4: Python basic import with pip dependency propagation +// ============================================================================= + +test.skipIf(isCI)("Python: imported script's pip dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPy); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 5: Diamond dependency - A imports B and C, both import D +// ============================================================================= + +test.skipIf(isCI)("Python: diamond dependency pattern propagates correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Diamond: A -> B, A -> C, B -> D, C -> D + const scriptA = `from f.test.script_b import func_b +from f.test.script_c import func_c + +def main(): + return func_b() + func_c() +`; + const scriptB = `from f.test.script_d import func_d + +def func_b(): + return "B" + func_d() +`; + const scriptC = `from f.test.script_d import func_d + +def func_c(): + return "C" + func_d() +`; + const scriptD = `import requests + +def func_d(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/script_a.py`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.py`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.py`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_d.py`, scriptD); + await writeFile(`${tempDir}/f/test/script_d.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + const lockD = await readFile(`${tempDir}/f/test/script_d.script.lock`, "utf-8").catch(() => ""); + + expect(lockD).toContain("requests"); + expect(lockB).toContain("requests"); + expect(lockC).toContain("requests"); + expect(lockA).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 6: Script isolation - unrelated script not marked stale +// ============================================================================= + +test("Script isolation: unrelated script not affected by changes", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // A imports B, C is isolated + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `export function helper() { return "B"; } +`; + const scriptC = `export async function main() { return "isolated"; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("All metadata up-to-date"); + + // Change script_b + await writeFile(`${tempDir}/f/test/script_b.ts`, + `export function helper() { return "B changed"; } +`); + + // script_a and script_b should be stale, script_c should NOT be mentioned + const check2 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("script_b"); + expect(check2.stdout).toContain("script_a"); + expect(check2.stdout).not.toMatch(/script_c/); + }); +}); + +// ============================================================================= +// Test 7: Python relative imports with dot syntax +// ============================================================================= + +test.skipIf(isCI)("Python: relative imports with dot syntax work correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/mymodule`, { recursive: true }); + + // Using relative import syntax + const mainPy = `from .helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/mymodule/main.py`, mainPy); + await writeFile(`${tempDir}/f/mymodule/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/mymodule/helper.py`, helperPy); + await writeFile(`${tempDir}/f/mymodule/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/mymodule/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/mymodule/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/mymodule/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 8: Adding new import updates importer's lock +// ============================================================================= + +test.skipIf(isCI)("Python: adding new import updates importer's lock correctly", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Initial: main imports helper, helper has no external deps + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPyInitial = `def helper_func(): + return "no deps" +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPyInitial); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + // Generate initial locks + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Add new script with pip dep + const utilsPy = `import requests + +def get_version(): + return requests.__version__ +`; + await writeFile(`${tempDir}/f/test/utils.py`, utilsPy); + await writeFile(`${tempDir}/f/test/utils.script.yaml`, defaultMetadata); + + // Modify helper to import utils + const helperPyWithImport = `from f.test.utils import get_version + +def helper_func(): + return get_version() +`; + await writeFile(`${tempDir}/f/test/helper.py`, helperPyWithImport); + + // Regenerate - main should now have requests + const afterAdd = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(afterAdd.code).toBe(0); + + const lockUtils = await readFile(`${tempDir}/f/test/utils.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + + expect(lockUtils).toContain("requests"); + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); diff --git a/cli/test/relative_imports_wasm.test.ts b/cli/test/relative_imports_wasm.test.ts new file mode 100644 index 0000000000..d1d537b001 --- /dev/null +++ b/cli/test/relative_imports_wasm.test.ts @@ -0,0 +1,1235 @@ +/** + * Tests for relative import resolution: + * 1. WASM parser unit tests — verify parse_ts/py_relative_imports work correctly + * 2. E2E tests — verify dependency propagation through scripts, flows, apps, and raw apps + * using the CLI generate-metadata command against a real backend + */ + +import { expect, test, describe, beforeAll, afterAll } from "bun:test"; +import { readFile, readdir, writeFile, mkdir } from "node:fs/promises"; +import { loadParser } from "../src/utils/metadata.ts"; +import { extractRelativeImports } from "../src/utils/relative_imports.ts"; +import { withTestBackend, type TestBackend, createRemoteWorkspaceDeps } from "./test_backend.ts"; +import { + createLocalScript, + createLocalFlow, + createLocalApp, + createLocalRawApp, +} from "./test_fixtures.ts"; +import { setNonDottedPaths } from "../src/utils/resource_folders.ts"; + +// TODO: re-enable on CI when python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +// ============================================================================= +// WASM Parser Unit Tests +// ============================================================================= + +describe("WASM TS parser exports parse_ts_relative_imports", () => { + test("parse_ts_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-ts"); + expect(typeof mod.parse_ts_relative_imports).toBe("function"); + }); + + test("resolves dot-relative import", async () => { + const code = `import { helper } from "./helper";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves double-dot-relative import", async () => { + const code = `import { utils } from "../utils/helper";\nexport async function main() { return utils(); }`; + const result = await extractRelativeImports(code, "f/folder/sub/script", "bun"); + expect(result).toEqual(["f/folder/utils/helper"]); + }); + + test("resolves absolute windmill import", async () => { + const code = `import { shared } from "/f/shared/utils";\nexport async function main() { return shared(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/shared/utils"]); + }); + + test("ignores external package imports", async () => { + const code = `import lodash from "lodash";\nimport axios from "axios";\nexport async function main() { return lodash.map([]); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual([]); + }); + + test("strips .ts extension from imports", async () => { + const code = `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves mixed relative and external imports", async () => { + const code = `import { helper } from "./helper";\nimport { utils } from "../utils";\nimport lodash from "lodash";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper", "f/utils"]); + }); + + test("works with named imports", async () => { + const code = `import { slugify, capitalize } from "./string_helpers";\nexport async function main() { return slugify("test"); }`; + const result = await extractRelativeImports(code, "f/utils/http_client", "bun"); + expect(result).toEqual(["f/utils/string_helpers"]); + }); +}); + +describe("WASM Python parser exports parse_py_relative_imports", () => { + test("parse_py_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-py-imports"); + expect(typeof mod.parse_py_relative_imports).toBe("function"); + }); + + test("resolves python relative import", async () => { + const code = `from f.utils.formatter import format_stats\ndef main(values: list):\n return format_stats(values)`; + const result = await extractRelativeImports(code, "f/data/process", "python3"); + expect(result).toEqual(["f/utils/formatter"]); + }); +}); + +// ============================================================================= +// Helper: find all .lock files recursively in a directory +// ============================================================================= + +async function findLockFiles(dir: string): Promise { + const entries = await readdir(dir, { recursive: true }); + return entries + .filter((e) => e.endsWith(".lock")) + .map((e) => `${dir}/${e}`); +} + +async function anyLockContains(dir: string, needle: string): Promise { + const lockFiles = await findLockFiles(dir); + for (const lockFile of lockFiles) { + const content = await readFile(lockFile, "utf-8").catch(() => ""); + if (content.includes(needle)) return true; + } + return false; +} + +async function createRemoteScript( + backend: TestBackend, + scriptPath: string, + content: string = 'export async function main() { return "hello"; }', + language: string = "bun" +): Promise { + // Archive any existing script at this path first to avoid hash conflicts + await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/delete/p/${encodeURIComponent(scriptPath)}`, + { method: "POST" } + ).catch(() => {}); + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content, + language, + summary: "Test script", + description: "Created by integration test", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + const respText = await resp.text(); + if (resp.status >= 300) { + console.log(`createRemoteScript ${scriptPath} (${language}) failed: ${resp.status} ${respText}`); + } + expect(resp.status).toBeLessThan(300); +} + +// ============================================================================= +// E2E Tests: Dependency propagation through relative imports +// ============================================================================= + +const helperScript = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + +const importerScript = `import { helper } from "/f/test/helper.ts"; +export async function main() { return helper(); } +`; + +const pyHelperScript = `import requests + +def helper(): + return requests.__version__ +`; + +const pyImporterScript = `from f.test.py_helper import helper + +def main(): + return helper() +`; + +for (const nonDotted of [false, true]) { +describe(`E2E: relative import dependency propagation via generate-metadata (${nonDotted ? "non-dotted" : "dotted"} paths)`, () => { + const inlineSuffix = nonDotted ? "" : ".inline_script"; + const flowSuffix = nonDotted ? "__flow" : ".flow"; + const appSuffix = nonDotted ? "__app" : ".app"; + const rawAppSuffix = nonDotted ? "__raw_app" : ".raw_app"; + const wmillYaml = nonDotted + ? `defaultTs: bun\nincludes: ["**"]\nexcludes: []\nnonDottedPaths: true` + : `defaultTs: bun\nincludes: ["**"]\nexcludes: []`; + + beforeAll(() => { + setNonDottedPaths(nonDotted); + }); + afterAll(() => { + setNonDottedPaths(false); + }); + test("script importing another script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // helper has lodash dep + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + // consumer imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + + expect(helperLock).toContain("lodash"); + expect(consumerLock).toContain("lodash"); + }); + }); + + test("flow inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // Helper script lock should have lodash + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Flow inline script lock should also have lodash (transitive via helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + const flowHasLodash = await anyLockContains(flowDir, "lodash"); + expect(flowHasLodash).toBe(true); + }); + }); + + test("app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // App inline script lock should have lodash (transitive via helper) + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + const appHasLodash = await anyLockContains(appDir, "lodash"); + expect(appHasLodash).toBe(true); + }); + }); + + test("raw app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Raw app inline script lock should have lodash (transitive via helper) + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + const rawAppHasLodash = await anyLockContains(rawAppDir, "lodash"); + expect(rawAppHasLodash).toBe(true); + }); + }); + + test("modifying leaf script marks all dependents as stale", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("up-to-date"); + + // Modify the leaf helper script (change content but keep lodash dep) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION + " v2"; }\n` + ); + + // All dependents should now be detected as stale + const check2 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("helper"); + expect(check2.stdout).toContain("consumer"); + expect(check2.stdout).toContain("my_flow"); + expect(check2.stdout).toContain("my_app"); + expect(check2.stdout).toContain("my_raw_app"); + }); + }); + + test("new script importing locally modified helper gets local deps not remote", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy helper (lodash) to backend, then pull locally + // Content includes literal \n to exercise Postgres bytea cast bug (content::bytea fails on backslash) + const helperWithBackslash = `import _ from "lodash";\nexport function helper() { return "line1\\nline2"; }\n`; + await createRemoteScript(backend, "f/test/helper", helperWithBackslash); + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + if (pull.code !== 0) { + console.log("PULL STDOUT:", pull.stdout); + console.log("PULL STDERR:", pull.stderr); + } + expect(pull.code).toBe(0); + + // Modify helper locally to use axios instead of lodash (NOT pushed) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + + // Regenerate helper metadata — helper is stale (content changed from deployed) + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Create a new consumer that imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Consumer is stale (new), helper is NOT stale (metadata up-to-date). + // Helper differs from deployed (axios vs lodash). + // Diff endpoint should detect mismatch, upload local helper. + // Consumer's lock must have axios (local), not lodash (deployed). + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("axios"); + expect(consumerLock).not.toContain("lodash"); + }); + }); + + test("new script importing unpushed helper gets transitive deps in lock", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create helper (lodash dep) and generate its metadata + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Now create a NEW consumer that imports helper + // Helper is not stale (metadata up-to-date) and was never pushed to remote + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Run 2: only consumer is stale (new). Helper is NOT stale. + // Consumer's lock must include lodash (transitive dep from local helper) + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + test.skipIf(isCI)("dependency change triggers lock regeneration for flows and apps", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Step 1: Create helper scripts locally and deploy them to remote via push + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript(tempDir, "f/test", "py_helper", "python3", pyHelperScript); + + // Generate metadata for scripts only, then push to deploy them on remote + const genScripts = await backend.runCLICommand( + ["generate-metadata", "--yes", "--skip-flows", "--skip-apps"], + tempDir + ); + expect(genScripts.code).toBe(0); + + const push = await backend.runCLICommand(["sync", "push", "--yes"], tempDir); + if (push.code !== 0) { + console.log("PUSH STDOUT:", push.stdout); + console.log("PUSH STDERR:", push.stderr); + } + expect(push.code).toBe(0); + + // Step 2: Now create flows/apps that import the deployed helpers + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalFlow(tempDir, "f/test", "my_py_flow", pyImporterScript, "python3"); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Step 3: Generate initial metadata for everything + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (initial.code !== 0) { + console.log("INITIAL STDOUT:", initial.stdout); + console.log("INITIAL STDERR:", initial.stderr); + } + expect(initial.code).toBe(0); + + // Verify flow directories have correct structure — no extra files created + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect((await readdir(flowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.ts`, "flow.yaml"]); + const pyFlowDir = `${tempDir}/f/test/my_py_flow${flowSuffix}`; + expect((await readdir(pyFlowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.py`, "flow.yaml"]); + + // Verify TS flow/app locks have lodash + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + + // Verify Python flow lock has requests + expect(await anyLockContains(pyFlowDir, "requests")).toBe(true); + + // Step 5: Modify helpers LOCALLY (NOT pushed to remote — this is the key scenario) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + await createLocalScript( + tempDir, + "f/test", + "py_helper", + "python3", + `import pandas\n\ndef helper():\n return pandas.__version__\n` + ); + + // Step 6: Regenerate — flow locks must use LOCAL helper content, not remote deployed version + const regen = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (regen.code !== 0) { + console.log("REGEN STDOUT:", regen.stdout); + console.log("REGEN STDERR:", regen.stderr); + } + expect(regen.code).toBe(0); + + // TS flow lock should now have axios (from local helper), not lodash (from remote) + expect(await anyLockContains(flowDir, "axios")).toBe(true); + expect(await anyLockContains(flowDir, "lodash")).toBe(false); + + // Python flow lock should now have pandas (from local helper), not requests (from remote) + const pyFlowLockFiles = await findLockFiles(pyFlowDir); + for (const f of pyFlowLockFiles) { + const c = await readFile(f, "utf-8").catch(() => ""); + console.log(`PY FLOW LOCK [${f}]: ${c.substring(0, 500)}`); + } + expect(await anyLockContains(pyFlowDir, "pandas")).toBe(true); + expect(await anyLockContains(pyFlowDir, "requests")).toBe(false); + + // TS app lock should now have axios, not lodash + expect(await anyLockContains(appDir, "axios")).toBe(true); + expect(await anyLockContains(appDir, "lodash")).toBe(false); + + // Raw app lock should now have axios, not lodash + expect(await anyLockContains(rawAppDir, "axios")).toBe(true); + expect(await anyLockContains(rawAppDir, "lodash")).toBe(false); + }); + }); + + test.skipIf(isCI)("locally modified workspace deps are used for lock generation instead of remote", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy named Python workspace dep "test" with requests on the remote + await createRemoteWorkspaceDeps(backend, "python3", "requests", "test"); + + // Pull to get remote state locally + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Empty the workspace dep locally (NOT pushed) — simulates removing all deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/test.requirements.in`, + "" + ); + + // Create a Python script that uses the named workspace dep via #requirements: test + await createLocalScript( + tempDir, + "f/test", + "my_script", + "python3", + `#requirements: test\n\ndef main():\n return "hello"\n` + ); + + // Generate metadata — local dep is empty, so lock must NOT contain requests (from remote) + const gen = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock must use local (empty) workspace deps, not remote (requests) + expect(scriptLock).not.toContain("requests"); + + // Second run should be idempotent — no stale items + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("unchanged workspace deps do not cause dependents to be stale on subsequent runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/test", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First generate-metadata — everything is stale, locks get generated + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code, `generate-metadata failed:\nSTDOUT: ${gen1.stdout}\nSTDERR: ${gen1.stderr}`).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + expect(scriptLock).toContain("axios"); + + // Second generate-metadata — nothing changed, should report "All metadata up-to-date" + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code, `generate-metadata failed:\nSTDOUT: ${gen2.stdout}\nSTDERR: ${gen2.stderr}`).toBe(0); + + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("All metadata up-to-date"); + // Should NOT show workspace deps or scripts as stale + expect(output).not.toContain("stale metadata"); + }); + }); + + test("diff endpoint correctly identifies mismatched scripts and workspace deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + const { generateHash } = await import("../src/utils/utils.ts"); + const { setClient } = await import("../src/core/client.ts"); + const { diffRawScriptsWithDeployed } = await import("../gen/services.gen.ts"); + + setClient(backend.token, backend.baseUrl); + + // Deploy a script and two workspace deps (bun + python) + const scriptContent = `export async function main() { return "hello"; }`; + await createRemoteScript(backend, "f/test/deployed_script", scriptContent); + + const bunDepsContent = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", bunDepsContent); + + const scriptHash = await generateHash(scriptContent); + const bunDepsHash = await generateHash(bunDepsContent); + const wrongHash = await generateHash("totally different content"); + + // Call 1: matching hash, same path → should NOT be mismatched + const call1 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { "f/test/deployed_script": scriptHash }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: bunDepsHash }, + ], + }, + }); + expect(call1).not.toContain("f/test/deployed_script"); + expect(call1).not.toContain("dependencies/package.json"); + + // Call 2: wrong hash same path + right hash wrong path → both should be mismatched + const call2 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { + "f/test/deployed_script": wrongHash, + "f/test/nonexistent_script": scriptHash, + }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: wrongHash }, + { path: "dependencies/requirements.in", language: "python3", hash: bunDepsHash }, + ], + }, + }); + // Same path, wrong hash → mismatched + expect(call2).toContain("f/test/deployed_script"); + expect(call2).toContain("dependencies/package.json"); + // Wrong path, right hash → mismatched (endpoint should not match by hash alone) + expect(call2).toContain("f/test/nonexistent_script"); + expect(call2).toContain("dependencies/requirements.in"); + }); + }); + + test("folder arg includes importers outside the folder by default", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A from a different directory + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // First: generate-metadata globally to establish baseline locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Now modify helper to use axios instead + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run generate-metadata for f/lib only — consumer (in f/app) should also be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer's lock should now have axios (updated even though it's outside f/lib) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("axios"); + }); + }); + + test("--strict-folder-boundaries skips importers outside the folder and warns", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // Establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Modify helper to use axios + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run with --strict-folder-boundaries — consumer should NOT be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer lock should still have lodash (not updated) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("lodash"); + expect(consumerLock2).not.toContain("axios"); + + // Output should contain a warning about the skipped importer + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("Warning"); + expect(output).toContain("f/app/consumer"); + + // Running again with same args should report up-to-date (not stuck in loop) + const gen3 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + expect(gen3.code).toBe(0); + const output3 = gen3.stdout + gen3.stderr; + expect(output3).toContain("All metadata up-to-date"); + }); + }); + + // TODO: consider adding --skip-workspace-dependencies flag to generate-metadata + test("folder arg includes workspace dependencies in lock generation", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // Run generate-metadata for f/mydir only — should still include workspace deps content + // TODO: consider adding --skip-workspace-dependencies flag + const gen = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/mydir/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock should use local workspace deps (axios), not remote (lodash) + expect(scriptLock).toContain("axios"); + expect(scriptLock).not.toContain("lodash"); + + // Running again should report up-to-date (not stuck in loop) + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("strict folder boundaries with workspace deps does not loop", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First run with strict + folder + const gen1 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not stuck in loop + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #1: flow/app with mismatched workspace deps — hash inconsistency causes perpetual staleness + test("flow/app/raw app with workspace deps does not loop across runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a flow, app, and raw app with inline scripts + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `export async function main() { return "hello from flow"; }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `export async function main() { return "hello from app"; }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `export async function main() { return "hello from raw app"; }` + ); + + // First run — generates locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not loop + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #2: flow/app/raw app importing locally-modified helper uses local content not remote + test("flow/app/raw app importing locally modified helper uses local content", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper with lodash dep — only exists locally, never pushed + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Flow, app, and raw app inline scripts import the helper + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // All locks should contain lodash (transitive dep from local helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + }); + }); + + // Cross-directory relative imports with ../ + test("cross-directory relative import with ../ propagates deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper in f/shared with lodash dep + await createLocalScript( + tempDir, + "f/shared", + "utils", + "bun", + `import _ from "lodash";\nexport function utils() { return _.VERSION; }` + ); + + // Script in f/app imports via ../shared/utils + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { utils } from "../shared/utils.ts";\nexport async function main() { return utils(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + // Multi-level transitive chain: A -> B -> C, C changes, A must update + test("3-level transitive chain propagates staleness", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // C has dayjs dep + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import dayjs from "dayjs";\nexport function c() { return dayjs(); }` + ); + + // B imports C + await createLocalScript( + tempDir, + "f/chain", + "b", + "bun", + `import { c } from "./c.ts";\nexport function b() { return c(); }` + ); + + // A imports B + await createLocalScript( + tempDir, + "f/chain", + "a", + "bun", + `import { b } from "/f/chain/b.ts";\nexport async function main() { return b(); }` + ); + + // First run — establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const aLock1 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock1).toContain("dayjs"); + + // Modify C to use uuid instead + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import { v4 } from "uuid";\nexport function c() { return v4(); }` + ); + + // Second run — A should be updated transitively (C changed -> B stale -> A stale) + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + + const aLock2 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock2).toContain("uuid"); + }); + }); +}); +} // end for nonDotted diff --git a/cli/test/resource_folders_unit.test.ts b/cli/test/resource_folders_unit.test.ts index da8ccd7ece..80732ebc1c 100644 --- a/cli/test/resource_folders_unit.test.ts +++ b/cli/test/resource_folders_unit.test.ts @@ -15,6 +15,7 @@ import { isAppPath, isRawAppPath, isFolderResourcePath, + isFolderResourcePathAnyFormat, detectFolderResourceType, isRawAppBackendPath, isAppInlineScriptPath, @@ -214,6 +215,39 @@ describe("isFolderResourcePath", () => { }); }); +// This is the bug that isFolderResourcePathAnyFormat fixes: +// when nonDottedPaths is false (default), isFolderResourcePath misses non-dotted paths +// like "f/my_raw__raw_app/backend/a.ts", causing raw app backend scripts to leak +// into the standalone script list during generate-metadata. +describe("isFolderResourcePathAnyFormat", () => { + test("detects non-dotted paths even when global setting is dotted", () => { + setNonDottedPaths(false); + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow__flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard__app/inline.ts")).toBe(true); + }); + + test("detects dotted paths even when global setting is non-dotted", () => { + setNonDottedPaths(true); + expect(isFolderResourcePathAnyFormat("f/my_raw.raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow.flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard.app/inline.ts")).toBe(true); + }); + + test("rejects non-folder-resource paths", () => { + expect(isFolderResourcePathAnyFormat("f/my_script.ts")).toBe(false); + expect(isFolderResourcePathAnyFormat("f/var.variable.yaml")).toBe(false); + }); + + test("confirms isFolderResourcePath fails for mismatched format (the bug)", () => { + setNonDottedPaths(false); + // isFolderResourcePath misses non-dotted paths when setting is dotted + expect(isFolderResourcePath("f/my_raw__raw_app/backend/a.ts")).toBe(false); + // isFolderResourcePathAnyFormat catches it + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + }); +}); + describe("detectFolderResourceType", () => { test("detects flow type", () => { expect(detectFolderResourceType("f/x.flow/flow.yaml")).toBe("flow"); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 40e8ac4fdf..93f42bee54 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -6,7 +6,8 @@ * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): * @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests) - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * * This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript * If you add new helpers, update cross-links in the files above. diff --git a/cli/test/test_backend.ts b/cli/test/test_backend.ts index 16fb79528d..34ef1f240f 100644 --- a/cli/test/test_backend.ts +++ b/cli/test/test_backend.ts @@ -24,7 +24,8 @@ * @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * - * This file contains: API-based creation helpers (createTestApp, createTestResource, etc.) + * This file contains: API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * If you add new helpers, update cross-links in the files above. */ @@ -64,6 +65,10 @@ export interface TestBackend { listAllApps?(): Promise; listAllResources?(): Promise; listAllVariables?(): Promise; + + // Methods for creating apps and flows with custom inline scripts + createAppWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; + createFlowWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; } /** @@ -342,6 +347,88 @@ class CargoBackendAdapter implements TestBackend { if (!response.ok) return []; return response.json(); } + + async createAppWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + value: { + type: "app", + grid: [ + { + id: "button1", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: inlineScriptContent, + language, + }, + }, + }, + }, + }, + ], + hiddenInlineScripts: [], + css: {}, + norefreshbar: false, + }, + summary: "Test app with inline script", + policy: { + on_behalf_of: null, + on_behalf_of_email: null, + triggerables: {}, + execution_mode: "viewer", + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create app ${path}: ${error}`); + } + await response.text(); + } + + async createFlowWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/flows/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + summary: "Test flow with inline script", + description: `Flow at ${path}`, + value: { + modules: [ + { + id: "a", + value: { + type: "rawscript", + content: inlineScriptContent, + language, + input_transforms: {}, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create flow ${path}: ${error}`); + } + await response.text(); + } } /** @@ -580,6 +667,38 @@ export async function createNonAdminUser( return await loginResp.text(); } +/** + * Create workspace dependencies via the API (e.g. a shared package.json for bun scripts). + */ +export async function createRemoteWorkspaceDeps( + backend: TestBackend, + language: string, + content: string, + name?: string, +): Promise { + if (!backend.apiRequest) { + throw new Error("Backend does not support apiRequest"); + } + + const resp = await backend.apiRequest( + `/api/w/${backend.workspace}/workspace_dependencies/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + workspace_id: backend.workspace, + language, + content, + ...(name ? { name } : {}), + }), + } + ); + if (!resp.ok) { + throw new Error(`Failed to create workspace deps (${resp.status}): ${await resp.text()}`); + } + await resp.text(); +} + // Re-export for convenience export type { CargoBackendConfig } from "./cargo_backend.ts"; export type { ContainerConfig } from "./containerized_backend.ts"; diff --git a/cli/test/test_fixtures.ts b/cli/test/test_fixtures.ts index cde57e1a95..1d2f6f6bb5 100644 --- a/cli/test/test_fixtures.ts +++ b/cli/test/test_fixtures.ts @@ -8,7 +8,8 @@ * - Local creation functions: Create fixtures AND write them to disk * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * * This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.) @@ -29,6 +30,7 @@ import { getFolderSuffix, getMetadataFileName, getModuleFolderSuffix, + getNonDottedPaths, } from "../src/utils/resource_folders.ts"; // ============================================================================= @@ -47,7 +49,8 @@ export interface ScriptFixture { export interface FlowFixture { metadata: FileFixture; - inlineScript: FileFixture; + inlineScript?: FileFixture; + inlineLock?: FileFixture; } export interface AppFixture { @@ -153,16 +156,34 @@ kind: script */ export function createFlowFixture( name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): FlowFixture { const flowSuffix = getFolderSuffix("flow"); const metadataFile = getMetadataFileName("flow", "yaml"); - const scriptContent = - inlineScriptContent ?? - `export async function main() {\n return "Hello from flow ${name}";\n}`; + const defaultContent = language === "python3" + ? `def main():\n return "Hello from flow ${name}"` + : `export async function main() {\n return "Hello from flow ${name}";\n}`; - return { + const scriptContent = inlineScriptContent ?? defaultContent; + + const langMap: Record = { bun: "bun", python3: "python3" }; + const extMap: Record = { bun: "ts", python3: "py" }; + + const ext = extMap[language]; + // With dotted paths (.flow), inline scripts use .inline_script suffix (a.inline_script.ts) + // With non-dotted paths (__flow), they don't (a.ts) + const inlineSuffix = getNonDottedPaths() ? "" : ".inline_script"; + const scriptFile = `a${inlineSuffix}.${ext}`; + const lockFile = `a${inlineSuffix}.lock`; + + const lockLine = lockContent !== undefined + ? `\n lock: "!inline ${lockFile}"` + : ""; + + const result: FlowFixture = { metadata: { path: `${name}${flowSuffix}/${metadataFile}`, content: `summary: "${name} flow" @@ -172,9 +193,8 @@ value: - id: a value: type: rawscript - content: | - ${scriptContent.split("\n").join("\n ")} - language: bun + content: "!inline ${scriptFile}"${lockLine} + language: ${langMap[language]} input_transforms: {} schema: $schema: "https://json-schema.org/draft/2020-12/schema" @@ -184,10 +204,19 @@ schema: `, }, inlineScript: { - path: `${name}${flowSuffix}/a.inline_script.ts`, + path: `${name}${flowSuffix}/${scriptFile}`, content: scriptContent, }, }; + + if (lockContent !== undefined) { + result.inlineLock = { + path: `${name}${flowSuffix}/${lockFile}`, + content: lockContent, + }; + } + + return result; } // ============================================================================= @@ -211,10 +240,14 @@ schema: * * @keywords app fixture, create app, local app */ -export function createAppFixture(name: string): AppFixture { +export function createAppFixture(name: string, inlineScriptContent?: string): AppFixture { const appSuffix = getFolderSuffix("app"); const metadataFile = getMetadataFileName("app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main() {\n return "hello from app";\n}`; + const indented = scriptContent.split("\n").join("\n "); + return { metadata: { path: `${name}${appSuffix}/${metadataFile}`, @@ -231,9 +264,7 @@ value: type: runnableByName inlineScript: content: | - export async function main() { - return "hello from app"; - } + ${indented} language: bun hiddenInlineScripts: [] css: {} @@ -270,10 +301,13 @@ policy: * * @keywords raw app fixture, create raw app, local raw app, react app */ -export function createRawAppFixture(name: string): RawAppFixture { +export function createRawAppFixture(name: string, inlineScriptContent?: string): RawAppFixture { const rawAppSuffix = getFolderSuffix("raw_app"); const metadataFile = getMetadataFileName("raw_app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main(x: string) {\n return x\n}`; + return { metadata: { path: `${name}${rawAppSuffix}/${metadataFile}`, @@ -312,14 +346,15 @@ root.render() }`, }, inlineScript: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.ts`, - content: `export async function main(x: string) { - return x -} -`, + path: `${name}${rawAppSuffix}/backend/a.ts`, + content: scriptContent + "\n", + }, + inlineScriptMeta: { + path: `${name}${rawAppSuffix}/backend/a.yaml`, + content: `type: inline\n`, }, inlineScriptLock: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.lock`, + path: `${name}${rawAppSuffix}/backend/a.lock`, content: ``, }, }; @@ -388,13 +423,16 @@ export async function createLocalFlow( tempDir: string, path: string, name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): Promise { - const fixture = createFlowFixture(name, inlineScriptContent); + const fixture = createFlowFixture(name, inlineScriptContent, language, lockContent); const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`; await mkdir(flowDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -418,13 +456,15 @@ export async function createLocalFlow( export async function createLocalApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createAppFixture(name); + const fixture = createAppFixture(name, inlineScriptContent); const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`; await mkdir(appDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -449,12 +489,13 @@ export async function createLocalApp( export async function createLocalRawApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createRawAppFixture(name); + const fixture = createRawAppFixture(name, inlineScriptContent); const rawAppSuffix = getFolderSuffix("raw_app"); const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`; - await mkdir(`${appDir}/inline_scripts`, { recursive: true }); + await mkdir(`${appDir}/backend`, { recursive: true }); for (const file of Object.values(fixture)) { const fullPath = `${tempDir}/${path}/${file.path}`; diff --git a/cli/test/unified_generate_metadata.test.ts b/cli/test/unified_generate_metadata.test.ts index a25d7f8bd3..ba14f79a15 100644 --- a/cli/test/unified_generate_metadata.test.ts +++ b/cli/test/unified_generate_metadata.test.ts @@ -641,7 +641,7 @@ describe("generate-metadata with script modules", () => { expect(output).toContain("order_workflow"); // Module files should NOT appear as separate stale scripts (only within [changed modules: ...]) const lines = output.split("\n"); - const staleLines = lines.filter((l: string) => l.includes("f/test/")); + const staleLines = lines.filter((l: string) => l.includes("f/test/") || l.includes("f\\test\\")); expect(staleLines.length).toBe(1); expect(staleLines[0]).toContain("order_workflow"); }); @@ -745,9 +745,6 @@ describe("generate-metadata with script modules", () => { expect(result3.code).toEqual(0); const output3 = result3.stdout + result3.stderr; expect(output3).toContain("order_workflow"); - expect(output3).toContain("helper.ts"); - // utils.ts was not modified, should not be listed as changed - expect(output3).not.toContain("utils.ts"); }); }); From 3c8d351c9722a089133871019d27cf3bc3cdc159 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 23 Mar 2026 21:04:28 +0000 Subject: [PATCH 05/48] fix: improve SQS retries --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f30dcc7a1f..b027ed7410 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -fe8f0d1d7448464c98474d994e6492c0a45e8e38 +5165096b104dd95da22e738d52a74e4f9b95a5e7 \ No newline at end of file From c13b95f8b20824f7021a6853b6e40edbc0018d68 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 23 Mar 2026 23:44:38 +0100 Subject: [PATCH 06/48] Fix SAML Redirect (#8486) * Fix SAML redirect * Fix SAML redirect 2 * ee repo ref * Apply suggestion from @claude[bot] Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * chore: update ee-repo-ref to 50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 This commit updates the EE repository reference after PR #475 was merged in windmill-ee-private. Previous ee-repo-ref: c56747af8c420dd2222829f303b7fe6009ab9892 New ee-repo-ref: 50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 Automated by sync-ee-ref workflow. --------- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- frontend/src/lib/components/Login.svelte | 7 +++++++ .../routes/(root)/(logged)/user/(user)/login/+page.svelte | 6 +++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b027ed7410..3537bbe745 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5165096b104dd95da22e738d52a74e4f9b95a5e7 \ No newline at end of file +50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index b9c0de018d..1e375cb53d 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -341,6 +341,13 @@ btnClasses="mt-2 w-full" on:click={() => { if (saml) { + if (rd) { + try { + localStorage.setItem('rd', rd) + } catch (e) { + console.error('Could not persist redirection to local storage', e) + } + } window.location.href = saml } else { sendUserToast('No SAML login available', true) diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index 4d1a58c70d..63843578c9 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -24,7 +24,11 @@ const email = page.url.searchParams.get('email') ?? '' const password = page.url.searchParams.get('password') ?? '' const error = page.url.searchParams.get('error') ?? undefined - const rd = page.url.searchParams.get('rd') ?? undefined + const rdFromStorage = localStorage.getItem('rd') || undefined + if (rdFromStorage) { + localStorage.removeItem('rd') + } + const rd = page.url.searchParams.get('rd') ?? rdFromStorage let showPassword = false let firstTime = $state(false) From e0d35ade72689d9896cb258765d28cad00bd8005 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:23:06 +0100 Subject: [PATCH 07/48] chore: fix Claude action + add skills for codex + update autonomous mode docs (#8489) * chore: fix Claude action overlap with /ai-fast * chore: add Codex skills under .agents * chore: remove user_invocable from Codex skills * docs: require draft PR creation in autonomous mode --- .agents/skills/commit/SKILL.md | 59 ++ .agents/skills/local-review/SKILL.md | 97 +++ .agents/skills/native-trigger/SKILL.md | 777 ++++++++++++++++++++++++ .agents/skills/pr/SKILL.md | 109 ++++ .agents/skills/refine/SKILL.md | 38 ++ .agents/skills/rust-backend/SKILL.md | 107 ++++ .agents/skills/svelte-frontend/SKILL.md | 80 +++ .github/workflows/claude.yml | 8 +- docs/autonomous-mode.md | 4 +- 9 files changed, 1274 insertions(+), 5 deletions(-) create mode 100644 .agents/skills/commit/SKILL.md create mode 100644 .agents/skills/local-review/SKILL.md create mode 100644 .agents/skills/native-trigger/SKILL.md create mode 100644 .agents/skills/pr/SKILL.md create mode 100644 .agents/skills/refine/SKILL.md create mode 100644 .agents/skills/rust-backend/SKILL.md create mode 100644 .agents/skills/svelte-frontend/SKILL.md diff --git a/.agents/skills/commit/SKILL.md b/.agents/skills/commit/SKILL.md new file mode 100644 index 0000000000..d610fa3f65 --- /dev/null +++ b/.agents/skills/commit/SKILL.md @@ -0,0 +1,59 @@ +--- +name: commit +description: Create a git commit with conventional commit format. MUST use anytime you want to commit changes. +--- + +# Git Commit Skill + +Create a focused, single-line commit following conventional commit conventions. + +## Instructions + +1. **Analyze changes**: Run `git status` and `git diff` to understand what was modified +2. **Stage only modified files**: Add files individually by name. NEVER use `git add -A` or `git add .` +3. **Write commit message**: Follow the conventional commit format as a single line + +## Conventional Commit Format + +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code change that neither fixes a bug nor adds a feature +- `docs`: Documentation only changes +- `style`: Formatting, missing semicolons, etc (no code change) +- `test`: Adding or correcting tests +- `chore`: Maintenance tasks, dependency updates, etc +- `perf`: Performance improvement + +### Rules +- Message MUST be a single line (no multi-line messages) +- Description should be lowercase, imperative mood ("add" not "added") +- No period at the end +- Keep under 72 characters total + +### Examples +``` +feat: add token usage tracking for AI providers +fix: resolve null pointer in job executor +refactor: extract common validation logic +docs: update API endpoint documentation +chore: upgrade sqlx to 0.7 +``` + +## Execution Steps + +1. Run `git status` to see all changes +2. Run `git diff` to understand the changes in detail +3. Run `git log --oneline -5` to see recent commit style +4. Stage ONLY the modified/relevant files: `git add ...` +5. Create the commit with conventional format: + ```bash + git commit -m ": + + Co-Authored-By: Claude Opus 4.5 " + ``` +6. Run `git status` to verify the commit succeeded diff --git a/.agents/skills/local-review/SKILL.md b/.agents/skills/local-review/SKILL.md new file mode 100644 index 0000000000..ad701ac367 --- /dev/null +++ b/.agents/skills/local-review/SKILL.md @@ -0,0 +1,97 @@ +--- +name: local-review +description: Code review a pull request for bugs and CLAUDE.md compliance. MUST use when asked to review code. +--- + +# Local Code Review Skill + +Review a pull request for real bugs and CLAUDE.md compliance violations. This review targets HIGH SIGNAL issues only. + +## Review Philosophy + +- **Only flag issues you are certain about.** If you are not sure an issue is real, do not flag it. False positives erode trust and waste reviewer time. +- Think like a senior engineer doing a final review — flag things that would cause incidents, not things that are merely imperfect. + +## What to Flag + +- Code that won't compile or parse (syntax errors, type errors, missing imports) +- Code that will definitely produce wrong results regardless of inputs +- Clear, unambiguous CLAUDE.md violations (quote the exact rule being violated) +- Security issues in introduced code (injection, auth bypass, data exposure) +- Incorrect logic that will fail in production + +## What NOT to Flag + +- Code style or quality concerns +- Potential issues that depend on specific inputs or runtime state +- Subjective suggestions or improvements +- Pre-existing issues not introduced by this PR +- Pedantic nitpicks a senior engineer wouldn't flag +- Issues a linter or type checker will catch +- General quality concerns unless explicitly prohibited in CLAUDE.md +- Issues silenced via lint ignore comments + +## Execution Steps + +1. **Determine the PR scope**: + - If an argument is provided, use it as the PR number or branch + - Otherwise, detect from the current branch vs main + - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` + +2. **Find relevant CLAUDE.md files**: + - Read the root `CLAUDE.md` + - Check for CLAUDE.md files in directories containing changed files + +3. **Get the diff and metadata**: + - `gh pr diff` or `git diff main...HEAD` for the full diff + - `gh pr view` or `git log main..HEAD --oneline` for context + +4. **Read changed files** where the diff alone is insufficient to understand context + +5. **Review for**: + - CLAUDE.md compliance — check each rule against the changed code + - Bugs and logic errors — will this code work correctly? + - Security issues — injection, auth, data exposure in new code + +6. **Self-validate each finding**: Before reporting, ask yourself: + - "Is this definitely a real issue, not a false positive?" + - "Would a senior engineer flag this in review?" + - If the answer to either is no, discard the finding + +7. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag) + +## Output Format + +``` +## Code review + +Found N issues: + +1. () + + +2. () + +``` + +If no issues are found: + +``` +## Code review + +No issues found. Checked for bugs and CLAUDE.md compliance. +``` + +## Posting Comments (--comment flag) + +If the user passes `--comment`, post findings as inline PR comments using: + +```bash +gh pr review --comment --body "" +``` + +Or for inline comments on specific lines: + +```bash +gh api repos/{owner}/{repo}/pulls/{pr}/reviews -f body="" -f event="COMMENT" -f comments="[...]" +``` diff --git a/.agents/skills/native-trigger/SKILL.md b/.agents/skills/native-trigger/SKILL.md new file mode 100644 index 0000000000..026c1900bf --- /dev/null +++ b/.agents/skills/native-trigger/SKILL.md @@ -0,0 +1,777 @@ +# Skill: Adding Native Trigger Services + +This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. + +## Architecture Overview + +The native trigger system consists of: + +1. **Database Layer** - PostgreSQL tables and enum types +2. **Backend Rust Implementation** - Core trait, handlers, and service modules in the `windmill-native-triggers` crate +3. **Frontend Svelte Components** - Configuration forms and UI components + +### Key Files + +| Component | Path | +|-----------|------| +| Core module with `External` trait | `backend/windmill-native-triggers/src/lib.rs` | +| Generic CRUD handlers | `backend/windmill-native-triggers/src/handler.rs` | +| Background sync logic | `backend/windmill-native-triggers/src/sync.rs` | +| OAuth/workspace integration | `backend/windmill-native-triggers/src/workspace_integrations.rs` | +| Re-export shim (windmill-api) | `backend/windmill-api/src/native_triggers/mod.rs` | +| TriggerKind enum | `backend/windmill-common/src/triggers.rs` | +| JobTriggerKind enum | `backend/windmill-common/src/jobs.rs` | +| Frontend service registry | `frontend/src/lib/components/triggers/native/utils.ts` | +| Frontend trigger utilities | `frontend/src/lib/components/triggers/utils.ts` | +| Trigger badges (icons + counts) | `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte` | +| Workspace integrations UI | `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte` | +| OAuth config form component | `frontend/src/lib/components/workspaceSettings/OAuthClientConfig.svelte` | +| OpenAPI spec | `backend/windmill-api/openapi.yaml` | +| Reference: Nextcloud module | `backend/windmill-native-triggers/src/nextcloud/` | +| Reference: Google module | `backend/windmill-native-triggers/src/google/` | + +### Crate Structure + +The native trigger code lives in the `windmill-native-triggers` crate (`backend/windmill-native-triggers/`). The `windmill-api` crate re-exports everything via a shim: + +```rust +// backend/windmill-api/src/native_triggers/mod.rs +pub use windmill_native_triggers::*; +``` + +All new service modules go in `backend/windmill-native-triggers/src/`. + +--- + +## Core Concepts + +### The `External` Trait + +Every native trigger service implements the `External` trait defined in `lib.rs`: + +```rust +#[async_trait] +pub trait External: Send + Sync + 'static { + // Associated types: + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + // Constants: + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + const AUTH_ENDPOINT: &'static str; + + // Required methods: + async fn create(&self, w_id, oauth_data, webhook_token, data, db, tx) -> Result; + async fn update(&self, w_id, oauth_data, external_id, webhook_token, data, db, tx) -> Result; + async fn get(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn delete(&self, w_id, oauth_data, external_id, db, tx) -> Result<()>; + async fn exists(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors); + fn external_id_and_metadata_from_response(&self, resp) -> (String, Option); + + // Methods with defaults: + async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result; + fn service_config_from_create_response(&self, data, resp) -> Option; + fn additional_routes(&self) -> axum::Router; + async fn http_client_request(&self, url, method, workspace_id, tx, db, headers, body) -> Result; +} +``` + +Key design points: +- **`update()` returns `serde_json::Value`** - the resolved service_config to store. Each service is responsible for building the final config. +- **`maintain_triggers()`** - periodic background maintenance. Each service implements its own strategy (Nextcloud: reconcile with external state; Google: renew expiring channels). +- **No `list_all()` in the trait** - services that need it (Nextcloud) implement it privately; services that don't (Google) use different maintenance strategies. +- **No `get_external_id_from_trigger_data()` or `extract_service_config_from_trigger_data()`** - removed in favor of the `maintain_triggers` pattern. + +### Create Lifecycle: Two Paths + +The `create_native_trigger` handler in `handler.rs` supports two creation flows, controlled by `service_config_from_create_response()`: + +**Path A: Short (Google pattern)** - `service_config_from_create_response()` returns `Some(config)`: +1. `create()` registers on external service +2. `external_id_and_metadata_from_response()` extracts the ID +3. `service_config_from_create_response()` builds the config directly from input data + response metadata +4. Stores trigger in DB -- done, no extra round-trip + +Use this when the external_id is known before the create call (e.g., Google generates the channel_id as a UUID upfront and includes it in the webhook URL). + +**Path B: Long (Nextcloud pattern)** - `service_config_from_create_response()` returns `None` (default): +1. `create()` registers on external service (webhook URL has no external_id yet) +2. `external_id_and_metadata_from_response()` extracts the ID +3. `update()` is called to fix the webhook URL with the now-known external_id +4. `update()` returns the resolved service_config +5. Stores trigger in DB + +Use this when the external_id is assigned by the remote service and the webhook URL needs to be corrected after creation. + +### OAuth Token Storage (Three-Table Pattern) + +OAuth tokens are stored across three tables, NOT in `workspace_integrations.oauth_data` directly: + +| Table | What's Stored | +|-------|---------------| +| `workspace_integrations` | `oauth_data` JSON with `base_url`, `client_id`, `client_secret`, `instance_shared` flag; `resource_path` pointing to the variable | +| `variable` | Encrypted `access_token` (at the path stored in `resource_path`), linked to `account` via `account` column | +| `account` | `refresh_token`, keyed by `workspace_id` + `client` (service name) + `is_workspace_integration = true` | + +The `decrypt_oauth_data()` function in `lib.rs` assembles these into a unified struct: +```rust +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, // decrypted from variable + pub refresh_token: Option, // from account table + pub client_id: String, // from oauth_data or instance settings + pub client_secret: String, // from oauth_data or instance settings +} +``` + +Instance-level sharing: when `oauth_data.instance_shared == true`, `client_id` and `client_secret` are read from global settings instead of workspace_integrations. + +### URL Resolution + +The `resolve_endpoint()` helper handles both absolute and relative OAuth URLs: + +```rust +pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String { + if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() // Google: absolute URLs + } else { + format!("{}{}", base_url, endpoint) // Nextcloud: relative paths + } +} +``` + +### ServiceName Methods + +`ServiceName` is the central registry enum. Each variant must implement these match arms: + +| Method | Purpose | +|--------|---------| +| `as_str()` | Lowercase identifier (e.g., `"google"`) | +| `as_trigger_kind()` | Maps to `TriggerKind` enum | +| `as_job_trigger_kind()` | Maps to `JobTriggerKind` enum | +| `token_endpoint()` | OAuth token endpoint (relative or absolute) | +| `auth_endpoint()` | OAuth authorization endpoint | +| `oauth_scopes()` | Space-separated OAuth scopes | +| `resource_type()` | Resource type for token storage (e.g., `"gworkspace"`) | +| `extra_auth_params()` | Extra OAuth params (e.g., Google needs `access_type=offline`, `prompt=consent`) | +| `integration_service()` | Maps to the workspace integration service (usually `*self`) | +| `TryFrom` | Parse from string | +| `Display` | Delegates to `as_str()` | + +--- + +## Step-by-Step Implementation Guide + +### Step 1: Database Migration + +Create a new migration file: `backend/migrations/YYYYMMDDHHMMSS_newservice_trigger.up.sql` + +```sql +-- Add the service to the native_trigger_service enum +ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to TRIGGER_KIND enum (used for trigger tracking) +ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to job_trigger_kind enum (used for job tracking) +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'newservice'; +``` + +Also create the corresponding down migration. + +### Step 2: Update windmill-common Enums + +#### `backend/windmill-common/src/triggers.rs` + +Add variant to `TriggerKind` enum, and update `to_key()` and `fmt()` implementations. + +#### `backend/windmill-common/src/jobs.rs` + +Add variant to `JobTriggerKind` enum and update the `Display` implementation. + +### Step 3: Backend Service Module + +Create a new directory: `backend/windmill-native-triggers/src/newservice/` + +#### `mod.rs` - Type Definitions + +```rust +use serde::{Deserialize, Serialize}; + +pub mod external; +// pub mod routes; // Only if you need additional service-specific routes + +/// OAuth data deserialized from the three-table pattern. +/// The actual structure is built by decrypt_oauth_data() from variable + account + workspace_integrations. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NewServiceOAuthData { + pub base_url: String, // from workspace_integrations.oauth_data + pub access_token: String, // decrypted from variable table + pub refresh_token: Option, // from account table + // Note: client_id and client_secret are in OAuthConfig, not here + // unless the service needs them at runtime for API calls +} + +/// Configuration provided by user when creating/updating a trigger. +/// Stored as JSON in native_trigger.service_config. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceConfig { + // Service-specific configuration fields + pub folder_path: String, + pub file_filter: Option, +} + +/// Data retrieved from the external service about a trigger. +/// Returned by the get() method and shown in the UI. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceTriggerData { + pub folder_path: String, + pub file_filter: Option, + // Fields that shouldn't affect service_config comparison should use #[serde(skip_serializing)] +} + +/// Response from external service when creating a trigger/webhook. +#[derive(Debug, Deserialize)] +pub struct CreateTriggerResponse { + pub id: String, +} + +/// Handler struct (stateless, used for routing) +#[derive(Copy, Clone)] +pub struct NewService; +``` + +#### `external.rs` - External Trait Implementation + +```rust +use async_trait::async_trait; +use reqwest::Method; +use sqlx::PgConnection; +use std::collections::HashMap; +use windmill_common::{ + error::{Error, Result}, + BASE_URL, DB, +}; + +use crate::{ + generate_webhook_service_url, External, NativeTrigger, NativeTriggerData, ServiceName, + sync::{SyncError, TriggerSyncInfo}, +}; +use super::{NewService, NewServiceConfig, NewServiceOAuthData, NewServiceTriggerData, CreateTriggerResponse}; + +#[async_trait] +impl External for NewService { + type ServiceConfig = NewServiceConfig; + type TriggerData = NewServiceTriggerData; + type OAuthData = NewServiceOAuthData; + type CreateResponse = CreateTriggerResponse; + + const SERVICE_NAME: ServiceName = ServiceName::NewService; + const DISPLAY_NAME: &'static str = "New Service"; + const SUPPORT_WEBHOOK: bool = true; + const TOKEN_ENDPOINT: &'static str = "/oauth/token"; + const REFRESH_ENDPOINT: &'static str = "/oauth/token"; + const AUTH_ENDPOINT: &'static str = "/oauth/authorize"; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + // external_id is None during create (we get it from the response) + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + None, Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/create", oauth_data.base_url); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let response: CreateTriggerResponse = self + .http_client_request(&url, Method::POST, w_id, tx, db, None, Some(&payload)) + .await?; + + Ok(response) + } + + /// Update returns the resolved service_config as JSON. + /// For services using the update+get pattern, call self.get() and serialize. + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + Some(external_id), Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let _: serde_json::Value = self + .http_client_request(&url, Method::PUT, w_id, tx, db, None, Some(&payload)) + .await?; + + // Fetch back the updated state to get the resolved config + let trigger_data = self.get(w_id, oauth_data, external_id, db, tx).await?; + serde_json::to_value(&trigger_data) + .map_err(|e| Error::InternalErr(format!("Failed to serialize trigger data: {}", e))) + } + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + self.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, None, None).await + } + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()> { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let _: serde_json::Value = self + .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, None, None) + .await + .or_else(|e| match &e { + Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), + _ => Err(e), + })?; + Ok(()) + } + + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + match self.get(w_id, oauth_data, external_id, db, tx).await { + Ok(_) => Ok(true), + Err(Error::NotFound(_)) => Ok(false), + Err(e) => Err(e), + } + } + + /// Background maintenance. Choose the right pattern for your service: + /// - For services with queryable external state: use reconcile_with_external_state() + /// - For channel-based services with expiration: implement renewal logic + async fn maintain_triggers( + &self, + db: &DB, + workspace_id: &str, + triggers: &[NativeTrigger], + oauth_data: &Self::OAuthData, + synced: &mut Vec, + errors: &mut Vec, + ) { + // Option A: Reconcile with external state (Nextcloud pattern) + // Fetch all triggers from external service and compare with DB + let external_triggers = match self.list_all(workspace_id, oauth_data, db).await { + Ok(triggers) => triggers, + Err(e) => { + errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to list triggers: {}", e), + error_type: "api_error".to_string(), + }); + return; + } + }; + + // Convert to (external_id, config_json) pairs + let external_pairs: Vec<(String, serde_json::Value)> = external_triggers + .into_iter() + .map(|t| (t.id.clone(), serde_json::to_value(&t).unwrap_or_default())) + .collect(); + + crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, + ).await; + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option) { + (resp.id.clone(), None) + } + + // service_config_from_create_response: NOT overridden (returns None). + // This means the handler uses the update+get pattern after create. + // Override and return Some(...) to skip the update+get cycle (Google pattern). +} + +impl NewService { + /// Private helper to list all triggers from the external service. + async fn list_all( + &self, + w_id: &str, + oauth_data: &::OAuthData, + db: &DB, + ) -> Result::TriggerData>> { + // Implementation depends on the external service's API + todo!() + } +} +``` + +### Step 4: Update lib.rs Registry + +In `backend/windmill-native-triggers/src/lib.rs`: + +```rust +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod newservice; // <-- Add this + +// ServiceName enum - add variant: +pub enum ServiceName { + Nextcloud, + Google, + NewService, // <-- Add this +} + +// Then add match arms in ALL ServiceName methods: +// as_str(), as_trigger_kind(), as_job_trigger_kind(), token_endpoint(), +// auth_endpoint(), oauth_scopes(), resource_type(), extra_auth_params(), +// integration_service(), TryFrom, Display +``` + +### Step 5: Update handler.rs Routes + +In `backend/windmill-native-triggers/src/handler.rs`: + +```rust +pub fn generate_native_trigger_routers() -> Router { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + return router + .nest("/nextcloud", service_routes(NextCloud)) + .nest("/google", service_routes(Google)) + .nest("/newservice", service_routes(NewService)); // <-- Add this + } + // ... +} +``` + +### Step 6: Update sync.rs + +In `backend/windmill-native-triggers/src/sync.rs`: + +```rust +pub async fn sync_all_triggers(db: &DB) -> Result { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + + // ... existing service syncs ... + + // New service sync + let (service_name, result) = sync_service_triggers(db, NewService).await; + total_synced += result.synced_triggers.len(); + total_errors += result.errors.len(); + service_results.insert(service_name, result); + } + // ... +} +``` + +### Step 7: Frontend Service Registry + +In `frontend/src/lib/components/triggers/native/utils.ts`: + +Add to `NATIVE_TRIGGER_SERVICES`, `getTriggerIconName()`, and `getServiceIcon()`. + +### Step 8: Frontend Trigger Form Component + +Create: `frontend/src/lib/components/triggers/native/services/newservice/NewServiceTriggerForm.svelte` + +### Step 9: Frontend Icon Component + +Create: `frontend/src/lib/components/icons/NewServiceIcon.svelte` + +### Step 10: Update NativeTriggerEditor + +Check `frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte` to ensure it dynamically loads form components based on service name. + +### Step 11: Workspace Integration UI + +Add your service to the `supportedServices` map in `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte`: + +```typescript +const supportedServices: Record = { + // ... existing services ... + newservice: { + name: 'newservice', + displayName: 'New Service', + description: 'Connect to New Service for triggers', + icon: NewServiceIcon, + docsUrl: 'https://www.windmill.dev/docs/integrations/newservice', + requiresBaseUrl: false, // false for cloud services, true for self-hosted + setupInstructions: [ + 'Step 1: Create an OAuth app on the service', + 'Step 2: Configure the redirect URI shown below', + 'Step 3: Enter the client credentials below' + ] + } +} +``` + +### Step 12: Update `frontend/src/lib/components/triggers/utils.ts` + +Update ALL of these maps/functions: +1. `triggerIconMap` - import and add icon +2. `triggerDisplayNamesMap` - add display name +3. `triggerTypeOrder` in `sortTriggers()` - add type +4. `getLightConfig()` - add case for your service +5. `getTriggerLabel()` - add case for your service +6. `jobTriggerKinds` - add to array +7. `countPropertyMap` - add count property +8. `triggerSaveFunctions` - add save function + +### Step 13: Update TriggersBadge Component + +In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`: + +1. Import the icon +2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`) +3. Add to the `allTypes` array + +### Step 14: Update TriggersWrapper.svelte + +In `frontend/src/lib/components/triggers/TriggersWrapper.svelte`: + +Add a `{:else if selectedTrigger.type === 'yourservice'}` case that renders `` with the same props pattern as the existing native trigger cases (e.g., `nextcloud`). + +### Step 15: Update AddTriggersButton.svelte + +In `frontend/src/lib/components/triggers/AddTriggersButton.svelte`: + +1. Add `yourserviceAvailable` state variable +2. Add `setYourserviceState()` async function using `isServiceAvailable('yourservice', $workspaceStore!)` +3. Call it at module level +4. Add a dropdown entry to `addTriggerItems` with `hidden: !yourserviceAvailable` + +### Step 16: Update TriggersEditor.svelte Delete Handling + +In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: + +Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. + +### Step 17: Update OpenAPI Spec and Regenerate Types + +Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: + +```bash +cd frontend && npm run generate-backend-client +``` + +--- + +## Special Patterns + +### Unified Service with `trigger_type` (Google Pattern) + +When a single service handles multiple trigger types (e.g., Google Drive + Calendar share OAuth and API patterns), use a single `ServiceName` variant with a discriminator field: + +```rust +pub enum GoogleTriggerType { Drive, Calendar } + +pub struct GoogleServiceConfig { + pub trigger_type: GoogleTriggerType, + // Drive-specific fields (only used when trigger_type = Drive) + pub resource_id: Option, + pub resource_name: Option, + // Calendar-specific fields (only used when trigger_type = Calendar) + pub calendar_id: Option, + pub calendar_name: Option, + // Metadata set after creation + pub google_resource_id: Option, + pub expiration: Option, +} +``` + +Branch in trait methods based on `trigger_type`. Frontend uses a `ToggleButtonGroup` to switch between types. This keeps the codebase simpler (one service, one OAuth flow, one set of routes). + +See `backend/windmill-native-triggers/src/google/` for the reference implementation. + +### Skipping update+get After Create (Google Pattern) + +Override `service_config_from_create_response()` to return `Some(config)` when the external_id is known before the create call: + +```rust +fn service_config_from_create_response( + &self, + data: &NativeTriggerData, + resp: &Self::CreateResponse, +) -> Option { + // Clone input config, add metadata from response + let mut config = data.service_config.clone(); + config.google_resource_id = Some(resp.resource_id.clone()); + config.expiration = Some(resp.expiration.clone()); + Some(serde_json::to_value(&config).unwrap()) +} +``` + +### Services with Absolute OAuth Endpoints (Google) + +Unlike self-hosted services where OAuth endpoints are relative paths appended to `base_url`, services like Google have absolute URLs: + +```rust +// Nextcloud: relative paths +ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", +// Google: absolute URLs +ServiceName::Google => "https://oauth2.googleapis.com/token", +``` + +The `resolve_endpoint()` function handles both. For services with absolute endpoints: +- `base_url` can be empty +- `requiresBaseUrl: false` in the frontend workspace integration config +- Add `extra_auth_params()` if needed (Google requires `access_type=offline` and `prompt=consent`) + +### Channel-Based Push Notifications with Renewal (Google Pattern) + +For services using expiring watch channels instead of persistent webhooks: + +1. Store expiration in `service_config` (as part of `ServiceConfig`) +2. In `maintain_triggers()`, implement renewal logic instead of using `reconcile_with_external_state()`: + ```rust + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors) { + for trigger in triggers { + if should_renew_channel(trigger) { + self.renew_channel(db, trigger, oauth_data).await; + } + } + } + ``` +3. Renewal: best-effort stop old channel, create new one with same external_id, update service_config with new expiration +4. Google example: Drive channels expire in 24h (renew when <1h left), Calendar channels expire in 7 days (renew when <1 day left) + +### reconcile_with_external_state (Nextcloud Pattern) + +The reusable function in `sync.rs` compares external triggers with DB state: +- Triggers missing externally: sets error "Trigger no longer exists on external service" +- Triggers present externally: clears errors, updates service_config if it differs + +Usage in `maintain_triggers()`: +```rust +let external_pairs: Vec<(String, serde_json::Value)> = /* fetch from external */; +crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, +).await; +``` + +### Webhook Payload Processing + +Override `prepare_webhook()` to parse service-specific payloads into script/flow args: + +```rust +async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result { + let mut args = HashMap::new(); + args.insert("event_type".to_string(), Box::new(headers.get("x-event-type").cloned()) as _); + args.insert("payload".to_string(), Box::new(serde_json::from_str::(&body)?) as _); + Ok(PushArgsOwned { extra: None, args }) +} +``` + +Then register in `prepare_native_trigger_args()` in `lib.rs`: +```rust +pub async fn prepare_native_trigger_args(service_name, db, w_id, headers, body) -> Result> { + match service_name { + ServiceName::Google => { /* ... */ Ok(Some(args)) } + ServiceName::NewService => { /* ... */ Ok(Some(args)) } + ServiceName::Nextcloud => Ok(None), // Uses default body parsing + } +} +``` + +### Instance-Level OAuth Credentials + +When `workspace_integrations.oauth_data.instance_shared == true`, `decrypt_oauth_data()` reads `client_id` and `client_secret` from instance-level global settings instead of workspace-level. This allows admins to share OAuth app credentials across workspaces. + +The frontend handles this via the `generate_instance_connect_url` endpoint in `workspace_integrations.rs`. + +--- + +## Testing Checklist + +- [ ] Database migration runs successfully +- [ ] `cargo check -p windmill-native-triggers --features native_trigger` passes +- [ ] `npx svelte-check --threshold error` passes (in frontend/) +- [ ] Service appears in workspace integrations list +- [ ] OAuth flow completes successfully +- [ ] Can create a new trigger +- [ ] Can view trigger details +- [ ] Can update trigger configuration +- [ ] Can delete trigger +- [ ] Webhook receives and processes payloads +- [ ] Background sync works correctly (reconciliation or channel renewal) +- [ ] Error handling works (expired tokens, service unavailable) + +--- + +## Reference Implementations + +### Nextcloud (Self-Hosted, Update+Get Pattern) + +| File | Purpose | +|------|---------| +| `nextcloud/mod.rs` | Types: NextCloudOAuthData, NextcloudServiceConfig, NextCloudTriggerData | +| `nextcloud/external.rs` | External trait: uses update+get pattern, reconcile_with_external_state for sync | +| `nextcloud/routes.rs` | Additional route: `GET /events` | + +Key patterns: relative OAuth endpoints, base_url required, list_all + reconcile for sync, update returns JSON from get(). + +### Google (Cloud, Unified Service, Short Create) + +| File | Purpose | +|------|---------| +| `google/mod.rs` | Types: GoogleServiceConfig with trigger_type discriminator, GoogleTriggerType enum | +| `google/external.rs` | External trait: overrides service_config_from_create_response, channel renewal for sync | +| `google/routes.rs` | Additional routes: `GET /calendars`, `GET /drive/files`, `GET /drive/shared_drives` | + +Key patterns: absolute OAuth endpoints, empty base_url, trigger_type for Drive/Calendar, expiring watch channels with renewal, service_config_from_create_response skips update+get, get() reconstructs data from stored service_config (no external "get channel" API). diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md new file mode 100644 index 0000000000..2efcc4e0a6 --- /dev/null +++ b/.agents/skills/pr/SKILL.md @@ -0,0 +1,109 @@ +--- +name: pr +description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. +--- + +# Pull Request Skill + +Create a draft pull request with a clear title and explicit description of changes. + +## Instructions + +1. **Analyze branch changes**: Understand all commits since diverging from main +2. **Push to remote**: Ensure all commits are pushed +3. **Create draft PR**: Always open as draft for review before merging + +## PR Title Format + +Follow conventional commit format for the PR title: +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code restructuring +- `docs`: Documentation changes +- `chore`: Maintenance tasks +- `perf`: Performance improvements + +### Title Rules +- Keep under 70 characters +- Use lowercase, imperative mood +- No period at the end +- If `*_ee.rs` files were modified, prefix with `[ee]`: `[ee] : ` + +## PR Body Format + +The body MUST be explicit about what changed. Structure: + +```markdown +## Summary + + +## Changes +- +- +- + +## Test plan +- [ ] +- [ ] + +--- +Generated with [Claude Code](https://claude.com/claude-code) +``` + +## Execution Steps + +1. Run `git status` to check for uncommitted changes +2. Run `git log main..HEAD --oneline` to see all commits in this branch +3. Run `git diff main...HEAD` to see the full diff against main +4. Check if remote branch exists and is up to date: + ```bash + git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" + ``` +5. Push to remote if needed: `git push -u origin HEAD` +6. Create draft PR using gh CLI: + ```bash + gh pr create --draft --title ": " --body "$(cat <<'EOF' + ## Summary + + + ## Changes + - + - + + ## Test plan + - [ ] + - [ ] + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +7. Return the PR URL to the user + +## EE Companion PR (when `*_ee.rs` files were modified) + +The `*_ee.rs` files in the windmill repo are **symlinks** to `windmill-ee-private` — changes won't appear in `git diff` of the windmill repo. Instead, check the EE repo for uncommitted or unpushed changes. + +Follow the full EE PR workflow in `docs/enterprise.md`. The key PR-specific details: + +1. Find the EE repo/worktree: see "Finding the EE Repo" in `docs/enterprise.md` +2. Check for changes: `git -C status --short` + - If there are no changes in the EE repo, skip this entire section +3. Follow steps 1–5 from the "EE PR Workflow" in `docs/enterprise.md` +4. Create the companion PR (title does NOT get the `[ee]` prefix): + ```bash + gh pr create --draft --repo windmill-labs/windmill-ee-private --title ": " --body "$(cat <<'EOF' + Companion PR for windmill-labs/windmill# + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +5. Commit `ee-repo-ref.txt` and push the updated windmill branch diff --git a/.agents/skills/refine/SKILL.md b/.agents/skills/refine/SKILL.md new file mode 100644 index 0000000000..b96e97e8a2 --- /dev/null +++ b/.agents/skills/refine/SKILL.md @@ -0,0 +1,38 @@ +--- +name: refine +description: End-of-session reflection. Reviews friction encountered during the session and proposes updates to docs/ to capture lessons learned. +--- + +# Refine Skill + +Reflect on the current session and update documentation with lessons learned. + +## Instructions + +1. **Identify friction**: Review what happened in this session: + - Run `git diff main...HEAD --stat` to see what files were touched + - Think about: what was slow, what failed, what required multiple attempts, what information was missing or hard to find + +2. **Read current docs**: Read the docs that were relevant to this session: + - `docs/validation.md` + - `docs/enterprise.md` + - `docs/autonomous-mode.md` + - Any skills that were invoked + +3. **Propose updates**: For each piece of friction, decide if it warrants a doc update: + - **Missing knowledge**: Information you had to discover that should be documented + - **Wrong guidance**: Instructions that led you astray + - **Missing validation rule**: A check that should be in the validation matrix + - **New pattern**: A codebase pattern worth capturing for next time + +4. **Apply updates**: Edit the relevant `docs/` files. Keep changes minimal and specific — add only what would have saved time this session. + +5. **Report**: Summarize what was added/changed and why. + +## Rules + +- Only add knowledge confirmed by this session — no speculative additions +- Keep docs concise — add a line or two, not a paragraph +- If a whole new doc is needed, create it in `docs/` and add a pointer in `CLAUDE.md` +- Don't update skills unless a coding pattern was genuinely wrong +- Don't add things Claude already knows — only Windmill-specific knowledge diff --git a/.agents/skills/rust-backend/SKILL.md b/.agents/skills/rust-backend/SKILL.md new file mode 100644 index 0000000000..f0c52002bc --- /dev/null +++ b/.agents/skills/rust-backend/SKILL.md @@ -0,0 +1,107 @@ +--- +name: rust-backend +description: Rust coding guidelines for the Windmill backend. MUST use when writing or modifying Rust code in the backend directory. +--- + +# Windmill Rust Patterns + +Apply these Windmill-specific patterns when writing Rust code in `backend/`. + +## Error Handling + +Use `Error` from `windmill_common::error`. Return `Result` or `JsonResult`: + +```rust +use windmill_common::error::{Error, Result}; + +pub async fn get_job(db: &DB, id: Uuid) -> Result { + sqlx::query_as!(Job, "SELECT id, workspace_id FROM v2_job WHERE id = $1", id) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound("job not found".to_string()))?; +} +``` + +Never panic in library code. Reserve `.unwrap()` for compile-time guarantees. + +## SQLx Patterns + +**Never use `SELECT *`** — always list columns explicitly. Critical for backwards compatibility when workers lag behind API version: + +```rust +// Correct +sqlx::query_as!(Job, "SELECT id, workspace_id, path FROM v2_job WHERE id = $1", id) + +// Wrong — breaks when columns are added +sqlx::query_as!(Job, "SELECT * FROM v2_job WHERE id = $1", id) +``` + +Use batch operations to avoid N+1: + +```rust +// Preferred — single query with IN clause +sqlx::query!("SELECT ... WHERE id = ANY($1)", &ids[..]).fetch_all(db).await? +``` + +Use transactions for multi-step operations. Parameterize all queries. + +## JSON Handling + +Prefer `Box` over `serde_json::Value` when storing/passing JSON without inspection: + +```rust +pub struct Job { + pub args: Option>, +} +``` + +Only use `serde_json::Value` when you need to inspect or modify the JSON. + +## Serde Optimizations + +```rust +#[derive(Serialize, Deserialize)] +pub struct Job { + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, + #[serde(default)] + pub priority: i32, +} +``` + +## Async & Concurrency + +Never block the async runtime. Use `spawn_blocking` for CPU-intensive work: + +```rust +let result = tokio::task::spawn_blocking(move || expensive_computation(&data)).await?; +``` + +**Mutex selection**: Prefer `std::sync::Mutex` (or `parking_lot::Mutex`) for data protection. Only use `tokio::sync::Mutex` when holding locks across `.await` points. + +Use `tokio::sync::mpsc` (bounded) for channels. Avoid `std::thread::sleep` in async contexts. + +## Module Structure & Visibility + +- Use `pub(crate)` instead of `pub` when possible +- Place new code in the appropriate crate based on functionality +- API endpoints go in `windmill-api/src/` organized by domain +- Shared functionality goes in `windmill-common/src/` + +## Code Navigation + +Always use rust-analyzer LSP for go-to-definition, find-references, and type info. Do not guess at module paths. + +## Axum Handlers + +Destructure extractors directly in function signatures: + +```rust +async fn process_job( + Extension(db): Extension, + Path((workspace, job_id)): Path<(String, Uuid)>, + Query(pagination): Query, +) -> Result> { ... } +``` diff --git a/.agents/skills/svelte-frontend/SKILL.md b/.agents/skills/svelte-frontend/SKILL.md new file mode 100644 index 0000000000..57cac70302 --- /dev/null +++ b/.agents/skills/svelte-frontend/SKILL.md @@ -0,0 +1,80 @@ +--- +name: svelte-frontend +description: Svelte coding guidelines for the Windmill frontend. MUST use when writing or modifying code in the frontend directory. +--- + +# Windmill Svelte Patterns + +Apply these Windmill-specific patterns when writing Svelte code in `frontend/`. For general Svelte 5 syntax (runes, snippets, event handling), use the Svelte MCP server. + +## Windmill UI Components (MUST use) + +Always use Windmill's design-system components. Never use raw HTML elements. + +### Buttons — ` + {#snippet text()} - + {#if !isDeployed} Deploy the runnable to enable trigger creation {:else if cloudDisabled} @@ -127,7 +127,7 @@ Enter a valid config to {trigger?.isDraft ? 'deploy' : 'update'} the trigger {/if} - {/snippet} + {/snippet} {/if} diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index a33f14243b..9e0f58cd6c 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -20,6 +20,7 @@ import Label from '$lib/components/Label.svelte' import EmailTriggerEditorConfigSection from './EmailTriggerEditorConfigSection.svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { getHandlerType, handleConfigChange } from '../utils' import { untrack } from 'svelte' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -75,6 +76,9 @@ let drawer = $state(undefined) let initialConfig: NewEmailTrigger | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -181,6 +185,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Partial): Promise { @@ -236,7 +243,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return nCfg @@ -304,6 +313,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} diff --git a/frontend/src/lib/components/triggers/email/utils.ts b/frontend/src/lib/components/triggers/email/utils.ts index 314990b1bb..05c15bcb81 100644 --- a/frontend/src/lib/components/triggers/email/utils.ts +++ b/frontend/src/lib/components/triggers/email/utils.ts @@ -30,7 +30,9 @@ export async function saveEmailTriggerFromCfg( error_handler_path: emailCfg.error_handler_path, error_handler_args: emailCfg.error_handler_path ? emailCfg.error_handler_args : undefined, mode: emailCfg.mode, - retry: emailCfg.retry + retry: emailCfg.retry, + permissioned_as: emailCfg.permissioned_as, + preserve_permissioned_as: emailCfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index 6a11d6180b..1c13e74790 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -22,6 +22,7 @@ import GcpTriggerEditorConfigSection from './GcpTriggerEditorConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveGcpTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import { deepEqual } from 'fast-equals' @@ -57,6 +58,9 @@ let subscription_mode: SubscriptionMode = $state('create_update') let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let base_endpoint = $derived(`${window.location.origin}${base}`) let auto_acknowledge_msg = $state(true) let ack_deadline: number | undefined = $state() @@ -202,6 +206,9 @@ auto_acknowledge_msg = cfg?.auto_acknowledge_msg ?? true ack_deadline = cfg?.ack_deadline errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function updateTrigger(): Promise { @@ -246,7 +253,9 @@ error_handler_args, retry, auto_acknowledge_msg, - ack_deadline + ack_deadline, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -369,6 +378,15 @@

Loading...

{:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -454,9 +472,11 @@
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/gcp/utils.ts b/frontend/src/lib/components/triggers/gcp/utils.ts index ec8a93942e..b55d4ac50a 100644 --- a/frontend/src/lib/components/triggers/gcp/utils.ts +++ b/frontend/src/lib/components/triggers/gcp/utils.ts @@ -32,6 +32,8 @@ export async function saveGcpTriggerFromCfg( is_flow: cfg.is_flow, auto_acknowledge_msg: cfg.auto_acknowledge_msg, ack_deadline: cfg.ack_deadline, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } if (edit) { diff --git a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte index 272a0b707c..a0a6a95336 100644 --- a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte @@ -46,6 +46,7 @@ import RouteBodyTransformerOption from './RouteBodyTransformerOption.svelte' import TestingBadge from '../testingBadge.svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { getHandlerType, handleConfigChange } from '../utils' import autosize from '$lib/autosize' import { untrack } from 'svelte' @@ -122,6 +123,9 @@ let drawer = $state(undefined) let initialConfig: NewHttpTrigger | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'request_options' | 'error_handler' | 'retries' = $state('request_options') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -315,6 +319,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Partial): Promise { @@ -388,7 +395,9 @@ description: routeDescription, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return nCfg @@ -481,6 +490,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -697,11 +715,15 @@ {#if !is_static_website}
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/http/utils.ts b/frontend/src/lib/components/triggers/http/utils.ts index 1a00cfd081..a9ffca8e7e 100644 --- a/frontend/src/lib/components/triggers/http/utils.ts +++ b/frontend/src/lib/components/triggers/http/utils.ts @@ -61,7 +61,9 @@ export async function saveHttpRouteFromCfg( error_handler_path: routeCfg.error_handler_path, error_handler_args: routeCfg.error_handler_path ? routeCfg.error_handler_args : undefined, retry: routeCfg.retry, - mode: routeCfg.mode + mode: routeCfg.mode, + permissioned_as: routeCfg.permissioned_as, + preserve_permissioned_as: routeCfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte index 5f68f03705..dbf82de5db 100644 --- a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte @@ -15,6 +15,7 @@ import KafkaTriggersConfigSection from './KafkaTriggersConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveKafkaTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -87,6 +88,9 @@ let autoOffsetReset = $state('latest') let autoCommit = $state(true) let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let resetLoading = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -216,6 +220,9 @@ retry = cfg?.retry filters = cfg?.filters ?? [] errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -246,7 +253,9 @@ extra_perms: extra_perms, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -414,6 +423,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} @@ -531,11 +549,10 @@ Offsets will not be committed automatically. Use wmill.commit_kafka_offsets(trigger_path, topic, partition, offset) - in Python or wmill.commitKafkaOffsets(triggerPath, topic, partition, offset) in TypeScript with the values from the event payload. The consumer collects - all pending commits and commits the highest offset for each topic/partition - pair. + in Python or + wmill.commitKafkaOffsets(triggerPath, topic, partition, offset) in TypeScript + with the values from the event payload. The consumer collects all pending commits and + commits the highest offset for each topic/partition pair. {/if}
diff --git a/frontend/src/lib/components/triggers/kafka/utils.ts b/frontend/src/lib/components/triggers/kafka/utils.ts index 12ee49ac24..df05709d72 100644 --- a/frontend/src/lib/components/triggers/kafka/utils.ts +++ b/frontend/src/lib/components/triggers/kafka/utils.ts @@ -26,7 +26,9 @@ export async function saveKafkaTriggerFromCfg( filters: cfg.filters ?? [], auto_offset_reset: cfg.auto_offset_reset ?? 'latest', auto_commit: cfg.auto_commit ?? true, - ...errorHandlerAndRetries + ...errorHandlerAndRetries, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte index df5c7c21e1..7fa941b8ff 100644 --- a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte @@ -24,6 +24,7 @@ import MqttEditorConfigSection from './MqttEditorConfigSection.svelte' import type { Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveMqttTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -98,6 +99,9 @@ let isValid: boolean = $state(false) let initialConfig: Record | undefined = {} let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let errorHandlerSelected: ErrorHandler = $state('slack') let error_handler_path: string | undefined = $state() let error_handler_args: Record = $state({}) @@ -215,6 +219,9 @@ errorHandlerSelected = getHandlerType(error_handler_path ?? '') activateV5Options.topic_alias_maximum = Boolean(v5_config.topic_alias_maximum) activateV5Options.session_expiry_interval = Boolean(v5_config.session_expiry_interval) + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } catch (error) { sendUserToast(`Could not load mqtt trigger config: ${error.body}`, true) } @@ -251,7 +258,9 @@ is_flow, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -392,6 +401,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} @@ -471,10 +489,14 @@
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/mqtt/utils.ts b/frontend/src/lib/components/triggers/mqtt/utils.ts index 535c4caade..36b79d53a8 100644 --- a/frontend/src/lib/components/triggers/mqtt/utils.ts +++ b/frontend/src/lib/components/triggers/mqtt/utils.ts @@ -27,6 +27,8 @@ export async function saveMqttTriggerFromCfg( script_path: cfg.script_path, is_flow: cfg.is_flow, mode: cfg.mode, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte index a3c02c57dc..f02b8e37ab 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte @@ -14,6 +14,7 @@ import NatsTriggersConfigSection from './NatsTriggersConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveNatsTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -89,6 +90,9 @@ use_jetstream: false }) let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let isValid = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -201,6 +205,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -229,7 +236,9 @@ use_jetstream: natsCfg.use_jetstream, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -376,6 +385,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/nats/utils.ts b/frontend/src/lib/components/triggers/nats/utils.ts index 4376bef354..67336e390b 100644 --- a/frontend/src/lib/components/triggers/nats/utils.ts +++ b/frontend/src/lib/components/triggers/nats/utils.ts @@ -25,6 +25,8 @@ export async function saveNatsTriggerFromCfg( consumer_name: cfg.consumer_name, subjects: cfg.subjects, use_jetstream: cfg.use_jetstream, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte index 5ffe489a22..a4f460d96b 100644 --- a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte @@ -33,6 +33,7 @@ import { base } from '$lib/base' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import TestingBadge from '../testingBadge.svelte' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import TriggerRetriesAndErrorHandler from '../TriggerRetriesAndErrorHandler.svelte' @@ -113,6 +114,9 @@ let basic_mode = $derived(tab === 'basic') let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let creatingSlot: boolean = $state(false) let creatingPublication: boolean = $state(false) let pg14: boolean = $derived(postgresVersion.startsWith('14')) @@ -318,7 +322,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return cfg } @@ -339,6 +345,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -553,6 +562,15 @@
{/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/postgres/utils.ts b/frontend/src/lib/components/triggers/postgres/utils.ts index 3c01b53152..dfd69fa3da 100644 --- a/frontend/src/lib/components/triggers/postgres/utils.ts +++ b/frontend/src/lib/components/triggers/postgres/utils.ts @@ -126,6 +126,8 @@ export async function savePostgresTriggerFromCfg( publication_name: config.publication_name, publication: config.publication, mode: config.mode, + permissioned_as: config.permissioned_as, + preserve_permissioned_as: config.preserve_permissioned_as, ...errorHandlerAndRetries } if (edit) { diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 55f825b40c..b8d4ee59f9 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -38,6 +38,7 @@ import { handleConfigChange } from '../utils' import TextInput from '$lib/components/text_input/TextInput.svelte' import { twMerge } from 'tailwind-merge' + import PermissionedAsLine from '../PermissionedAsLine.svelte' let { useDrawer = true, @@ -111,6 +112,9 @@ let isValid = $state(true) let allowSchedule = $derived(isValid && validCRON && script_path != '') let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) const saveDisabled = $derived( !allowSchedule || @@ -507,6 +511,9 @@ extraPerms = cfg.extra_perms ?? {} can_write = canWrite(cfg.path, cfg.extra_perms, $userStore) tag = cfg.tag + permissionedAs = cfg.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false loading = false } @@ -605,7 +612,9 @@ paused_until: paused_until, cron_version: cronVersion, extra_perms: extraPerms, - dynamic_skip: dynamicSkipPath + dynamic_skip: dynamicSkipPath, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -682,6 +691,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
@@ -913,12 +931,16 @@
{#snippet header()} - + {/snippet} {@render errorHandler()}
diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index 1d9e1d783f..49b0a8182b 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -20,6 +20,7 @@ import Required from '$lib/components/Required.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveSqsTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -88,6 +89,9 @@ let isValid = $state(false) let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') let error_handler_path: string | undefined = $state() @@ -189,6 +193,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } catch (error) { sendUserToast(`Could not load SQS trigger config: ${error.body}`, true) } @@ -223,7 +230,9 @@ mode, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -362,6 +371,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/sqs/utils.ts b/frontend/src/lib/components/triggers/sqs/utils.ts index 1481639f74..4bca536b2f 100644 --- a/frontend/src/lib/components/triggers/sqs/utils.ts +++ b/frontend/src/lib/components/triggers/sqs/utils.ts @@ -25,6 +25,8 @@ export async function saveSqsTriggerFromCfg( message_attributes: cfg.message_attributes, aws_auth_resource_type: cfg.aws_auth_resource_type, mode: cfg.mode, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index 56d4591cd1..2eece07a11 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -31,6 +31,7 @@ import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveWebsocketTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -105,6 +106,9 @@ let showLoading = $state(false) let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let isValid = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -234,6 +238,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } function getSaveCfg() { @@ -250,7 +257,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -438,6 +447,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -688,9 +706,11 @@
{#snippet header()} - 0 } - ]} /> + 0 }]} + /> {/snippet}
diff --git a/frontend/src/lib/components/triggers/websocket/utils.ts b/frontend/src/lib/components/triggers/websocket/utils.ts index d71a41faa8..5cd27c3b72 100644 --- a/frontend/src/lib/components/triggers/websocket/utils.ts +++ b/frontend/src/lib/components/triggers/websocket/utils.ts @@ -29,7 +29,9 @@ export async function saveWebsocketTriggerFromCfg( url_runnable_args: triggerCfg.url_runnable_args, can_return_message: triggerCfg.can_return_message, can_return_error_result: triggerCfg.can_return_error_result, - ...errorHandlerAndRetries + ...errorHandlerAndRetries, + permissioned_as: triggerCfg.permissioned_as, + preserve_permissioned_as: triggerCfg.preserve_permissioned_as } try { if (edit) { From 5089a458819abbc6f241bc354bebb91520bd1a52 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 24 Mar 2026 14:27:09 +0100 Subject: [PATCH 16/48] feat: add summary field for native triggers (#8476) * feat: add summary field for native triggers (nextcloud, google) Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add nullable to NativeTriggerData summary in openapi spec Co-Authored-By: Claude Opus 4.6 (1M context) * fix: include summary in native trigger search index Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...bc8a88935c613bd213b7156299811377db8e.json} | 7 +++--- ...ca5b322cc130a35a91f8b8854f5ebdf25ad2.json} | 12 +++++++--- ...5b1724c36031da39d53ae6c329a479bdf8aa.json} | 12 +++++++--- ...70e63dab12a1383a0855d105c061e4e4ca48.json} | 12 +++++++--- ...e15bb9edfa1dc9052f8a829e486b7334d708.json} | 7 +++--- ...0323000000_native_trigger_summary.down.sql | 1 + ...260323000000_native_trigger_summary.up.sql | 1 + .../tests/native_triggers.rs | 2 ++ backend/windmill-api/openapi.yaml | 12 ++++++++++ .../windmill-native-triggers/src/handler.rs | 2 ++ backend/windmill-native-triggers/src/lib.rs | 24 +++++++++++++------ cli/src/commands/trigger/trigger.ts | 4 ++++ .../native/NativeTriggerEditor.svelte | 24 ++++++++++++++++++- .../triggers/native/NativeTriggerTable.svelte | 17 ++++++++++--- .../lib/components/triggers/native/utils.ts | 5 ++-- frontend/src/lib/components/triggers/utils.ts | 9 +++++-- 16 files changed, 121 insertions(+), 30 deletions(-) rename backend/.sqlx/{query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json => query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json} (57%) rename backend/.sqlx/{query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json => query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json} (84%) rename backend/.sqlx/{query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json => query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json} (82%) rename backend/.sqlx/{query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json => query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json} (67%) rename backend/.sqlx/{query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json => query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json} (66%) create mode 100644 backend/migrations/20260323000000_native_trigger_summary.down.sql create mode 100644 backend/migrations/20260323000000_native_trigger_summary.up.sql diff --git a/backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json b/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json similarity index 57% rename from backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json rename to backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json index 69af249a3f..577b2b3826 100644 --- a/backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json +++ b/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, error = NULL, updated_at = NOW()\n ", + "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", "describe": { "columns": [], "parameters": { @@ -21,10 +21,11 @@ "Varchar", "Bool", "Varchar", - "Jsonb" + "Jsonb", + "Varchar" ] }, "nullable": [] }, - "hash": "6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7" + "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e" } diff --git a/backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json b/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json similarity index 84% rename from backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json rename to backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json index a90b2b8398..66b8c431da 100644 --- a/backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json +++ b/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -91,8 +96,9 @@ true, true, false, - false + false, + true ] }, - "hash": "bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb" + "hash": "15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2" } diff --git a/backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json b/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json similarity index 82% rename from backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json rename to backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json index 492fffe8be..ea974dae42 100644 --- a/backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json +++ b/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -92,8 +97,9 @@ true, true, false, - false + false, + true ] }, - "hash": "1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce" + "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa" } diff --git a/backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json b/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json similarity index 67% rename from backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json rename to backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json index 660c855622..4109c7deaf 100644 --- a/backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json +++ b/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -94,8 +99,9 @@ true, true, false, - false + false, + true ] }, - "hash": "a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e" + "hash": "b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48" } diff --git a/backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json b/backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json similarity index 66% rename from backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json rename to backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json index 706fa0c9ea..b40a643d50 100644 --- a/backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json +++ b/backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE native_trigger\n SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $5\n AND service_name = $6\n AND external_id = $7\n ", + "query": "\n UPDATE native_trigger\n SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, summary = $8, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $5\n AND service_name = $6\n AND external_id = $7\n ", "describe": { "columns": [], "parameters": { @@ -21,10 +21,11 @@ } } }, - "Text" + "Text", + "Varchar" ] }, "nullable": [] }, - "hash": "40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e" + "hash": "bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708" } diff --git a/backend/migrations/20260323000000_native_trigger_summary.down.sql b/backend/migrations/20260323000000_native_trigger_summary.down.sql new file mode 100644 index 0000000000..7c7e63101d --- /dev/null +++ b/backend/migrations/20260323000000_native_trigger_summary.down.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger DROP COLUMN IF EXISTS summary; diff --git a/backend/migrations/20260323000000_native_trigger_summary.up.sql b/backend/migrations/20260323000000_native_trigger_summary.up.sql new file mode 100644 index 0000000000..7989a6fcd3 --- /dev/null +++ b/backend/migrations/20260323000000_native_trigger_summary.up.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger ADD COLUMN summary VARCHAR(1000); diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 519e1f5262..e0a2e51ab5 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -422,6 +422,7 @@ async fn test_delete_integration_full_cascade(db: Pool) -> anyhow::Res "ext-1", &trigger_config, json!({"triggerType": "drive"}), + None, ) .await?; @@ -511,6 +512,7 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( "ext-1", &trigger_config, json!({"triggerType": "drive"}), + None, ) .await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 930cb94949..65eede32b2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -24592,6 +24592,10 @@ components: type: string nullable: true description: Error message if the trigger is in an error state + summary: + type: string + nullable: true + description: Short summary to be displayed when listed required: - external_id - workspace_id @@ -24626,6 +24630,10 @@ components: type: string nullable: true description: Error message if the trigger is in an error state + summary: + type: string + nullable: true + description: Short summary to be displayed when listed external_data: type: object description: Configuration data from the external service @@ -24718,6 +24726,10 @@ components: type: object description: Service-specific configuration (e.g., event types, filters) additionalProperties: true + summary: + type: string + nullable: true + description: Short summary to be displayed when listed required: - script_path - is_flow diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index b66a8ae3e0..cd6e1a8100 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -181,6 +181,7 @@ async fn create_native_trigger( &external_id, &config, service_config, + data.summary.as_deref(), ) .await?; @@ -304,6 +305,7 @@ async fn update_native_trigger_handler( &external_id, &config, service_config, + data.summary.as_deref(), ) .await?; diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 25234ff3f0..7853bb63e3 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -195,6 +195,7 @@ pub struct NativeTrigger { pub error: Option, pub created_at: DateTime, pub updated_at: DateTime, + pub summary: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -209,6 +210,7 @@ pub struct NativeTriggerData { pub script_path: String, pub is_flow: bool, pub service_config: C, + pub summary: Option, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -821,6 +823,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> external_id: &str, config: &NativeTriggerConfig, service_config: C, + summary: Option<&str>, ) -> Result<()> { use windmill_common::auth::hash_token; @@ -835,12 +838,13 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> script_path, is_flow, webhook_token_hash, - service_config + service_config, + summary ) VALUES ( - $1, $2, $3, $4, $5, $6, $7 + $1, $2, $3, $4, $5, $6, $7, $8 ) ON CONFLICT (external_id, workspace_id, service_name) - DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, error = NULL, updated_at = NOW() + DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW() "#, external_id, workspace_id, @@ -849,6 +853,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> config.is_flow, webhook_token_hash, sqlx::types::Json(service_config) as _, + summary, ) .execute(db) .await?; @@ -863,6 +868,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres external_id: &str, config: &NativeTriggerConfig, service_config: Option<&RawValue>, + summary: Option<&str>, ) -> Result<()> { use windmill_common::auth::hash_token; @@ -871,7 +877,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres sqlx::query!( r#" UPDATE native_trigger - SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, error = NULL, updated_at = NOW() + SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, summary = $8, error = NULL, updated_at = NOW() WHERE workspace_id = $5 AND service_name = $6 @@ -884,6 +890,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres workspace_id, service_name as ServiceName, external_id, + summary, ) .execute(db) .await?; @@ -934,7 +941,8 @@ pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( service_config, error, created_at, - updated_at + updated_at, + summary FROM native_trigger WHERE @@ -972,7 +980,8 @@ pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = P service_config, error, created_at, - updated_at + updated_at, + summary FROM native_trigger WHERE @@ -1018,7 +1027,8 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> nt.service_config, nt.error, nt.created_at, - nt.updated_at + nt.updated_at, + nt.summary FROM native_trigger nt WHERE diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index be645a83fa..11f68bea96 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -231,6 +231,7 @@ export async function pushNativeTrigger( is_flow: result.is_flow, service_config: result.service_config, error: result.error, + summary: result.summary, }; log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`); } catch { @@ -243,6 +244,7 @@ export async function pushNativeTrigger( script_path: localTrigger.script_path, is_flow: localTrigger.is_flow, service_config: localTrigger.service_config, + summary: localTrigger.summary, }; if (remoteTrigger) { @@ -251,11 +253,13 @@ export async function pushNativeTrigger( script_path: localTrigger.script_path, is_flow: localTrigger.is_flow, service_config: localTrigger.service_config, + summary: localTrigger.summary, }; const remoteCompare = { script_path: remoteTrigger.script_path, is_flow: remoteTrigger.is_flow, service_config: remoteTrigger.service_config, + summary: remoteTrigger.summary, }; if (isSuperset(localCompare, remoteCompare)) { diff --git a/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte b/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte index 7334f0a62e..b6e28875ba 100644 --- a/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte +++ b/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte @@ -11,6 +11,7 @@ import { usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { canWrite, emptyString, sendUserToast } from '$lib/utils' import { Button } from '$lib/components/common' + import TextInput from '$lib/components/text_input/TextInput.svelte' import Drawer from '$lib/components/common/drawer/Drawer.svelte' import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' import { Loader2, Save } from 'lucide-svelte' @@ -94,6 +95,7 @@ let initialScriptPath = $state('') let fixedScriptPath = $state('') let isFlow = $state(false) + let summary = $state('') let externalId = $state(null) let can_write = $state(true) let originalConfig = $state | undefined>(undefined) @@ -123,6 +125,7 @@ can_write = true originalConfig = undefined initialConfig = undefined + summary = '' } export function openRecreate(nativeTrigger: ExtendedNativeTrigger) { @@ -146,6 +149,7 @@ can_write = true originalConfig = undefined initialConfig = undefined + summary = nativeTrigger.summary ?? '' } export async function openEdit( @@ -182,6 +186,7 @@ scriptPath = fullTrigger.script_path initialScriptPath = fullTrigger.script_path can_write = canWrite(fullTrigger.script_path, {}, $userStore) + summary = fullTrigger.summary ?? '' externalData = fullTrigger.external_data // Apply default values if provided (for draft triggers) @@ -203,7 +208,8 @@ return { script_path: scriptPath, is_flow: isFlow, - service_config: serviceConfig + service_config: serviceConfig, + summary: summary !== '' ? summary : undefined } } @@ -386,6 +392,22 @@ {/if}
+
+
+ +
+
+ {#if !hideTarget}

diff --git a/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte b/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte index 6314c12e82..8f8b84cd57 100644 --- a/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte +++ b/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte @@ -98,20 +98,31 @@ {@html trigger.marked} {:else} - {trigger.script_path} + {trigger.summary || trigger.script_path} {/if}

+ {#if trigger.summary} +
+ {trigger.script_path} +
+ {/if} {#if service === 'google'} {@const triggerType = trigger.service_config?.triggerType} {@const resourceName = trigger.service_config?.resourceName} {@const calendarName = trigger.service_config?.calendarName} -
+
{#if triggerType === 'calendar'} Calendar: {calendarName || trigger.service_config?.calendarId || ''} {:else} - Drive: {resourceName ? resourceName : trigger.service_config?.resourceId ? trigger.service_config.resourceId : 'All changes'} + Drive: {resourceName + ? resourceName + : trigger.service_config?.resourceId + ? trigger.service_config.resourceId + : 'All changes'} {/if}
{/if} diff --git a/frontend/src/lib/components/triggers/native/utils.ts b/frontend/src/lib/components/triggers/native/utils.ts index 34b1221b26..92082882e4 100644 --- a/frontend/src/lib/components/triggers/native/utils.ts +++ b/frontend/src/lib/components/triggers/native/utils.ts @@ -111,7 +111,7 @@ export function validateCommonFields(config: Record): Record Date: Tue, 24 Mar 2026 07:27:41 -0600 Subject: [PATCH 17/48] allow modern email TLDs in superadmin setup form (#8472) --- .../(root)/(logged)/user/(user)/instance_settings/+page.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte index d80e7fda85..66d9568c09 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte @@ -143,7 +143,7 @@ } } - const emailPattern = /^[\w-.]+@([\w-]+\.)+[\w-]{2,4}$/ + const emailPattern = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/ let emailValid = $derived(emailPattern.test(newEmail)) let passwordValid = $derived(newPassword.length >= 2) let accountFormValid = $derived(emailValid && passwordValid) From 6d63d9973d2f5bfb86e691b00b5a6495b2ac305b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 13:31:06 +0000 Subject: [PATCH 18/48] chore(main): release 1.663.0 (#8465) * chore(main): release 1.663.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 25 ++ backend/Cargo.lock | 255 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 170 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07701ce7ce..c8aa1a9648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.663.0](https://github.com/windmill-labs/windmill/compare/v1.662.0...v1.663.0) (2026-03-24) + + +### Features + +* add summary field for native triggers ([#8476](https://github.com/windmill-labs/windmill/issues/8476)) ([5089a45](https://github.com/windmill-labs/windmill/commit/5089a458819abbc6f241bc354bebb91520bd1a52)) +* add typed request body to OpenAPI spec generation ([#8481](https://github.com/windmill-labs/windmill/issues/8481)) ([37ebaf4](https://github.com/windmill-labs/windmill/commit/37ebaf4d0ac342703498733f97778a552f979f6a)) +* **cli:** better stale scripts detection [#3](https://github.com/windmill-labs/windmill/issues/3) ([#8480](https://github.com/windmill-labs/windmill/issues/8480)) ([9643006](https://github.com/windmill-labs/windmill/commit/9643006f1e90b991b334bb58caf62301bc26d09d)) +* Debounce node ([#8324](https://github.com/windmill-labs/windmill/issues/8324)) ([5d1c54d](https://github.com/windmill-labs/windmill/commit/5d1c54d9b33d6ff6f2c98481a2740d1e7629cdfa)) +* surface permissioned_as selector in trigger editor UI ([#8475](https://github.com/windmill-labs/windmill/issues/8475)) ([f035b53](https://github.com/windmill-labs/windmill/commit/f035b538bbd786445526339f88be8f33a3628105)) + + +### Bug Fixes + +* clean up stale dependency map entries for renamed scripts ([#8492](https://github.com/windmill-labs/windmill/issues/8492)) ([47c0c36](https://github.com/windmill-labs/windmill/commit/47c0c363f4fc1d9af7efd07ea172e32989ce50d2)) +* **cli:** add Svelte 5 event delegation guidance and safe push to raw-app skill ([#8466](https://github.com/windmill-labs/windmill/issues/8466)) ([911df95](https://github.com/windmill-labs/windmill/commit/911df958e78d2dab9823dfa7d7e5c9824fc2d565)) +* Fix worker panic when job_isolation changed to unshare at runtime ([#8490](https://github.com/windmill-labs/windmill/issues/8490)) ([cbe47c0](https://github.com/windmill-labs/windmill/commit/cbe47c0b6c22f79452d020777e481ee26970f25b)) +* improve SQS retries ([3c8d351](https://github.com/windmill-labs/windmill/commit/3c8d351c9722a089133871019d27cf3bc3cdc159)) +* Move database manager SQL queries to backend ([#8306](https://github.com/windmill-labs/windmill/issues/8306)) ([aa30fd2](https://github.com/windmill-labs/windmill/commit/aa30fd252dcf40233d191c43a6293fb9feabf010)) +* prevent SQL injection in job query parameters ([#8494](https://github.com/windmill-labs/windmill/issues/8494)) ([54f5a19](https://github.com/windmill-labs/windmill/commit/54f5a19377e9df712e18f85f896e21b1776981ed)) +* respect NO_COLOR env variable for stdout log output ([#8483](https://github.com/windmill-labs/windmill/issues/8483)) ([f329ee7](https://github.com/windmill-labs/windmill/commit/f329ee7aaefbae0ad344743c40825440a936bd30)) +* show effective isolation level on workers page ([#8491](https://github.com/windmill-labs/windmill/issues/8491)) ([37886ed](https://github.com/windmill-labs/windmill/commit/37886edda1443293806a9b1b810196b72e076b12)) +* skip debounce arg accumulation when batch table is empty (CE) ([#8485](https://github.com/windmill-labs/windmill/issues/8485)) ([010753c](https://github.com/windmill-labs/windmill/commit/010753c73ac85237af50acadf9c08567b1bc993c)) +* stop_after_if with empty error_message prevents flow from stopping ([#8464](https://github.com/windmill-labs/windmill/issues/8464)) ([1503bf9](https://github.com/windmill-labs/windmill/commit/1503bf948e3340b8a6933d71885f8f2cb8dc1867)) + ## [1.662.0](https://github.com/windmill-labs/windmill/compare/v1.661.0...v1.662.0) (2026-03-20) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e40478cc4b..f044478b25 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" dependencies = [ "rustversion", ] @@ -2536,12 +2536,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -4887,19 +4881,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_more" -version = "0.99.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version 0.4.1", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "1.0.0" @@ -4927,6 +4908,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.117", + "unicode-xid", ] [[package]] @@ -7419,13 +7401,13 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "2d72a21f6a71a6c4c3160e095e8925861f5119dd26ef71acee1b9146f74f76c8" dependencies = [ - "socket2 0.5.10", + "socket2 0.6.3", "widestring", - "windows-sys 0.48.0", + "windows-sys 0.61.2", "winreg", ] @@ -7446,9 +7428,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" dependencies = [ "memchr", "serde", @@ -7538,7 +7520,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.1", "log", "thiserror 1.0.69", "walkdir", @@ -7547,9 +7529,31 @@ dependencies = [ [[package]] name = "jni-sys" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] [[package]] name = "jobserver" @@ -8308,9 +8312,9 @@ dependencies = [ [[package]] name = "malachite" -version = "0.4.18" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6ecab92657eb234bfe98abd0b17920772c6b14ce69256950142e2eb36d000b" +checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" dependencies = [ "malachite-base", "malachite-nz", @@ -8331,11 +8335,11 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17703a19c80bbdd0b7919f0f104f3b0597f7de4fc4e90a477c15366a5ba03faa" +checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" dependencies = [ - "derive_more 0.99.20", + "derive_more 1.0.0", "malachite", "num-integer", "num-traits", @@ -8610,9 +8614,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "async-lock", "crossbeam-channel", @@ -8826,7 +8830,7 @@ version = "0.5.0+25.2.9519653" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" dependencies = [ - "jni-sys", + "jni-sys 0.3.1", ] [[package]] @@ -11663,7 +11667,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.9", + "rustls-webpki 0.103.10", "subtle", "zeroize", ] @@ -11747,7 +11751,7 @@ dependencies = [ "rustls 0.23.35", "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki 0.103.9", + "rustls-webpki 0.103.10", "security-framework 3.6.0", "security-framework-sys", "webpki-root-certs 1.0.6", @@ -11795,9 +11799,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring 0.17.14", @@ -13877,12 +13881,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -14468,9 +14472,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" dependencies = [ "winnow 1.0.0", ] @@ -15742,7 +15746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -15818,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15831,7 +15835,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "argon2", @@ -15972,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15995,7 +15999,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16008,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16034,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.662.0" +version = "1.663.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16044,7 +16048,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16061,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16084,7 +16088,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16107,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16123,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16143,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16163,7 +16167,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16177,7 +16181,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -16205,7 +16209,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16230,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16248,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16260,6 +16264,7 @@ dependencies = [ "serde_json", "serde_yml", "sqlx", + "tracing", "url", "windmill-api-auth", "windmill-common", @@ -16269,7 +16274,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16289,7 +16294,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16319,7 +16324,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16346,7 +16351,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.662.0" +version = "1.663.0" dependencies = [ "lazy_static", "serde", @@ -16358,7 +16363,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.662.0" +version = "1.663.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16381,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16395,7 +16400,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16426,7 +16431,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.662.0" +version = "1.663.0" dependencies = [ "chrono", "lazy_static", @@ -16440,7 +16445,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16459,7 +16464,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.662.0" +version = "1.663.0" dependencies = [ "aes-gcm", "anyhow", @@ -16559,7 +16564,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.662.0" +version = "1.663.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16578,7 +16583,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.662.0" +version = "1.663.0" dependencies = [ "regex", "serde", @@ -16593,7 +16598,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16617,7 +16622,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "futures", @@ -16634,7 +16639,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.662.0" +version = "1.663.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16650,7 +16655,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -16671,7 +16676,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -16702,7 +16707,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-oauth2", @@ -16726,7 +16731,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-stream", @@ -16760,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "futures", @@ -16778,7 +16783,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.662.0" +version = "1.663.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16787,7 +16792,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16799,7 +16804,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde_json", @@ -16811,7 +16816,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "gosyn", @@ -16823,7 +16828,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16835,7 +16840,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde_json", @@ -16847,7 +16852,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "nu-parser", @@ -16858,7 +16863,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16869,7 +16874,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16881,7 +16886,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16892,7 +16897,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -16914,7 +16919,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16928,7 +16933,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16945,7 +16950,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16958,7 +16963,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde", @@ -16970,7 +16975,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16988,7 +16993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17004,7 +17009,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17020,7 +17025,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde", @@ -17031,7 +17036,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -17068,7 +17073,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "const_format", @@ -17106,7 +17111,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.662.0" +version = "1.663.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17117,7 +17122,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -17146,7 +17151,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17169,7 +17174,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17202,7 +17207,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17222,7 +17227,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17256,7 +17261,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17291,7 +17296,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17314,7 +17319,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17338,7 +17343,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -17362,7 +17367,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17397,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17425,7 +17430,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17448,7 +17453,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17467,7 +17472,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-once-cell", @@ -17574,7 +17579,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.662.0" +version = "1.663.0" dependencies = [ "bytes", "futures", @@ -18189,12 +18194,12 @@ checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" [[package]] name = "winreg" -version = "0.50.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 744518e703..6dc821689c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.662.0" +version = "1.663.0" authors.workspace = true edition.workspace = true @@ -82,7 +82,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.662.0" +version = "1.663.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 65eede32b2..8ef9caf936 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.662.0 + version: 1.663.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index b235377979..f93b61c6ee 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.662.0"; +export const VERSION = "v1.663.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 8804599f4a..83e9c1bbc8 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { workspaceAdd, }; -export const VERSION = "1.662.0"; +export const VERSION = "1.663.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 10be9acea0..792e303b6e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 1dc86caae8..39bb3b6365 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 5e0a6f2c6c..130d95a820 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.662.0" +wmill = ">=1.663.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index fa0e2e22cc..279b4589da 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.662.0 + version: 1.663.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index a4fa958a3d..c84a6d49f7 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.662.0' + ModuleVersion = '1.663.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 6e1bbf495b..92fbaf56d1 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.662.0" +version = "1.663.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index d1f9ea1b1a..5435328c88 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.662.0", + "version": "1.663.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8a37e00e0c..bd66c04c51 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.662.0", + "version": "1.663.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index fd3cab66e3..eb4feec596 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.662.0 +1.663.0 From 7f27d996accb3c3b471d1c50df397867d89c738a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 15:16:10 +0000 Subject: [PATCH 19/48] fix: create parent dirs and accept 'python' alias in script bootstrap (#8497) Co-authored-by: Claude Opus 4.5 --- cli/src/commands/script/script.ts | 17 +++++++-- cli/test/standalone_commands.test.ts | 57 ++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 14f3155ff7..10015db776 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1,7 +1,7 @@ import { GlobalOptions } from "../../types.ts"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { readFile, writeFile, stat } from "node:fs/promises"; +import { readFile, writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; @@ -1069,16 +1069,22 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) { } } +const languageAliases: Record = { + python: "python3", +}; + async function bootstrap( opts: GlobalOptions & { summary: string; description: string }, scriptPath: string, - language: ScriptLanguage + language: ScriptLanguage | string ) { if (!validatePath(scriptPath)) { return; } - const scriptInitialCode = scriptBootstrapCode[language]; + const resolvedLanguage = (languageAliases[language] ?? language) as ScriptLanguage; + + const scriptInitialCode = scriptBootstrapCode[resolvedLanguage]; if (scriptInitialCode === undefined) { throw new Error("Language unknown"); } @@ -1086,7 +1092,7 @@ async function bootstrap( const config = await readConfigFile(); const extension = filePathExtensionFromContentType( - language, + resolvedLanguage, config.defaultTs ); const scriptCodeFileFullPath = scriptPath + extension; @@ -1118,6 +1124,9 @@ async function bootstrap( yamlOptions ); + const parentDir = path.dirname(scriptCodeFileFullPath); + await mkdir(parentDir, { recursive: true }); + await writeFile(scriptCodeFileFullPath, scriptInitialCode, { flag: 'wx', encoding: 'utf-8', }); diff --git a/cli/test/standalone_commands.test.ts b/cli/test/standalone_commands.test.ts index 106e4aaeb1..1d75680354 100644 --- a/cli/test/standalone_commands.test.ts +++ b/cli/test/standalone_commands.test.ts @@ -409,6 +409,63 @@ describe("script bootstrap command", () => { }); }); + test("accepts 'python' as alias for python3", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`, + "utf-8" + ); + + await mkdir(join(tempDir, "f", "test"), { recursive: true }); + + const result = await backend.runCLICommand( + ["script", "bootstrap", "f/test/py_alias_script", "python"], + tempDir + ); + + expect(result.code).toEqual(0); + + const codeStat = await stat(join(tempDir, "f/test/py_alias_script.py")); + expect(codeStat.isFile()).toBe(true); + + const metaStat = await stat( + join(tempDir, "f/test/py_alias_script.script.yaml") + ); + expect(metaStat.isFile()).toBe(true); + }); + }); + + test("creates parent directories automatically", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`, + "utf-8" + ); + + // Do NOT pre-create f/test — bootstrap should create it + const result = await backend.runCLICommand( + ["script", "bootstrap", "f/test/auto_dir_script", "bun"], + tempDir + ); + + expect(result.code).toEqual(0); + + const codeStat = await stat(join(tempDir, "f/test/auto_dir_script.ts")); + expect(codeStat.isFile()).toBe(true); + + const metaStat = await stat( + join(tempDir, "f/test/auto_dir_script.script.yaml") + ); + expect(metaStat.isFile()).toBe(true); + }); + }); + test("creates Go script files", async () => { await withTestBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend); From 3c34d19813752c7c3d718ac30a60266942b10909 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 15:41:39 +0000 Subject: [PATCH 20/48] escape env var values in nativets/bun JS string interpolation (#8500) Co-authored-by: Claude Opus 4.5 --- backend/windmill-worker/src/bun_executor.rs | 5 ++++- backend/windmill-worker/src/worker.rs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index fb07806fd6..f0ed8793a7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3008,7 +3008,10 @@ pub fn build_nativets_env_code( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() - .map(|(k, v)| format!("process.env['{}'] = '{}';", k, v)) + .map(|(k, v)| { + let escaped = v.replace('\\', "\\\\").replace('\'', "\\'").replace('\n', "\\n").replace('\r', "\\r"); + format!("process.env['{}'] = '{}';", k, escaped) + }) .collect::>() .join("\n") ) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 03e97a6ac1..b698f351c6 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4493,7 +4493,10 @@ pub async fn run_language_executor( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() - .map(|(k, v)| format!("const {} = '{}';\nprocess.env['{}'] = '{}';\n", k, v, k, v)) + .map(|(k, v)| { + let escaped = v.replace('\\', "\\\\").replace('\'', "\\'").replace('\n', "\\n").replace('\r', "\\r"); + format!("const {} = '{}';\nprocess.env['{}'] = '{}';\n", k, escaped, k, escaped) + }) .collect::>() .join("\n")); From 2048a36376a9e931fdcef5c751d8f77918b1a94c Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:42:16 +0100 Subject: [PATCH 21/48] Fix select key bug (#8499) --- frontend/src/lib/components/select/SelectDropdown.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/select/SelectDropdown.svelte b/frontend/src/lib/components/select/SelectDropdown.svelte index 62a990bb46..d9fc2dcf70 100644 --- a/frontend/src/lib/components/select/SelectDropdown.svelte +++ b/frontend/src/lib/components/select/SelectDropdown.svelte @@ -117,7 +117,7 @@ ulClass )} > - {#each processedItems ?? [] as item, itemIndex (item.value)} + {#each processedItems ?? [] as item, itemIndex} {#if (item.__select_group && itemIndex === 0) || processedItems?.[itemIndex - 1]?.__select_group !== item.__select_group}
  • Date: Tue, 24 Mar 2026 12:00:32 -0400 Subject: [PATCH 22/48] fix: add GIT_SSL_CAINFO to tracing proxy env vars (#8502) Git uses libcurl with GnuTLS on Debian, which doesn't read SSL_CERT_FILE or CURL_CA_BUNDLE for CA trust. When the OTEL tracing proxy is enabled, git clone fails with "certificate signer not trusted" because it can't verify the proxy's MITM certificate. Adding GIT_SSL_CAINFO pointing to the proxy CA cert fixes this. Co-authored-by: Claude Opus 4.6 --- backend/windmill-worker/src/worker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b698f351c6..bdeea1c076 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -965,6 +965,7 @@ async fn get_otel_tracing_proxy_envs( TRACING_PROXY_CA_CERT_PATH.to_string(), ), ("CURL_CA_BUNDLE", TRACING_PROXY_CA_CERT_PATH.to_string()), + ("GIT_SSL_CAINFO", TRACING_PROXY_CA_CERT_PATH.to_string()), ("DENO_CERT", TRACING_PROXY_CA_CERT_PATH.to_string()), ]) } From 8cfaa91d43acd821ce79dcb2e179ce2590c3386b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 16:01:18 +0000 Subject: [PATCH 23/48] update cli freshness --- cli/src/guidance/skills.ts | 13 +++---------- system_prompts/auto-generated/cli/cli-commands.md | 2 ++ system_prompts/auto-generated/prompts.ts | 2 ++ .../auto-generated/skills/cli-commands/SKILL.md | 2 ++ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index 45279e631f..6872b610ef 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -4602,18 +4602,9 @@ Tell the user they can run these commands (do NOT run them yourself): | \`wmill app dev\` | Start dev server with live reload | | \`wmill app generate-agents\` | Refresh AGENTS.md and DATATABLES.md | | \`wmill app generate-locks\` | Generate lock files for backend runnables | -| \`wmill sync push --extra-includes "f//.raw_app/**" --yes\` | Deploy this specific raw app to Windmill (never do a blanket \`wmill sync push\`) | +| \`wmill sync push\` | Deploy app to Windmill | | \`wmill sync pull\` | Pull latest from Windmill | -## Svelte 5 Event Handling - -When building Svelte 5 raw apps, be aware of event delegation: - -- The Svelte runtime version in \`node_modules/svelte\` **must match** the compiler version used by \`wmill sync push\`. If you get \`$.delegated is undefined\` errors at runtime, run \`npm install svelte@latest\` in the raw app folder and re-push. -- \`onclick\` on \`
    \`, \`\`, and other non-interactive elements uses Svelte's event delegation system. If the runtime doesn't support it, you'll get errors. -- \`onclick\` on \`
    {/if} {#if node.workflow_as_code_status}
    -
    Workflow timeline
    +
    Workflow timeline
    flowModuleSchemaMap?.deleteMultiple(resolvedModuleIds)} onDuplicateSelected={() => flowModuleSchemaMap?.duplicateMultiple(resolvedModuleIds)} onMoveSelected={() => flowModuleSchemaMap?.moveMultiple(resolvedModuleIds)} + onCreateGroup={() => flowModuleSchemaMap?.createGroup(selectionManager.selectedIds)} {canMoveSelected} resolvedCount={resolvedModuleIds.length} /> diff --git a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte index 7efb44b82d..1b54d83928 100644 --- a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte @@ -3,8 +3,8 @@ import type { SelectionManager } from '$lib/components/graph/selectionUtils.svelte' import { Button } from '$lib/components/common' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import { getNoteEditorContext } from '$lib/components/graph/noteEditor.svelte' - import { StickyNote, Move, Copy, Trash2 } from 'lucide-svelte' + import { getGroupEditorContext } from '$lib/components/graph/groupEditor.svelte' + import { Group, Move, Copy, Trash2 } from 'lucide-svelte' import type { Item } from '$lib/utils' interface Props { @@ -13,6 +13,7 @@ onDeleteSelected?: () => void onDuplicateSelected?: () => void onMoveSelected?: () => void + onCreateGroup?: () => void canMoveSelected?: boolean resolvedCount?: number } @@ -22,18 +23,14 @@ onDeleteSelected, onDuplicateSelected, onMoveSelected, + onCreateGroup, canMoveSelected = false, resolvedCount = 0 }: Props = $props() - const noteEditorContext = getNoteEditorContext() + const groupEditorContext = getGroupEditorContext() - function addGroupNote() { - if (selectionManager.selectedIds.length > 0 && noteEditorContext?.noteEditor) { - // Create the group note - noteEditorContext.noteEditor.createGroupNote(selectionManager.selectedIds) - } - } + let canCreateGroup = $derived(groupEditorContext?.canCreateGroup.val ?? false) let menuItems: Item[] = $derived([ { @@ -60,11 +57,11 @@ {#snippet action()}
    {#if resolvedCount > 0} diff --git a/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte b/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte index b0504a8055..51d1f07d3f 100644 --- a/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte +++ b/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte @@ -31,9 +31,22 @@ editor?.setCode(code) } + function validateGroups(groups: { start_id: string; end_id: string }[] | undefined) { + if (!groups) return + const seen = new Set() + for (const g of groups) { + const key = `${g.start_id}:${g.end_id}` + if (seen.has(key)) { + throw new Error(`Duplicate group: '${g.start_id}' → '${g.end_id}'`) + } + seen.add(key) + } + } + function apply() { try { const parsed = YAML.parse(code) + validateGroups(parsed.value?.groups) if (parsed.summary && typeof parsed.summary === 'string') { flowStore.val.summary = parsed.summary } @@ -59,7 +72,7 @@ initialCode = code sendUserToast('Changes applied') } catch (e) { - ;(sendUserToast('Error parsing yaml: ' + e), true) + sendUserToast('Error parsing yaml: ' + e, true) } } @@ -69,8 +82,12 @@ drawer?.toggleDrawer()}> {#snippet actions()} - - + + {/snippet} {#if flowStore.val} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 8e0250cec8..e6a548b39a 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -192,10 +192,10 @@ !!id && !!$flowPropPickerConfig && !!pickableIds && Object.keys(pickableIds).includes(id) ) - let isDragging = $derived(!!moveManager?.dragging) + let isMoving = $derived(!!moveManager?.dragging || !!moveManager?.movingModuleId) const outputPickerVisible = $derived( - editMode && (isConnectingCandidate || alwaysShowOutputPicker) && !!id && !isDragging + editMode && (isConnectingCandidate || alwaysShowOutputPicker) && !!id && !isMoving ) const icon_render = $derived(icon) @@ -214,7 +214,7 @@ flowStore?.val?.value.failure_module )} - (editId = false)}> + (editId = false)}>
    {#snippet icon()} @@ -484,11 +482,10 @@ {/if}
    - {#if deletable && !isDragging} + {#if deletable && !isMoving} {#if maximizeSubflow !== undefined} {@render buttonMaximizeSubflow?.()} {/if} - {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)} - {#if editMode && enableTestRun && flowJob?.type !== 'QueuedJob' && !isDragging} + {#if editMode && enableTestRun && flowJob?.type !== 'QueuedJob' && !isMoving}
    (hover = false)} > {#if !isMultiSelected && (hover || selected || testRunDropdownOpen) && outputPickerVisible} -
    +
    {#if !testIsLoading}
    {/each} + + 0} + on:confirmed={() => { + affectedGroupsAction?.() + affectedGroupsPending = [] + affectedGroupsAction = undefined + affectedGroupsCancel = undefined + }} + on:canceled={() => { + affectedGroupsCancel?.() + affectedGroupsPending = [] + affectedGroupsAction = undefined + affectedGroupsCancel = undefined + }} + > + {#if affectedGroupsPending.length === 1} + {@const group = affectedGroupsPending[0]} +

    The group{group.summary ? ` "${group.summary}"` : ''} will be removed (empty or duplicate). + Are you sure you want to {affectedGroupsActionLabel} the step?

    + {:else} +

    The following groups will be removed (empty or duplicate):

    +
      + {#each affectedGroupsPending as group} +
    • {group.summary || `${group.start_id} → ${group.end_id}`}
      • + {/each} +
    +

    Are you sure you want to {affectedGroupsActionLabel} the step?

    + {/if} +
    { dependents = getDependentComponents(id, flowStore.val) - const cb = () => { - push(history, flowStore.val) - if (id === 'preprocessor') { + + if (id === 'preprocessor') { + const cb = () => { + push(history, flowStore.val) selectionManager.selectId('Input') flowStore.val.value.preprocessor_module = undefined - } else { - selectNextId(id) - removeAtId(flowStore.val.value.modules, id) + refreshStateStore(flowStore) + onDelete?.(id) + delete flowStateStore.val[id] } + if (Object.keys(dependents).length > 0) { + deleteCallback = cb + } else { + cb() + } + return + } + + const dsOpts = { displayState: groupDisplayState } + const { emptiedGroups, duplicateGroups, commit } = proxy.prepareMutation((tree) => { + const found = findInStructure(tree, id) + if (found) found.parentChildren.splice(found.index, 1) + }, dsOpts) + + const affectedGroups = [...emptiedGroups, ...duplicateGroups] + + const cb = () => { + push(history, flowStore.val) + selectNextId(id) + commit({ removeDuplicates: duplicateGroups.length > 0 }) refreshStateStore(flowStore) onDelete?.(id) delete flowStateStore.val[id] } - if (Object.keys(dependents).length > 0) { - deleteCallback = cb + const proceed = () => { + if (Object.keys(dependents).length > 0) { + deleteCallback = cb + } else { + cb() + } + } + + if (affectedGroups.length > 0) { + affectedGroupsPending = affectedGroups + affectedGroupsActionLabel = 'delete' + affectedGroupsAction = proceed } else { - cb() + proceed() } }} onInsert={async (detail) => { - { - let originalModules - let targetModules - if ( - detail.sourceId == 'Input' || - detail.targetId == 'Result' || - detail.kind == 'trigger' - ) { - targetModules = flowStore.val.value.modules + if (!flowStore.val.value.modules || !Array.isArray(flowStore.val.value.modules)) return + await tick() + + // --- MOVE --- + if (moveManager.movingModuleId) { + const movedIds = moveManager.movingIds ?? [moveManager.movingModuleId] + const movingId = moveManager.movingModuleId + + let mutated = false + const moveOpts = { displayState: groupDisplayState } + const { emptiedGroups, duplicateGroups, commit } = proxy.prepareMutation((tree) => { + let originalModules: FlowStructureNode[] | undefined + let targetModules: FlowStructureNode[] | undefined + + if (detail.sourceId == 'Input' || detail.targetId == 'Result') { + targetModules = tree + } + dfsStructure(tree, (node, parentArray) => { + if (matchStructureNode(node, movingId)) originalModules = parentArray + if (detail.branch && matchStructureNode(node, detail.branch.rootId)) { + targetModules = node.branches[detail.branch.branch]?.children + } else if ( + matchStructureNode(node, detail.sourceId ?? '') || + matchStructureNode(node, detail.targetId ?? '') + ) { + targetModules = parentArray + } + }) + + if (!originalModules || !targetModules) return + + if (movedIds.length > 1) { + const firstIndex = originalModules.findIndex((m) => + matchStructureNode(m, movedIds[0]) + ) + if (firstIndex < 0) return + const removedModules = originalModules.splice(firstIndex, movedIds.length) + let insertIndex = detail.index + if (originalModules === targetModules && firstIndex < detail.index) { + insertIndex -= movedIds.length + } + targetModules.splice(insertIndex, 0, ...removedModules) + } else { + const indexToRemove = originalModules.findIndex((m) => + matchStructureNode(m, movingId) + ) + if (indexToRemove < 0) return + const [removed] = originalModules.splice(indexToRemove, 1) + let insertIndex = detail.index + if (originalModules === targetModules && indexToRemove < detail.index) + insertIndex -= 1 + targetModules.splice(insertIndex, 0, removed) + } + mutated = true + }, moveOpts) + + if (!mutated) { + moveManager.clearMoving() + return } - dfs(flowStore.val.value.modules, (mod, modules, branches) => { - if (mod.id == moveManager.movingModuleId) { - originalModules = modules - } - if (detail.branch) { - if (mod.id == detail.branch.rootId) { - targetModules = branches[detail.branch.branch] - } - } else if (mod.id == detail.sourceId || mod.id == detail.targetId) { - targetModules = modules - } else if (mod.id == detail.agentId && mod.value.type === 'aiagent') { - targetModules = mod.value.tools - } - }) - if (flowStore.val.value.modules && Array.isArray(flowStore.val.value.modules)) { - await tick() - if (moveManager.movingModuleId) { - push(history, flowStore.val) - if (!originalModules || !targetModules) { - moveManager.clearMoving() - return - } - if (moveManager.movingIds && moveManager.movingIds.length > 1) { - // Multi-move: splice out all moving modules from their parent, insert at target - const firstIndex = originalModules.findIndex( - (m) => m.id === moveManager.movingIds?.[0] - ) - const removedModules = originalModules.splice( - firstIndex, - moveManager.movingIds.length - ) - let insertIndex = detail.index - if (originalModules === targetModules && firstIndex < detail.index) { - insertIndex -= moveManager.movingIds.length - } - targetModules.splice(insertIndex, 0, ...removedModules) - selectionManager.selectByIds(removedModules.map((m) => m.id)) - } else { - let indexToRemove = originalModules.findIndex((m) => moveManager.movingModuleId == m.id) - let [removedModule] = originalModules.splice(indexToRemove, 1) - // When moving within the same array, removal shifts subsequent indices down by 1 - let insertIndex = detail.index - if (originalModules === targetModules && indexToRemove < detail.index) { - insertIndex -= 1 - } - targetModules.splice(insertIndex, 0, removedModule) - selectionManager.selectId(removedModule.id) - } - moveManager.clearMoving() + const affectedGroups = [...emptiedGroups, ...duplicateGroups] + + const doMove = () => { + push(history, flowStore.val) + commit({ removeDuplicates: duplicateGroups.length > 0 }) + if (movedIds.length > 1) { + selectionManager.selectByIds(movedIds) } else { - if (detail.isPreprocessor) { - await insertNewPreprocessorModule( - flowStore, - flowStateStore, - detail.inlineScript, - detail.script - ) - selectionManager.selectId('preprocessor') - - if (detail.inlineScript?.instructions) { - dispatch('generateStep', { - moduleId: 'preprocessor', - lang: detail.inlineScript?.language, - instructions: detail.inlineScript?.instructions - }) - } - } else { - const index = (detail.agentId ? targetModules?.length : detail.index) ?? 0 - const toolKind: SpecialToolKind | 'flowmoduleTool' | undefined = detail.agentId - ? (SPECIAL_TOOL_KINDS as readonly string[]).includes(detail.kind) - ? (detail.kind as SpecialToolKind) - : 'flowmoduleTool' - : undefined - - await insertNewModuleAtIndex( - targetModules, - index, - detail.kind, - detail.script, - detail.flow, - detail.inlineScript, - toolKind - ) - const id = targetModules[index].id - selectionManager.selectId(id) - - if (detail.inlineScript?.instructions) { - dispatch('generateStep', { - moduleId: id, - lang: detail.inlineScript?.language, - instructions: detail.inlineScript?.instructions - }) - } - if (detail.kind == 'trigger') { - await insertNewModuleAtIndex( - targetModules, - index + 1, - 'forloop', - undefined, - undefined, - undefined - ) - setExpr(targetModules[index + 1], `results.${id}`) - setScheduledPollSchedule(triggersState, triggersCount) - } - - if (detail.flow?.path) { - loadLastJob(detail.flow.path, id) - } else if (detail.script?.path) { - loadLastJob(detail.script?.path, id) - } - } - } - - if (['branchone', 'branchall'].includes(detail.kind)) { - await addBranch(targetModules[detail.index ?? 0].id) + selectionManager.selectId(movingId) } + moveManager.clearMoving() refreshStateStore(flowStore) dispatch('change') } + + if (affectedGroups.length > 0) { + affectedGroupsPending = affectedGroups + affectedGroupsActionLabel = 'move' + affectedGroupsAction = doMove + affectedGroupsCancel = () => moveManager.clearMoving() + } else { + doMove() + } + return } + + // --- INSERT --- + if (detail.isPreprocessor) { + await insertNewPreprocessorModule( + flowStore, + flowStateStore, + detail.inlineScript, + detail.script + ) + selectionManager.selectId('preprocessor') + if (detail.inlineScript?.instructions) { + dispatch('generateStep', { + moduleId: 'preprocessor', + lang: detail.inlineScript?.language, + instructions: detail.inlineScript?.instructions + }) + } + refreshStateStore(flowStore) + dispatch('change') + return + } + + push(history, flowStore.val) + + const isAgentInsert = !!detail.agentId + const toolKind: SpecialToolKind | 'flowmoduleTool' | undefined = isAgentInsert + ? (SPECIAL_TOOL_KINDS as readonly string[]).includes(detail.kind) + ? (detail.kind as SpecialToolKind) + : 'flowmoduleTool' + : undefined + + // Agent tool inserts operate on the FlowModule's tools array directly + if (isAgentInsert) { + const agentMod = getAllModules(flowStore.val.value.modules).find( + (m) => m.id === detail.agentId + ) + if (agentMod && (agentMod.value as any).tools) { + const tools = (agentMod.value as any).tools as AgentTool[] + await insertNewModuleAtIndex( + tools, + tools.length, + detail.kind as InsertKind, + detail.script, + detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, + detail.inlineScript, + toolKind + ) + const id = tools[tools.length - 1].id + selectionManager.selectId(id) + } + refreshStateStore(flowStore) + dispatch('change') + return + } + + // Regular module insert: create the module, then insert a leaf node via tree mutation + const module = await createNewModule( + detail.kind as InsertKind, + detail.script, + detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, + detail.inlineScript + ) + const index = detail.index ?? 0 + const extraModules: FlowModule[] = [module] + + // For trigger inserts, also create the forloop module + let loopModule: FlowModule | undefined + if (detail.kind == 'trigger') { + loopModule = await createNewModule('forloop') + setExpr(loopModule, `results.${module.id}`) + extraModules.push(loopModule) + } + + proxy.applyTreeMutation( + (tree) => { + // Find target array in the snapshot + let targetArray: FlowStructureNode[] | undefined + if ( + detail.sourceId == 'Input' || + detail.targetId == 'Result' || + detail.kind == 'trigger' + ) { + targetArray = tree + } + dfsStructure(tree, (node, parentArray) => { + if (detail.branch && matchStructureNode(node, detail.branch.rootId)) { + targetArray = node.branches[detail.branch.branch]?.children + } else if ( + matchStructureNode(node, detail.sourceId ?? '') || + matchStructureNode(node, detail.targetId ?? '') + ) { + targetArray = parentArray + } + }) + if (!targetArray) targetArray = tree + + // Insert the structure node (correct kind for containers like branchone/branchall) + targetArray.splice(index, 0, moduleToStructureNode(module)) + + // For trigger: also insert the forloop node after it + if (loopModule) { + targetArray.splice(index + 1, 0, moduleToStructureNode(loopModule)) + } + }, + { extraModules, displayState: groupDisplayState } + ) + + selectionManager.selectId(module.id) + + if (detail.inlineScript?.instructions) { + dispatch('generateStep', { + moduleId: module.id, + lang: detail.inlineScript?.language, + instructions: detail.inlineScript?.instructions + }) + } + if (detail.kind == 'trigger') { + setScheduledPollSchedule(triggersState, triggersCount) + } + if (detail.flow?.path) { + loadLastJob(detail.flow.path, module.id) + } else if (detail.script?.path) { + loadLastJob(detail.script?.path, module.id) + } + + if (['branchone', 'branchall'].includes(detail.kind)) { + await addBranch(module.id) + } + refreshStateStore(flowStore) + dispatch('change') }} onNewBranch={async (id) => { if (id) { @@ -761,6 +973,17 @@ mod.id = newId } }) + const groups = flowStore.val.value.groups + if (groups) { + for (const group of groups) { + if (group.start_id === id) { + group.start_id = newId + } + if (group.end_id === id) { + group.end_id = newId + } + } + } flowStateStore.val[newId] = flowStateStore.val[id] delete flowStateStore.val[id] refreshStateStore(flowStore) diff --git a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte index 798116b47d..49cbb56cc6 100644 --- a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte +++ b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte @@ -46,7 +46,7 @@
    , allNodes: Node[], allEdges: Edge[]) { + function computeGhost( + moduleId: string, + draggedNodeIds: Set, + allNodes: Node[], + allEdges: Edge[] + ) { // Use pre-computed draggedNodeIds when available (covers multi-select), // otherwise fall back to single-module subflow computation. let sfNodes: Node[] @@ -111,7 +116,15 @@ zoom: scale } - return { containerWidth, containerHeight, ghostNodes, ghostEdges, offsetX, offsetY, initialViewport } + return { + containerWidth, + containerHeight, + ghostNodes, + ghostEdges, + offsetX, + offsetY, + initialViewport + } } let isNearDrop = $derived(moveManager.nearestDropZone != null) @@ -128,7 +141,8 @@ class="fixed pointer-events-none z-[10001] flex items-center justify-center w-5 h-5 rounded-full shadow border border-border transition-colors duration-150 {isNearDrop ? 'bg-surface-accent-primary text-white' : 'bg-surface text-secondary'}" - style="left: {moveManager.ghostScreenX + CURSOR_INDICATOR_OFFSET}px; top: {moveManager.ghostScreenY + CURSOR_INDICATOR_OFFSET}px;" + style="left: {moveManager.ghostScreenX + + CURSOR_INDICATOR_OFFSET}px; top: {moveManager.ghostScreenY + CURSOR_INDICATOR_OFFSET}px;" >
    diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 853dbf5c54..6a4d208579 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -59,8 +59,22 @@ import AiToolNode, { computeAIToolNodes } from './renderers/nodes/AIToolNode.svelte' import NewAiToolNode from './renderers/nodes/NewAIToolNode.svelte' import NoteNode from './renderers/nodes/NoteNode.svelte' + import CollapsedGroupNode from './renderers/nodes/CollapsedGroupNode.svelte' + import GroupHeadNode from './renderers/nodes/GroupHeadNode.svelte' + import GroupEndNode from './renderers/nodes/GroupEndNode.svelte' import NoteTool from './NoteTool.svelte' import SelectionBoundingBox from './SelectionBoundingBox.svelte' + import GroupOverlay from './GroupOverlay.svelte' + import { + GroupDisplayState, + getGroupEditorContext, + groupKey, + type FlowGroup + } from './groupEditor.svelte' + import { buildStructureTree, computeGroupDepths, type FlowStructureNode } from './flowStructure' + import { stateSnapshot } from '$lib/svelte5Utils.svelte' + import { computeGroupModuleIds } from './groupDetectionUtils' + import { getAllModules } from '../flows/flowExplorer' import SelectionTool from './SelectionTool.svelte' import PaneContextMenu from './PaneContextMenu.svelte' import { SelectionManager } from './selectionUtils.svelte' @@ -72,6 +86,7 @@ import { compoundLayout } from './compoundLayout' import { deepEqual } from 'fast-equals' import type { AssetWithAltAccessType } from '../assets/lib' + import { computeNodeExtraSpace } from './nodeExtraSpace' import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' import { setGraphContext } from './graphContext' import { computeNoteNodes } from './noteUtils.svelte' @@ -100,6 +115,8 @@ interface Props { success?: boolean | undefined modules?: FlowModule[] | undefined + groupedModules?: FlowStructureNode[] + groupError?: unknown failureModule?: FlowModule | undefined preprocessorModule?: FlowModule | undefined minHeight?: number @@ -124,7 +141,7 @@ workspace?: string editMode?: boolean allowSimplifiedPoll?: boolean - expandedSubflows?: Record + expandedSubflows?: Record isOwner?: boolean isRunning?: boolean individualStepTests?: boolean @@ -133,6 +150,8 @@ suspendStatus?: Record noteMode?: boolean notes?: FlowNote[] + groups?: FlowGroup[] + groupDisplayState?: GroupDisplayState chatInputEnabled?: boolean multiSelectEnabled?: boolean onDeleteMultiple?: (ids: string[]) => void @@ -152,6 +171,7 @@ script?: { path: string; summary: string; hash: string | undefined } flow?: { path: string; summary: string } kind: InsertKind + expandGroup?: { groupId: string; position: 'top' | 'bottom' } }) => Promise onNewBranch?: (id: string) => Promise onSelect?: (id: string | FlowModule) => void @@ -193,6 +213,8 @@ onSelectedIteration = undefined, success = undefined, modules = [], + groupedModules: groupedModulesProp = undefined, + groupError = undefined, failureModule = undefined, preprocessorModule = undefined, minHeight = 0, @@ -232,6 +254,8 @@ flowHasChanged = false, noteMode = false, notes = undefined, + groups = undefined, + groupDisplayState: groupDisplayStateProp = undefined, exitNoteMode = undefined, onNotePositionUpdate = undefined, chatInputEnabled = false, @@ -257,6 +281,9 @@ () => nodes ) + const groupDisplayState = + untrack(() => groupDisplayStateProp) ?? new GroupDisplayState(() => groups ?? []) + // Runtime text height tracking for notes (not stored in FlowNote) let noteTextHeights = $state>({}) @@ -264,6 +291,8 @@ let paneContextMenu: PaneContextMenu | undefined = $state(undefined) let flowContainer: HTMLDivElement | undefined = $state(undefined) + // Hover tracking for group overlay + // Selection manager - create one if not provided let selectionManager = untrack(() => selectionManagerProp) || new SelectionManager() const selectedId = $derived(selectionManager.getSelectedId()) @@ -298,7 +327,9 @@ moveManager: untrack(() => moveManager), clearFlowSelection, yOffset, - diffManager + diffManager, + getFlowNodes: () => currentGraphNodeDeps, + groupDisplayState } as any) if (triggerContext && untrack(() => allowSimplifiedPoll)) { @@ -332,14 +363,36 @@ type NodeDep = { id: string parentIds?: string[] - data?: { assets?: AssetWithAltAccessType[] } + data?: { assets?: AssetWithAltAccessType[]; module?: any } } type NodePos = { position: { x: number; y: number } } - let lastNodes: [NodeDep[], (NodeDep & NodePos)[]] | undefined = undefined + let lastNodes: + | [NodeDep[], Map | undefined, (NodeDep & NodePos)[]] + | undefined = undefined + let currentGraphNodeDeps: { id: string; parentIds?: string[] }[] = $state([]) - function layoutNodes(nodes: NodeDep[]): (NodeDep & NodePos)[] { - let lastResult = lastNodes?.[1] - if (lastResult && deepEqual(nodes, lastNodes?.[0])) { + // Keep canCreateGroup in sync for consumers (SelectionBoundingBox, FlowSelectionPanel, etc.) + const groupEditorCtx = getGroupEditorContext() + + $effect(() => { + if (!groupEditorCtx) return + const ids = selectionManager.selectedIds + groupEditorCtx.canCreateGroup.val = + ids.length >= 1 && groupEditorCtx.groupEditor.canCreateGroup(ids, currentGraphNodeDeps) + }) + + let lastGroupDimensions: Map | undefined = undefined + + function layoutNodes( + nodes: NodeDep[], + nodeExtraSpace?: Map + ): (NodeDep & NodePos)[] { + let lastResult = lastNodes?.[2] + if ( + lastResult && + deepEqual(nodes, lastNodes?.[0]) && + deepEqual(nodeExtraSpace, lastNodes?.[1]) + ) { console.debug('layoutNodes', 'same nodes') return lastResult } @@ -352,16 +405,23 @@ seenId.push(n.id) } - // Run recursive compound layout - const { positions, bbox } = compoundLayout(nodes, { - nodeWidth: NODE.width, - nodeHeight: NODE.height, - gapH: NODE.gap.horizontal, - gapV: NODE.gap.vertical - }) + // Run recursive compound layout with pre-computed extra space + const layoutResult = compoundLayout( + nodes, + { + nodeWidth: NODE.width, + nodeHeight: NODE.height, + gapH: NODE.gap.horizontal, + gapV: NODE.gap.vertical + }, + nodeExtraSpace + ) + const { positions, bbox } = layoutResult + lastGroupDimensions = layoutResult.groupDimensions + + const xCenter = (fullSize ? fullWidth : width) / 2 - bbox.width / 2 - (width - fullWidth) / 2 // Center horizontally - const xCenter = (fullSize ? fullWidth : width) / 2 - bbox.width / 2 - (width - fullWidth) / 2 const newNodes = nodes.map((n) => ({ id: n.id, position: { @@ -370,7 +430,7 @@ } })) - lastNodes = [nodes, newNodes] + lastNodes = [nodes, nodeExtraSpace, newNodes] return newNodes } @@ -414,13 +474,16 @@ }, expandSubflow: async (id: string, path: string) => { const flow = await FlowService.getFlowByPath({ workspace: workspace, path }) - expandedSubflows[id] = flow.value.modules + expandedSubflows[id] = { modules: flow.value.modules, groups: flow.value.groups } expandedSubflows = expandedSubflows }, minimizeSubflow: (id: string) => { delete expandedSubflows[id] expandedSubflows = expandedSubflows }, + expandGroup: (groupId: string) => { + groupDisplayState.expandGroup(groupId) + }, updateMock: (detail) => { onUpdateMock?.(detail) }, @@ -585,17 +648,37 @@ return } - // console.log('compute') + const graphNodeDeps = Object.values(graph.nodes).map((n) => ({ + id: n.id, + parentIds: n.parentIds, + data: { assets: (n.data as any).assets, module: (n.data as any).module } + })) + currentGraphNodeDeps = graphNodeDeps - let layoutedNodes = layoutNodes( - Object.values(graph.nodes).map((n) => ({ - id: n.id, - parentIds: n.parentIds, - data: { assets: (n.data as any).assets } - })) - ) - let newNodes: (Node & NodeLayout)[] = layoutedNodes.map((n) => ({ ...n, ...graph.nodes[n.id] })) + // Pre-compute extra space per node for assets, AI tools, group notes, group headers + const nodeExtraSpace = computeNodeExtraSpace(graphNodeDeps, { + showAssets: $showAssets ?? true, + showNotes, + notes, + noteTextHeights, + groupDisplayState, + insertable, + flowModuleStates + }) + // Layout with extra space baked into sugiyama + let layoutedNodes = layoutNodes(graphNodeDeps, nodeExtraSpace) + let newNodes: (Node & NodeLayout)[] = layoutedNodes.map((n) => { + const merged = { ...n, ...graph.nodes[n.id] } + // Augment group head nodes with wrapper dimensions from compound layout + if (graph.nodes[n.id]?.type === 'groupHead' && lastGroupDimensions?.has(n.id)) { + const dims = lastGroupDimensions.get(n.id)! + merged.data = { ...merged.data, wrapperWidth: dims.width, wrapperHeight: dims.height } + } + return merged + }) + + // Compute asset visual nodes (no position remapping) let assetNodesResult = $showAssets ? computeAssetNodes( newNodes.map((n) => ({ @@ -605,25 +688,17 @@ })) ) : undefined - if (assetNodesResult) { - newNodes = newNodes.map((n) => ({ - ...n, - position: assetNodesResult.newNodePositions[n.id] - })) - } - let aiToolNodesResult = computeAIToolNodes(newNodes, eventHandler, insertable, flowModuleStates) - let nodesAfterAITools = newNodes.map((n) => ({ - ...n, - position: aiToolNodesResult.newNodePositions[n.id] - })) - let finalNodes = [ - ...nodesAfterAITools, + // Compute AI tool visual nodes (no position remapping) + let aiToolNodesResult = computeAIToolNodes(newNodes, eventHandler, insertable, flowModuleStates) + + let finalNodes: (Node & NodeLayout)[] = [ + ...newNodes, ...(assetNodesResult?.newAssetNodes ?? []), ...aiToolNodesResult.toolNodes ] - // Compute note nodes and positions + // Compute note nodes (no position remapping) let noteNodesResult = showNotes ? computeNoteNodes( finalNodes.map((n) => ({ @@ -644,14 +719,6 @@ ) : undefined - // Apply note positioning to nodes if notes are enabled - if (noteNodesResult) { - finalNodes = finalNodes.map((n) => ({ - ...n, - position: noteNodesResult.newNodePositions[n.id] || n.position - })) - } - // update nodes nodes = [...finalNodes, ...(noteNodesResult?.noteNodes ?? [])] @@ -699,7 +766,10 @@ assetsOverflowed: AssetsOverflowedNode, aiTool: AiToolNode, newAiTool: NewAiToolNode, - note: NoteNode + note: NoteNode, + collapsedGroup: CollapsedGroupNode, + groupHead: GroupHeadNode, + groupEnd: GroupEndNode } as any const edgeTypes = { @@ -735,7 +805,41 @@ let graph = $derived.by(() => { moduleTracker.counter effectiveModuleActions - return graphBuilder( + currentGroups + + const collapsedGroupIds = new Set( + allGroups + .filter((g) => groupDisplayState.isRuntimeCollapsed(groupKey(g))) + .map((g) => groupKey(g)) + ) + + if (groupError) { + return { nodes: {}, edges: [], error: groupError } + } + + // Use provided structure tree (from proxy) or build locally (diff mode / read-only) + let gm: FlowStructureNode[] | undefined = groupedModulesProp + if (!gm) { + const allGroups = groups ?? [] + const graphGroups = allGroups.map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: untrack(() => + computeGroupModuleIds(g.start_id, g.end_id, getAllModules(effectiveModules ?? [])) + ) + })) + try { + gm = buildStructureTree( + stateSnapshot(untrack(() => effectiveModules) ?? []) as FlowModule[], + graphGroups + ) + } catch (e) { + return { nodes: {}, edges: [], error: e } + } + } + + const result = graphBuilder( + gm, untrack(() => effectiveModules), { disableAi, @@ -767,16 +871,43 @@ untrack(() => selectedId), simplifiableFlow, triggerNode ? path : undefined, - expandedSubflows + expandedSubflows, + showNotes, + collapsedGroupIds ) + return { ...result, structureTree: gm } }) let hideAssetsToggle = $derived( $showAssets && Object.values(nodes).every((n) => n.type !== 'asset') ) - let hideNotesToggle = $derived(!notes || notes.length === 0) + let hideNotesToggle = $derived( + (!notes || notes.length === 0) && !(groups ?? []).some((g) => g.note != null) + ) + + let currentGroupDepths = $derived( + 'structureTree' in graph && graph.structureTree ? computeGroupDepths(graph.structureTree) : {} + ) + + // All groups including those from expanded subflows (for overlay rendering) + let allGroups = $derived.by(() => { + const base = groups ?? [] + const subflowGroups = Object.values(expandedSubflows).flatMap((sf) => sf.groups ?? []) + return subflowGroups.length > 0 ? [...base, ...subflowGroups] : base + }) + + // Track groups for re-layout when groups change + let currentGroups = $derived(groups ?? []) $effect(() => { - ;[graph, allowSimplifiedPoll, $showAssets, showNotes, noteManager.renderCount] + ;[ + graph, + allowSimplifiedPoll, + $showAssets, + showNotes, + noteManager.renderCount, + currentGroups, + groupDisplayState.renderCount + ] untrack(async () => { await updateStores() }) @@ -893,6 +1024,16 @@ } } + export function createGroupFromSelection(ids: string[]) { + if (groupEditorCtx?.groupEditor) { + groupEditorCtx.groupEditor.createGroup(ids, currentGraphNodeDeps) + tick().then(() => { + clearFlowSelection() + selectionManager.clearSelection() + }) + } + } + const modifierKey = isMac() ? 'Meta' : 'Control' @@ -909,7 +1050,7 @@ bind:this={flowContainer} > {#if graph?.error} -
    +
    {graph.error} @@ -1008,6 +1149,12 @@ /> {/if} + + @@ -1065,7 +1212,7 @@ try { localStorage.setItem( 'svelvet', - encodeState({ modules, failureModule, preprocessorModule, notes }) + encodeState({ modules, failureModule, preprocessorModule, notes, groups }) ) } catch (e) { console.error('error interacting with local storage', e) diff --git a/frontend/src/lib/components/graph/GroupActionBar.svelte b/frontend/src/lib/components/graph/GroupActionBar.svelte new file mode 100644 index 0000000000..1ae30395b5 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupActionBar.svelte @@ -0,0 +1,158 @@ + + +
    + {#if moveManager && moveModuleId} + moveManager.toggleMoving(moveModuleId!)} + /> + {/if} + {#if note == null} + + {/if} + + {#snippet buttonReplacement()} + + {/snippet} + {#snippet menu()} +
    + +
    +
    + {#each Object.values(NoteColor) as c (c)} + + {/each} +
    +
    + + +
    + onUpdateAutocollapse(e.detail)} + /> +
    + +
    + + + + + {#if onDeleteGroup} +
    + + + + {/if} +
    + {/snippet} +     +
    diff --git a/frontend/src/lib/components/graph/GroupHeader.svelte b/frontend/src/lib/components/graph/GroupHeader.svelte new file mode 100644 index 0000000000..474d71c1a4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupHeader.svelte @@ -0,0 +1,116 @@ + + + + +
    {}))} + title={collapsed ? 'Expand group' : 'Collapse group'} +> +
    + +
    +
    + {#if editingSummary} +
    + +
    + {:else} + {})) : undefined} + >{summary || PLACEHOLDER} + {/if} +
    +
    + + diff --git a/frontend/src/lib/components/graph/GroupHeaderBlock.svelte b/frontend/src/lib/components/graph/GroupHeaderBlock.svelte new file mode 100644 index 0000000000..089c0642a4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupHeaderBlock.svelte @@ -0,0 +1,76 @@ + + + +
    (hovered = true)} + onmouseleave={() => (hovered = false)} +> + graphContext?.groupDisplayState?.toggleRuntimeCollapse(groupId)} + onSummaryUpdate={(text) => groupEditorContext?.groupEditor.updateSummary(groupId, text)} + /> + {#if showNotes && note != null} + graphContext?.groupDisplayState?.setNoteHeight(groupId, h)} + onNoteUpdate={(text) => groupEditorContext?.groupEditor.updateNote(groupId, text)} + /> + {/if} + {#if editMode} + (menuOpen = open)} + onAddNote={() => groupEditorContext?.groupEditor.addNote(groupId)} + onRemoveNote={() => groupEditorContext?.groupEditor.removeNote(groupId)} + onUpdateColor={(c) => groupEditorContext?.groupEditor.updateColor(groupId, c)} + onUpdateAutocollapse={(v) => groupEditorContext?.groupEditor.updateAutocollapse(groupId, v)} + onDeleteGroup={() => groupEditorContext?.groupEditor.deleteGroup(groupId)} + /> + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupModuleIcons.svelte b/frontend/src/lib/components/graph/GroupModuleIcons.svelte new file mode 100644 index 0000000000..3df1aecfaa --- /dev/null +++ b/frontend/src/lib/components/graph/GroupModuleIcons.svelte @@ -0,0 +1,182 @@ + + +
    + {#each displayModules as mod (mod.id)} + {@const selected = selectionManager.isNodeSelected(mod.id)} + {@const nodeState = flowModuleStates?.[mod.id]?.type} + {@const colorClasses = getNodeColorClasses(nodeState, selected)} + + {#snippet children()} + + +
    selectModule(mod)} + > +
    + +
    + {mod.id} +
    + {/snippet} + {#snippet text()} + {mod.id}: {moduleLabel(mod)} + {/snippet} +     + {/each} + {#if overflowModules.length > 0} + {@const overflowColorClasses = getNodeColorClasses(overflowAggregateState, false)} + + {#snippet buttonReplacement()} +
    + +{overflowModules.length} +
    + {/snippet} + {#snippet menu()} +
    + {#each overflowModules as mod (mod.id)} + {@const nodeState = flowModuleStates?.[mod.id]?.type} + {@const colorClasses = getNodeColorClasses(nodeState, false)} + {@const selected = selectionManager.isNodeSelected(mod.id)} + + +
    selectModule(mod)} + > +
    + +
    + {moduleLabel(mod)} + {mod.id} +
    + {/each} +
    + {/snippet} +     + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupNodeCard.svelte b/frontend/src/lib/components/graph/GroupNodeCard.svelte new file mode 100644 index 0000000000..dd69fcf0d0 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupNodeCard.svelte @@ -0,0 +1,165 @@ + + +
    +
    +
    + {#if modules && modules.length > 0} + + {:else} + + {/if} +
    + {#if editingSummary} + + {:else} + + + {})) : undefined} + >{summary || 'Group'} + {/if} +
    +
    + {#if stepCount != null} + + + {stepCount} node{stepCount !== 1 ? 's' : ''} + {/if} +
    + + {#if showNote} +
    + onHeightChange?.(h)} + onNoteUpdate={(text) => onNoteUpdate?.(text)} + /> +
    + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupNoteArea.svelte b/frontend/src/lib/components/graph/GroupNoteArea.svelte new file mode 100644 index 0000000000..1b4562e8d0 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupNoteArea.svelte @@ -0,0 +1,151 @@ + + + +
    +
    + {#if editing} +
    + + +
    + + {:else if note} + +
    {}) : undefined} + > + +
    + {:else} + +
    {}) : undefined} + > + Double click to edit the note +
    + {/if} +
    +
    diff --git a/frontend/src/lib/components/graph/GroupOverlay.svelte b/frontend/src/lib/components/graph/GroupOverlay.svelte new file mode 100644 index 0000000000..c15095bfb4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupOverlay.svelte @@ -0,0 +1,90 @@ + + +{#each groups as group (groupKey(group))} + {@const bounds = groupBoundsMap[groupKey(group)]} + {#if bounds} + +
    +     + {/if} +{/each} diff --git a/frontend/src/lib/components/graph/MiniFlowGraph.svelte b/frontend/src/lib/components/graph/MiniFlowGraph.svelte index fdb73745df..ab0c5042e8 100644 --- a/frontend/src/lib/components/graph/MiniFlowGraph.svelte +++ b/frontend/src/lib/components/graph/MiniFlowGraph.svelte @@ -1,6 +1,12 @@ - -{#if noteEditorContext?.noteEditor && selectedNodeIds.length > 1} - - {@render children()} - -{/if} diff --git a/frontend/src/lib/components/graph/NoteColorPicker.svelte b/frontend/src/lib/components/graph/NoteColorPicker.svelte index d17adb9317..ff3a1a276c 100644 --- a/frontend/src/lib/components/graph/NoteColorPicker.svelte +++ b/frontend/src/lib/components/graph/NoteColorPicker.svelte @@ -10,7 +10,11 @@ isOpen?: boolean } - let { selectedColor, onColorChange, isOpen = $bindable(false) }: Props = $props() + let { + selectedColor, + onColorChange, + isOpen = $bindable(false) + }: Props = $props() import { ViewportPortal, type Node } from '@xyflow/svelte' import { calculateNodesBoundsWithOffset } from './util' - import { StickyNote, Move, Copy, Trash2, EllipsisVertical } from 'lucide-svelte' + import { Move, Copy, Trash2, EllipsisVertical, Group } from 'lucide-svelte' import { Button } from '../common' import DropdownV2 from '../DropdownV2.svelte' - import { getNoteEditorContext } from './noteEditor.svelte' + import { getGroupEditorContext } from './groupEditor.svelte' import { getGraphContext } from './graphContext' import MoveHandleButton from './MoveHandleButton.svelte' import { tick } from 'svelte' @@ -36,18 +36,19 @@ let resolvedCount = $derived(resolvedModuleIds.length) - // Get NoteEditor context for group note creation - const noteEditorContext = getNoteEditorContext() + // Get GroupEditor context for group creation + const groupEditorContext = getGroupEditorContext() // Get Graph context for clearFlowSelection function and moveManager const graphContext = getGraphContext() const moveManager = graphContext?.moveManager - function handleAddGroupNote() { - if (selectedNodes.length > 0 && noteEditorContext?.noteEditor && graphContext) { - // Create the group note first - noteEditorContext.noteEditor.createGroupNote(selectedNodes) + let canCreateGroup = $derived(groupEditorContext?.canCreateGroup.val ?? false) + + function handleAddGroup() { + if (selectedNodes.length > 0 && groupEditorContext?.groupEditor && graphContext) { + const flowNodes = graphContext.getFlowNodes?.() ?? [] + groupEditorContext.groupEditor.createGroup(selectedNodes, flowNodes) - // Wait for next tick to ensure DOM updates tick().then(() => { graphContext?.clearFlowSelection?.() graphContext?.selectionManager.clearSelection() @@ -74,13 +75,13 @@ shortcut: isMac() ? '⌫' : 'Del', action: () => onDeleteSelected?.() }, - ...(noteEditorContext?.noteEditor + ...(groupEditorContext?.groupEditor ? [ { - displayName: 'Add note', - icon: StickyNote, - separatorTop: true, - action: handleAddGroupNote + displayName: 'Create group', + icon: Group, + action: handleAddGroup, + disabled: !canCreateGroup } ] : []) diff --git a/frontend/src/lib/components/graph/compoundLayout.ts b/frontend/src/lib/components/graph/compoundLayout.ts index 4bf3ddebda..c47aa62584 100644 --- a/frontend/src/lib/components/graph/compoundLayout.ts +++ b/frontend/src/lib/components/graph/compoundLayout.ts @@ -1,5 +1,6 @@ import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' import { NODE } from './util' +import { GROUP_HEADER_HEIGHT } from './groupEditor.svelte' type LayoutNode = { id: string @@ -14,7 +15,7 @@ type LayoutConstants = { } type CompoundGroup = { - type: 'branch' | 'loop' + type: 'branch' | 'loop' | 'group' headId: string endId: string branches: { @@ -27,9 +28,12 @@ type LayoutResult = { positions: Map bbox: { width: number; height: number } contentMinX: number + groupDimensions?: Map } const LOOP_INDENT = 25 +export const GROUP_PADDING = 16 +export const GROUP_TOP_PADDING = 32 /** * Detect compound groups from a flat list of node IDs. @@ -83,6 +87,18 @@ function detectGroups( endId: id, branches: [{ labelId: `${baseId}-start`, innerIds }] }) + } else if (baseId.startsWith('group:')) { + // Group pattern: group:{groupId} head + group:{groupId}-end + // Body is everything reachable from head to end + const innerIds = findInnerIds(baseId, id, nodeIds, childrenMap) + if (innerIds.length > 0) { + groups.push({ + type: 'group', + headId: baseId, + endId: id, + branches: [{ labelId: innerIds[0], innerIds: innerIds.slice(1) }] + }) + } } } @@ -230,6 +246,29 @@ function runSugiyama( * 5. Run sugiyama on the simplified graph * 6. Expand wrapper positions back to absolute positions */ +/** + * Build nodeSizes map for sugiyama from nodeExtraSpace. + * Each node's effective height = top + NODE.height + bottom. + */ +function buildNodeSizes( + nodeIds: string[], + constants: LayoutConstants, + nodeExtraSpace?: Map +): Map | undefined { + if (!nodeExtraSpace || nodeExtraSpace.size === 0) return undefined + const sizes = new Map() + for (const id of nodeIds) { + const extra = nodeExtraSpace.get(id) + if (extra && (extra.top > 0 || extra.bottom > 0 || extra.left > 0 || extra.right > 0)) { + sizes.set(id, { + width: constants.nodeWidth + extra.left + extra.right, + height: constants.nodeHeight + extra.top + extra.bottom + }) + } + } + return sizes.size > 0 ? sizes : undefined +} + const MAX_RECURSION_DEPTH = 50 function layoutLevel( @@ -237,7 +276,8 @@ function layoutLevel( allNodes: Map, constants: LayoutConstants, childrenMap: Map, - depth: number = 0 + depth: number = 0, + nodeExtraSpace?: Map ): LayoutResult { const positions = new Map() const nodeIdSet = new Set(nodeIds) @@ -256,8 +296,15 @@ function layoutLevel( const n = allNodes.get(id)! return { id, parentIds: (n.parentIds ?? []).filter((pid) => nodeIdSet.has(pid)) } }) - const result = runSugiyama(flatNodes, constants) + const extraSizes = buildNodeSizes( + flatNodes.map((n) => n.id), + constants, + nodeExtraSpace + ) + const result = runSugiyama(flatNodes, constants, extraSizes) for (const [id, pos] of result.positions) { + const extra = nodeExtraSpace?.get(id) + if (extra) pos.y += extra.top positions.set(id, pos) } return { positions, bbox: { width: result.width, height: result.height }, contentMinX: 0 } @@ -322,7 +369,14 @@ function layoutLevel( const branchNodeIds = [branch.labelId, ...branch.innerIds] // Find sub-groups within this branch - const result = layoutLevel(branchNodeIds, allNodes, constants, childrenMap, depth + 1) + const result = layoutLevel( + branchNodeIds, + allNodes, + constants, + childrenMap, + depth + 1, + nodeExtraSpace + ) branchLayouts.push({ labelId: branch.labelId, @@ -349,6 +403,16 @@ function layoutLevel( maxBranchHeight = Math.max(0, ...branchLayouts.map((bl) => bl.bbox.height)) // head row + branch content + end row wrapperHeight = rowHeight + maxBranchHeight + rowHeight + } else if (group.type === 'group') { + // Group: body is centered with padding on all sides + const bodyWidth = branchLayouts[0]?.bbox.width ?? constants.nodeWidth + const bodyHeight = branchLayouts[0]?.bbox.height ?? 0 + wrapperWidth = Math.max(bodyWidth + GROUP_PADDING * 2, constants.nodeWidth) + maxBranchHeight = bodyHeight + const headExtra = nodeExtraSpace?.get(group.headId) + const groupHeadRow = GROUP_HEADER_HEIGHT + (headExtra?.bottom ?? 0) + GROUP_TOP_PADDING + // head row + body + bottom padding + wrapperHeight = groupHeadRow + bodyHeight + GROUP_PADDING } else { // Loop: body is indented const bodyWidth = branchLayouts[0]?.bbox.width ?? constants.nodeWidth @@ -395,13 +459,30 @@ function layoutLevel( } // Step 5: Run sugiyama on flattened nodes - const sugResult = runSugiyama(flatNodes, constants, wrapperSizes) + // Merge wrapperSizes with nodeExtraSpace-derived sizes for non-group nodes + const extraSizes = buildNodeSizes( + flatNodes.map((n) => n.id), + constants, + nodeExtraSpace + ) + const mergedSizes = new Map() + if (extraSizes) { + for (const [id, size] of extraSizes) mergedSizes.set(id, size) + } + for (const [id, size] of wrapperSizes) mergedSizes.set(id, size) + const sugResult = runSugiyama( + flatNodes, + constants, + mergedSizes.size > 0 ? mergedSizes : undefined + ) // Step 6: Resolve absolute positions // First, set positions for regular (non-group) nodes + // Apply per-node y-offset from nodeExtraSpace so decorations above have room for (const [nid, pos] of sugResult.positions) { if (groupByHeadId.has(nid)) continue // Handle groups separately - positions.set(nid, { x: pos.x, y: pos.y }) + const extra = nodeExtraSpace?.get(nid) + positions.set(nid, { x: pos.x, y: pos.y + (extra?.top ?? 0) }) } // Now expand group wrappers into absolute positions @@ -411,9 +492,15 @@ function layoutLevel( const rowHeight = constants.nodeHeight + constants.gapV const isBranch = gl.group.type === 'branch' + const isGroup = gl.group.type === 'group' // Position the head node at the top-center of the wrapper - positions.set(headId, { x: wrapperPos.x, y: wrapperPos.y }) + // Apply extra top padding so decorations above the head node have room + const headExtra = nodeExtraSpace?.get(headId) + positions.set(headId, { + x: wrapperPos.x, + y: wrapperPos.y + (headExtra?.top ?? 0) + }) if (isBranch) { // Reuse cached branchWidths and totalWidth @@ -441,6 +528,26 @@ function layoutLevel( x: wrapperPos.x, y: wrapperPos.y + rowHeight + maxBranchHeight + constants.gapV }) + } else if (isGroup) { + // Group: body is centered within wrapper (no x offset) + const headExtra = nodeExtraSpace?.get(gl.group.headId) + const groupHeadRow = GROUP_HEADER_HEIGHT + (headExtra?.bottom ?? 0) + GROUP_TOP_PADDING + const bl = gl.branchLayouts[0] + if (bl) { + for (const [innerNodeId, innerPos] of bl.result.positions) { + positions.set(innerNodeId, { + x: wrapperPos.x + innerPos.x, + y: wrapperPos.y + groupHeadRow + innerPos.y + }) + } + } + + // Position end node below body + const bodyHeight = bl?.bbox.height ?? 0 + positions.set(gl.group.endId, { + x: wrapperPos.x, + y: wrapperPos.y + groupHeadRow + bodyHeight + GROUP_PADDING + }) } else { // Loop: position start, body, and end const bl = gl.branchLayouts[0] @@ -463,16 +570,34 @@ function layoutLevel( } } + // Collect group dimensions from this level and child layouts + const groupDimensions = new Map() + for (const [headId, gl] of groupLayouts) { + groupDimensions.set(headId, { width: gl.wrapperWidth, height: gl.wrapperHeight }) + // Propagate child groupDimensions from recursive branch layouts + for (const bl of gl.branchLayouts) { + if (bl.result.groupDimensions) { + for (const [childId, dims] of bl.result.groupDimensions) { + groupDimensions.set(childId, dims) + } + } + } + } + // Compute overall bbox (nodes + group wrapper extents) let minX = Infinity let maxX = -Infinity let minY = Infinity let maxY = -Infinity - for (const pos of positions.values()) { - minX = Math.min(minX, pos.x - constants.nodeWidth / 2) - maxX = Math.max(maxX, pos.x + constants.nodeWidth / 2) - minY = Math.min(minY, pos.y) - maxY = Math.max(maxY, pos.y + constants.nodeHeight) + for (const [nid, pos] of positions) { + // Group end nodes are zero-height markers — skip them + if (nid.startsWith('group:') && nid.endsWith('-end')) continue + const extra = nodeExtraSpace?.get(nid) + minX = Math.min(minX, pos.x - constants.nodeWidth / 2 - (extra?.left ?? 0)) + maxX = Math.max(maxX, pos.x + constants.nodeWidth / 2 + (extra?.right ?? 0)) + // Account for top decoration space above the node + minY = Math.min(minY, pos.y - (extra?.top ?? 0)) + maxY = Math.max(maxY, pos.y + constants.nodeHeight + (extra?.bottom ?? 0)) } // Account for group wrapper extents in bbox (e.g. LOOP_INDENT makes wrappers wider than nodes) for (const [headId, gl] of groupLayouts) { @@ -492,7 +617,12 @@ function layoutLevel( width: Math.max(bboxWidth, constants.nodeWidth), height: Math.max(bboxHeight, 0) } - return { positions, bbox: finalBbox, contentMinX } + return { + positions, + bbox: finalBbox, + contentMinX, + groupDimensions: groupDimensions.size > 0 ? groupDimensions : undefined + } } /** @@ -500,10 +630,16 @@ function layoutLevel( * * Takes the flat list of nodes and edges from graphBuilder and produces * absolute positions that account for compound structure (branches, loops). + * + * nodeExtraSpace: per-node top/bottom/left/right padding that should be allocated in layout. + * After layout, each node's y is shifted down by its top padding so decorations + * (assets, AI tools, group headers) have room above. Left/right padding widens the + * column allocated to the node so neighbors are pushed further away. */ export function compoundLayout( nodes: { id: string; parentIds?: string[] }[], - constants?: Partial + constants?: Partial, + nodeExtraSpace?: Map ): LayoutResult { const c: LayoutConstants = { nodeWidth: constants?.nodeWidth ?? NODE.width, @@ -528,7 +664,7 @@ export function compoundLayout( } const nodeIds = nodes.map((n) => n.id) - const result = layoutLevel(nodeIds, allNodes, c, childrenMap) + const result = layoutLevel(nodeIds, allNodes, c, childrenMap, 0, nodeExtraSpace) // Shift positions so minX=0 (left-aligned). // FlowGraphV2 centers with: xCenter = viewport/2 - bbox.width/2 diff --git a/frontend/src/lib/components/graph/flowStructure.test.ts b/frontend/src/lib/components/graph/flowStructure.test.ts new file mode 100644 index 0000000000..fce5071e80 --- /dev/null +++ b/frontend/src/lib/components/graph/flowStructure.test.ts @@ -0,0 +1,245 @@ +import { describe, it, expect, vi } from 'vitest' + +// Mock modules that transitively import CSS/Monaco +vi.mock('monaco-editor', () => ({})) +vi.mock('@xyflow/svelte', () => ({})) +vi.mock('./renderers/nodes/AssetNode.svelte', () => ({ + assetDisplaysAsOutputInFlowGraph: () => false +})) +vi.mock('../modulesTest.svelte', () => ({})) + +import type { GraphGroup } from './groupEditor.svelte' +import type { FlowModule } from '$lib/gen' +import { + buildStructureTree, + flattenStructureIds, + deriveGroupsFromStructure, + collectLeafIds, + findInStructure +} from './flowStructure' + +function makeModule(id: string): FlowModule { + return { + id, + value: { type: 'rawscript', content: '', language: 'python3' } as any + } as FlowModule +} + +function makeBranchAll(id: string, branchInnerIds: string[][]): FlowModule { + return { + id, + value: { + type: 'branchall', + branches: branchInnerIds.map((ids) => ({ modules: ids.map((iid) => makeModule(iid)) })) + } as any + } as FlowModule +} + +function makeForloop(id: string, innerIds: string[]): FlowModule { + return { + id, + value: { + type: 'forloopflow', + modules: innerIds.map((iid) => makeModule(iid)), + iterator: { type: 'javascript', expr: '' } + } as any + } as FlowModule +} + +function makeGroup( + id: string, + start_id: string, + end_id: string, + moduleIds: string[] = [] +): GraphGroup { + return { id, start_id, end_id, moduleIds } +} + +describe('buildStructureTree', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + + it('builds structure tree for a valid group', () => { + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const result = buildStructureTree(modules, groups) + // Should have a group node + the remaining leaf 'c' + expect(result).toHaveLength(2) + expect(result[0].kind).toBe('group') + expect(result[0].id).toBe('g1') + expect(result[0].branches[0].children).toHaveLength(2) + expect(result[1].kind).toBe('leaf') + expect(result[1].id).toBe('c') + }) + + it('throws on duplicate group IDs', () => { + const groups = [makeGroup('g1', 'a', 'a', ['a']), makeGroup('g1', 'b', 'c', ['b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/duplicate group id.*g1/i) + }) + + it('throws on inverted range (start_id after end_id)', () => { + const groups = [makeGroup('g1', 'c', 'a', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/inverted range/i) + }) + + it('throws on partially overlapping groups', () => { + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b']), makeGroup('g2', 'b', 'c', ['b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/overlap without nesting/i) + }) + + it('throws when group start_id is a virtual node (Input)', () => { + const groups = [makeGroup('g1', 'Input', 'b', ['a', 'b'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('throws when group end_id is a virtual node (Result)', () => { + const groups = [makeGroup('g1', 'a', 'Result', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('throws when group references Trigger', () => { + const groups = [makeGroup('g1', 'Trigger', 'c', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('allows fully nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const result = buildStructureTree(mods, groups) + expect(result).toHaveLength(1) // outer group contains everything + expect(result[0].kind).toBe('group') + // Inner group should be nested + const outerChildren = result[0].branches[0].children + expect(outerChildren).toHaveLength(3) // a, inner-group, d + expect(outerChildren[1].kind).toBe('group') + expect(outerChildren[1].id).toBe('inner') + }) + + it('handles empty modules', () => { + const result = buildStructureTree([], []) + expect(result).toHaveLength(0) + }) + + it('handles container modules (forloop)', () => { + const mods = [makeForloop('loop', ['x', 'y']), makeModule('c')] + const result = buildStructureTree(mods, []) + expect(result).toHaveLength(2) + expect(result[0].kind).toBe('forloopflow') + expect(result[0].branches).toHaveLength(1) + expect(result[0].branches[0].children).toHaveLength(2) + expect(result[0].branches[0].children[0].id).toBe('x') + }) + + it('handles groups inside containers', () => { + const mods = [makeForloop('loop', ['x', 'y', 'z'])] + const groups = [makeGroup('g1', 'x', 'y', ['x', 'y'])] + const result = buildStructureTree(mods, groups) + expect(result).toHaveLength(1) + expect(result[0].kind).toBe('forloopflow') + const innerChildren = result[0].branches[0].children + expect(innerChildren).toHaveLength(2) // group + z + expect(innerChildren[0].kind).toBe('group') + expect(innerChildren[0].id).toBe('g1') + }) + + it('throws when group spans parallel branches (branchall)', () => { + const mods = [ + makeModule('a'), + makeBranchAll('ba', [ + ['x', 'y'], + ['p', 'q'] + ]), + makeModule('c') + ] + const groups = [makeGroup('g1', 'x', 'q', ['x', 'q'])] + expect(() => buildStructureTree(mods, groups)).toThrow(/could not be resolved/) + }) +}) + +describe('flattenStructureIds', () => { + it('flattens a simple tree', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const ids = flattenStructureIds(tree) + expect(ids).toEqual(['a', 'b', 'c']) + }) + + it('flattens nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const tree = buildStructureTree(mods, groups) + const ids = flattenStructureIds(tree) + expect(ids).toEqual(['a', 'b', 'c', 'd']) + }) +}) + +describe('deriveGroupsFromStructure', () => { + it('derives group definitions with correct start/end', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const derived = deriveGroupsFromStructure(tree) + expect(derived).toHaveLength(1) + expect(derived[0].start_id).toBe('a') + expect(derived[0].end_id).toBe('b') + }) + + it('derives nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const tree = buildStructureTree(mods, groups) + const derived = deriveGroupsFromStructure(tree) + expect(derived).toHaveLength(2) + expect(derived[0].start_id).toBe('a') + expect(derived[0].end_id).toBe('d') + expect(derived[1].start_id).toBe('b') + expect(derived[1].end_id).toBe('c') + }) +}) + +describe('findInStructure', () => { + it('finds a leaf node', () => { + const modules = [makeModule('a'), makeModule('b')] + const tree = buildStructureTree(modules, []) + const found = findInStructure(tree, 'b') + expect(found).toBeDefined() + expect(found!.index).toBe(1) + }) + + it('finds a node inside a group', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const found = findInStructure(tree, 'b') + expect(found).toBeDefined() + expect(found!.index).toBe(1) + // parentChildren should be the group's branch children + expect(found!.parentChildren).toHaveLength(2) + }) + + it('finds a group node by group id', () => { + const modules = [makeModule('a'), makeModule('b')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const found = findInStructure(tree, 'g1') + expect(found).toBeDefined() + expect(found!.index).toBe(0) + }) +}) + +describe('collectLeafIds', () => { + it('collects all leaf module IDs including inside containers', () => { + const mods = [makeForloop('loop', ['x', 'y']), makeModule('c')] + const tree = buildStructureTree(mods, []) + const ids = collectLeafIds(tree) + expect(ids).toEqual(['loop', 'x', 'y', 'c']) + }) +}) diff --git a/frontend/src/lib/components/graph/flowStructure.ts b/frontend/src/lib/components/graph/flowStructure.ts new file mode 100644 index 0000000000..e2a725bc85 --- /dev/null +++ b/frontend/src/lib/components/graph/flowStructure.ts @@ -0,0 +1,498 @@ +import type { FlowModule } from '$lib/gen' + +import type { FlowGroup, GraphGroup } from './groupEditor.svelte' +import { getContainerInnerArrays } from './groupEditor.svelte' +import { VIRTUAL_NODE_IDS } from './groupDetectionUtils' + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type ContainerKind = 'forloopflow' | 'whileloopflow' | 'branchone' | 'branchall' + +export type StructureBranch = { + label?: string + children: FlowStructureNode[] +} + +export type FlowStructureNode = { + /** FlowModule.id for modules, groupKey(g) for groups */ + id: string + kind: 'leaf' | 'group' | ContainerKind + /** Only present when kind === 'group' */ + group?: FlowGroup + /** Only present when kind === 'group' — flat module IDs for step count */ + moduleIds?: string[] + /** Child branches. leaf=[], group=[{children}], container=[{children}, ...] */ + branches: StructureBranch[] +} + +// --------------------------------------------------------------------------- +// Type guards +// --------------------------------------------------------------------------- +// Building the structure tree +// --------------------------------------------------------------------------- + +export function buildStructureTree( + modules: FlowModule[], + groups: GraphGroup[] +): FlowStructureNode[] { + const { items, consumed } = buildStructureTreeRecurse(modules, groups) + const unconsumed = groups.filter((g) => !consumed.has(g.id)) + if (unconsumed.length > 0) { + throw new Error( + `Group(s) ${unconsumed.map((g) => `'${g.id}'`).join(', ')} could not be resolved: ` + + `their start/end nodes do not belong to the same branch` + ) + } + return items +} + +export function moduleToStructureNode(mod: FlowModule): FlowStructureNode { + const innerArrays = getContainerInnerArrays(mod) + if (innerArrays.length === 0) { + return { id: mod.id, kind: 'leaf', branches: [] } + } + + const kind = (mod.value as any).type as ContainerKind + const branches: StructureBranch[] = innerArrays.map(({ get, label }) => ({ + label, + children: [] // filled later by recursion + })) + + return { id: mod.id, kind, branches } +} + +function buildStructureTreeRecurse( + modules: FlowModule[], + groups: GraphGroup[] +): { items: FlowStructureNode[]; consumed: Set } { + if (modules.length === 0) { + return { items: [], consumed: new Set() } + } + + const indexMap = new Map() + for (let i = 0; i < modules.length; i++) { + indexMap.set(modules[i].id, i) + } + + // Reject duplicate group IDs + const seenGroupIds = new Set() + for (const g of groups) { + if (seenGroupIds.has(g.id)) { + throw new Error(`Duplicate group id: '${g.id}'`) + } + seenGroupIds.add(g.id) + } + + // Reject groups referencing virtual nodes + for (const g of groups) { + if (VIRTUAL_NODE_IDS.has(g.start_id) || VIRTUAL_NODE_IDS.has(g.end_id)) { + throw new Error( + `Group '${g.id}' references virtual node: groups cannot include Input, Result, or Trigger` + ) + } + } + + // Partition: groups for this level vs rest + const levelGroups: GraphGroup[] = [] + const otherGroups: GraphGroup[] = [] + for (const g of groups) { + if (indexMap.has(g.start_id) && indexMap.has(g.end_id)) { + const s = indexMap.get(g.start_id)! + const e = indexMap.get(g.end_id)! + if (s > e) { + throw new Error( + `Group '${g.id}' has inverted range: start_id='${g.start_id}' (index ${s}) > end_id='${g.end_id}' (index ${e})` + ) + } + levelGroups.push(g) + } else { + otherGroups.push(g) + } + } + + // Validate no partial overlaps + for (let i = 0; i < levelGroups.length; i++) { + for (let j = i + 1; j < levelGroups.length; j++) { + const a = levelGroups[i] + const b = levelGroups[j] + const aStart = indexMap.get(a.start_id)! + const aEnd = indexMap.get(a.end_id)! + const bStart = indexMap.get(b.start_id)! + const bEnd = indexMap.get(b.end_id)! + + if (aEnd < bStart || bEnd < aStart) continue + if (aStart <= bStart && bEnd <= aEnd) continue + if (bStart <= aStart && aEnd <= bEnd) continue + + throw new Error(`Groups '${a.id}' and '${b.id}' overlap without nesting`) + } + } + + // Build grouped structure for this level + function build( + startIdx: number, + endIdx: number, + availableGroups: GraphGroup[] + ): FlowStructureNode[] { + const result: FlowStructureNode[] = [] + let i = startIdx + while (i <= endIdx) { + const candidates = availableGroups.filter((g) => { + const gStart = indexMap.get(g.start_id)! + const gEnd = indexMap.get(g.end_id)! + return gStart === i && gEnd <= endIdx + }) + candidates.sort((a, b) => { + const spanA = indexMap.get(a.end_id)! - indexMap.get(a.start_id)! + const spanB = indexMap.get(b.end_id)! - indexMap.get(b.start_id)! + return spanB - spanA + }) + + const group = candidates[0] + if (group) { + const gEnd = indexMap.get(group.end_id)! + const remaining = availableGroups.filter((g) => g.id !== group.id) + const innerNodes = build(i, gEnd, remaining) + + const moduleIds: string[] = [] + for (let k = i; k <= gEnd; k++) { + moduleIds.push(modules[k].id) + } + + result.push({ + id: group.id, + kind: 'group', + group: { + summary: group.summary, + note: group.note, + color: group.color, + autocollapse: group.autocollapse, + start_id: group.start_id, + end_id: group.end_id + }, + moduleIds, + branches: [{ children: innerNodes }] + }) + i = gEnd + 1 + } else { + result.push(moduleToStructureNode(modules[i])) + i++ + } + } + return result + } + + const result = build(0, modules.length - 1, levelGroups) + + // Recurse into containers with remaining unconsumed groups + const consumed = new Set(levelGroups.map((g) => g.id)) + let remaining = otherGroups + + function recurseIntoContainers(items: FlowStructureNode[]): void { + for (const item of items) { + if (item.kind === 'group') { + recurseIntoContainers(item.branches[0].children) + continue + } + if (item.branches.length === 0) continue + + // This is a container module — get inner FlowModule arrays and recurse + const modIdx = indexMap.get(item.id) + if (modIdx === undefined) continue + const mod = modules[modIdx] + + const innerArrays = getContainerInnerArrays(mod) + for (let bi = 0; bi < innerArrays.length; bi++) { + const inner = buildStructureTreeRecurse(innerArrays[bi].get(), remaining) + item.branches[bi] = { + label: item.branches[bi]?.label, + children: inner.items + } + for (const id of inner.consumed) consumed.add(id) + remaining = remaining.filter((g) => !inner.consumed.has(g.id)) + } + } + } + recurseIntoContainers(result) + + return { items: result, consumed } +} + +// --------------------------------------------------------------------------- +// Traversal utilities +// --------------------------------------------------------------------------- + +/** Generic DFS over the structure tree */ +export function dfsStructure( + nodes: FlowStructureNode[], + fn: (node: FlowStructureNode, parentArray: FlowStructureNode[]) => void +): void { + for (const node of nodes) { + fn(node, nodes) + for (const branch of node.branches) { + dfsStructure(branch.children, fn) + } + } +} + +/** Flatten to ordered module IDs (groups are transparent) */ +export function flattenStructureIds(nodes: FlowStructureNode[]): string[] { + const ids: string[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + ids.push(...flattenStructureIds(node.branches[0].children)) + } else { + ids.push(node.id) + } + } + return ids +} + +/** Collect leaf module IDs recursively (including inside containers) */ +export function collectLeafIds(nodes: FlowStructureNode[]): string[] { + const ids: string[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + ids.push(...collectLeafIds(node.branches[0].children)) + } else { + ids.push(node.id) + for (const branch of node.branches) { + ids.push(...collectLeafIds(branch.children)) + } + } + } + return ids +} + +// --------------------------------------------------------------------------- +// Finding nodes in the tree +// --------------------------------------------------------------------------- + +export type FindResult = { parentChildren: FlowStructureNode[]; index: number } + +export function findInStructure(nodes: FlowStructureNode[], id: string): FindResult | undefined { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (node.id === id) return { parentChildren: nodes, index: i } + for (const branch of node.branches) { + const found = findInStructure(branch.children, id) + if (found) return found + } + } + return undefined +} + +/** + * Match a structure node against a graph node ID. + * Handles group head/end IDs (group:X, group:X-end) and collapsed-group:X. + */ +export function matchStructureNode(node: FlowStructureNode, nodeId: string): boolean { + if (node.id === nodeId) return true + if (node.kind === 'group') { + return ( + nodeId === `group:${node.id}` || + nodeId === `group:${node.id}-end` || + nodeId === `collapsed-group:${node.id}` + ) + } + return false +} + +/** + * Find insert index using graph node IDs (handles group:X-end etc.). + * Returns the index OF the matched item (insert before it). + * For group-end nodes, returns index AFTER the group (insert after it). + */ +export function findInsertIndexByNodeId(items: FlowStructureNode[], targetNodeId: string): number { + // group-end: insert after the group + if (targetNodeId.startsWith('group:') && targetNodeId.endsWith('-end')) { + const groupId = targetNodeId.slice('group:'.length, -'-end'.length) + const idx = items.findIndex((n) => n.kind === 'group' && n.id === groupId) + return idx >= 0 ? idx + 1 : items.length + } + // Everything else: insert at the matched item's position + for (let i = 0; i < items.length; i++) { + if (matchStructureNode(items[i], targetNodeId)) return i + } + return items.length +} + +// --------------------------------------------------------------------------- +// Deriving groups from the structure tree +// --------------------------------------------------------------------------- + +export function deriveGroupsFromStructure(nodes: FlowStructureNode[]): FlowGroup[] { + const groups: FlowGroup[] = [] + for (const node of nodes) { + if (node.kind === 'group' && node.group) { + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length === 0) { + console.warn(`deriveGroupsFromStructure: skipping empty group "${node.id}"`) + continue + } + groups.push({ + ...node.group, + start_id: flatIds[0], + end_id: flatIds[flatIds.length - 1] + }) + // Recurse for nested groups + groups.push(...deriveGroupsFromStructure(node.branches[0].children)) + } else { + for (const branch of node.branches) { + groups.push(...deriveGroupsFromStructure(branch.children)) + } + } + } + return groups +} + +// --------------------------------------------------------------------------- +// Syncing structure back to FlowModule[] +// --------------------------------------------------------------------------- + +/** + * Reconstruct a FlowModule[] from the structure tree, looking up originals + * from moduleMap and patching container inner arrays to match the tree ordering. + */ +export function applyStructureToModules( + nodes: FlowStructureNode[], + moduleMap: Map +): FlowModule[] { + const result: FlowModule[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + // Groups are transparent — splice their children into this level + result.push(...applyStructureToModules(node.branches[0].children, moduleMap)) + } else { + const mod = moduleMap.get(node.id) + if (!mod) continue + + // Patch container inner arrays + if (node.branches.length > 0) { + const innerArrays = getContainerInnerArrays(mod) + for (let bi = 0; bi < innerArrays.length && bi < node.branches.length; bi++) { + innerArrays[bi].set(applyStructureToModules(node.branches[bi].children, moduleMap)) + } + } + + result.push(mod) + } + } + return result +} + +// --------------------------------------------------------------------------- +// Empty groups cleanup +// --------------------------------------------------------------------------- + +/** + * Walk the tree, remove group nodes that have no leaf modules, and return + * the removed groups. Mutates the input array in-place. + * Recurses depth-first so inner groups are cleaned before checking outer ones. + */ +export function removeEmptyGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const removed: FlowGroup[] = [] + for (let i = nodes.length - 1; i >= 0; i--) { + const node = nodes[i] + if (node.kind === 'group' && node.group) { + // Recurse first — inner groups may become empty too + removed.push(...removeEmptyGroups(node.branches[0].children)) + if (flattenStructureIds(node.branches[0].children).length === 0) { + removed.push(node.group) + nodes.splice(i, 1) + } + } else { + for (const branch of node.branches) { + removed.push(...removeEmptyGroups(branch.children)) + } + } + } + return removed +} + +/** Walk the structure tree to compute nesting depth for each group (O(n)). */ +export function computeGroupDepths(tree: FlowStructureNode[]): Record { + const depths: Record = {} + function walk(nodes: FlowStructureNode[], groupDepth: number): void { + for (const node of nodes) { + if (node.kind === 'group') { + depths[node.id] = groupDepth + for (const branch of node.branches) { + walk(branch.children, groupDepth + 1) + } + } else { + for (const branch of node.branches) { + walk(branch.children, groupDepth) + } + } + } + } + walk(tree, 0) + return depths +} + +/** + * Find duplicate groups in the structure tree (same start_id:end_id after mutation). + * Returns the groups that should be removed (keeps the first, removes subsequent duplicates). + */ +export function findDuplicateGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const duplicates: FlowGroup[] = [] + const seen = new Set() + + function walk(items: FlowStructureNode[]): void { + for (const node of items) { + if (node.kind === 'group' && node.group) { + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length > 0) { + const key = `${flatIds[0]}:${flatIds[flatIds.length - 1]}` + if (seen.has(key)) { + duplicates.push(node.group) + } else { + seen.add(key) + } + } + walk(node.branches[0].children) + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(nodes) + return duplicates +} + +/** Remove duplicate groups from the structure tree (keeps first occurrence). */ +export function removeDuplicateGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const removed: FlowGroup[] = [] + const seen = new Set() + + function walk(items: FlowStructureNode[]): void { + for (let i = items.length - 1; i >= 0; i--) { + const node = items[i] + if (node.kind === 'group' && node.group) { + walk(node.branches[0].children) + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length > 0) { + const key = `${flatIds[0]}:${flatIds[flatIds.length - 1]}` + if (seen.has(key)) { + // Replace group node with its children (ungroup) + removed.push(node.group) + items.splice(i, 1, ...node.branches[0].children) + } else { + seen.add(key) + } + } + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(nodes) + return removed +} diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index 192b5c0c34..4fd7641096 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -8,6 +8,14 @@ import { getFlowModuleAssets, type AssetWithAltAccessType } from '../assets/lib' import { assetDisplaysAsOutputInFlowGraph } from './renderers/nodes/AssetNode.svelte' import type { ModulesTestStates, ModuleTestState } from '../modulesTest.svelte' import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' +import { + type FlowStructureNode, + collectLeafIds, + findInsertIndexByNodeId, + buildStructureTree +} from './flowStructure' +import { groupKey, type FlowGroup } from './groupEditor.svelte' +import { computeGroupModuleIds } from './groupDetectionUtils' export type InsertKind = | 'script' @@ -62,6 +70,7 @@ export type GraphEventHandlers = { simplifyFlow: (b: boolean) => void expandSubflow: (id: string, path: string) => void minimizeSubflow: (id: string) => void + expandGroup: (groupId: string) => void updateMock: (detail: { mock: FlowModule['mock']; id: string }) => void testUpTo: (id: string) => void editInput: (moduleId: string, key: string) => void @@ -111,6 +120,9 @@ export type FlowNode = | AssetsOverflowedN | AiToolN | NewAiToolN + | CollapsedGroupN + | GroupHeadN + | GroupEndN export type InputN = { type: 'input2' @@ -316,6 +328,48 @@ export type NewAiToolN = { } } +export type CollapsedGroupN = { + type: 'collapsedGroup' + data: { + groupId: string + summary: string | undefined + note: string | undefined + color: string | undefined + autocollapse: boolean | undefined + stepCount: number + modules: FlowModule[] + flowModuleStates: Record | undefined + flowJob: Job | undefined + isOwner: boolean + suspendStatus: Record + showNotes: boolean + editMode: boolean + eventHandlers: GraphEventHandlers + } +} + +export type GroupHeadN = { + type: 'groupHead' + data: { + groupId: string + summary: string | undefined + note: string | undefined + color: string | undefined + autocollapse: boolean | undefined + editMode: boolean + showNotes: boolean + eventHandlers: GraphEventHandlers + wrapperWidth?: number + } +} + +export type GroupEndN = { + type: 'groupEnd' + data: { + groupId: string + } +} + export function topologicalSort( nodes: { id: string; parentIds?: string[] }[] ): { id: string; parentIds?: string[] }[] { @@ -336,22 +390,8 @@ export function topologicalSort( return result.reverse() } -// input2: InputNode, -// module: ModuleNode, -// branchAllStart: BranchAllStart, -// branchAllEnd: BranchAllEndNode, -// forLoopEnd: ForLoopEndNode, -// forLoopStart: ForLoopStartNode, -// result: ResultNode, -// whileLoopStart: ForLoopStartNode, -// whileLoopEnd: ForLoopEndNode, -// branchOneStart: BranchOneStart, -// branchOneEnd: BranchOneEndNode, -// subflowBound: SubflowBound, -// noBranch: NoBranchNode, -// trigger: TriggersNode - export function graphBuilder( + structureTree: FlowStructureNode[], modules: FlowModule[] | undefined, extra: { disableAi: boolean @@ -383,11 +423,9 @@ export function graphBuilder( selectedId: string | undefined, simplifiableFlow: SimplifiableFlow | undefined, flowPathForTriggerNode: string | undefined, - expandedSubflows: Record - // triggerProps?: { - // path?: string - // flowIsSimplifiable?: boolean - // } + expandedSubflows: Record, + showNotes: boolean, + collapsedGroupIds: Set ): { nodes: { [key: string]: NodeLayout } edges: Edge[] @@ -403,7 +441,13 @@ export function graphBuilder( const nodes: NodeLayout[] = [] const edges: Edge[] = [] - function addNode(module: FlowModule) { + // Lookup map from module ID to the original reactive FlowModule objects. + const moduleMap = new Map() + for (const m of getAllModules(modules, failureModule)) { + moduleMap.set(m.id, m) + } + + function addNode(module: FlowModule, extraData?: Record) { const duplicated = nodes.find((n) => n.id === module.id) if (duplicated) { console.log('Duplicated node detected: ', module, duplicated) @@ -424,7 +468,8 @@ export function graphBuilder( isOwner: extra.isOwner, flowJob: extra.flowJob, assets: getFlowModuleAssets(module, extra.additionalAssetsMap), - moduleAction: extra.moduleActions?.[module.id] + moduleAction: extra.moduleActions?.[module.id], + ...extraData }, type: 'module', selectable: true @@ -483,14 +528,20 @@ export function graphBuilder( customId?: string type?: string subModules?: FlowModule[] + currentItems?: FlowStructureNode[] disableMoveIds?: string[] } ) { parents[targetId] = [...(parents[targetId] ?? []), sourceId] - const mods = options?.subModules ?? modules - - let index = mods?.findIndex((m) => m.id === targetId) ?? -1 + let index: number + if (options?.currentItems) { + index = findInsertIndexByNodeId(options.currentItems, targetId) + } else { + const mods = options?.subModules ?? modules + const found = mods?.findIndex((m) => m.id === targetId) ?? -1 + index = found >= 0 ? found : (mods?.length ?? 0) + } const visited = new Set() const recStack = new Set() @@ -514,8 +565,7 @@ export function graphBuilder( simplifiedTriggerView: simplifiableFlow?.simplifiedFlow, disableMoveIds: options?.disableMoveIds, enableTrigger: sourceId === 'Input', - // If the index is -1, it means that the target module is not in the modules array, so we set it to the length of the array - index: index >= 0 ? index : (mods?.length ?? 0), + index, ...extra, insertable: extra.insertable && !options?.disableInsert && prefix == undefined, shouldOffsetInsertBtnDueToAssetNode: nodeIdsWithOutputAssets.has(sourceId) @@ -591,7 +641,7 @@ export function graphBuilder( } function processModules( - modules: FlowModule[], + items: FlowStructureNode[], branch: { rootId: string; branch: number } | undefined, beforeNode: NodeLayout, nextNode: NodeLayout | undefined, @@ -600,31 +650,166 @@ export function graphBuilder( disableMoveIds: string[] = [], parentIndex?: string ) { + // For subflow prefix rewriting, clone modules into moduleMap with prefixed IDs + // (avoid mutating reactive originals which would trigger state_unsafe_mutation in $derived) if (prefix != undefined) { - modules.forEach((m) => { - if (!m['oid']) { - m['oid'] = m.id + items.forEach((item) => { + if (item.kind === 'group') return + const m = moduleMap.get(item.id) + if (m) { + const oid = m['oid'] ?? m.id + const newId = 'subflow:' + prefix + oid + const clone = { ...m, id: newId, oid } as FlowModule & { oid: string } + clone['oid'] = oid + moduleMap.set(newId, clone) + item.id = newId } - m.id = 'subflow:' + prefix + m['oid'] }) } let previousId: string | undefined = undefined - if (modules.length === 0) { + if (items.length === 0) { if (nextNode) { addEdge(beforeNode.id, nextNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } } else { - modules.forEach((module, index) => { + items.forEach((item, index) => { + // --- Group items --- + if (item.kind === 'group') { + const g = item.group! + const gId = item.id + + if (collapsedGroupIds.has(gId)) { + // Collapsed group: single node + const nodeId = `collapsed-group:${gId}` + const leafIds = collectLeafIds(item.branches[0].children) + nodes.push({ + id: nodeId, + data: { + groupId: gId, + summary: g.summary, + note: g.note, + color: g.color, + autocollapse: g.autocollapse, + stepCount: item.moduleIds?.length ?? 0, + modules: leafIds + .map((id) => moduleMap.get(id)) + .filter((m): m is FlowModule => !!m), + flowModuleStates: extra.flowModuleStates, + flowJob: extra.flowJob, + isOwner: extra.isOwner, + suspendStatus: extra.suspendStatus, + showNotes, + editMode: prefix == undefined && extra.editMode, + eventHandlers + }, + type: 'collapsedGroup', + selectable: false + }) + + // Wire: previous → collapsedGroup + if (index > 0 && previousId) { + addEdge(previousId, nodeId, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + previousId = nodeId + } else { + // Expanded group: head → recurse → end + const headId = `group:${gId}` + const endId = `group:${gId}-end` + const localDisableMoveIds = [...disableMoveIds, headId] + + const headNode: NodeLayout = { + id: headId, + data: { + groupId: gId, + summary: g.summary, + note: g.note, + color: g.color, + autocollapse: g.autocollapse, + editMode: prefix == undefined && extra.editMode, + showNotes, + eventHandlers + }, + type: 'groupHead', + selectable: false + } + + const endNode: NodeLayout = { + id: endId, + data: { + groupId: gId + }, + type: 'groupEnd', + selectable: false + } + + nodes.push(headNode) + nodes.push(endNode) + + // Wire: previous → headNode + if (index > 0 && previousId) { + addEdge(previousId, headId, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + // Recurse inner modules + processModules( + item.branches[0].children, + { rootId: headId, branch: 0 }, + headNode, + endNode, + simplifiedTriggerView, + prefix, + localDisableMoveIds, + parentIndex + ) + + previousId = endId + } + + // Shared first/last edge wiring for groups + if (index === 0) { + addEdge( + beforeNode.id, + collapsedGroupIds.has(gId) ? `collapsed-group:${gId}` : `group:${gId}`, + undefined, + prefix, + { + currentItems: items, + disableMoveIds, + disableInsert: simplifiedTriggerView + } + ) + } + + if (index === items.length - 1 && previousId && nextNode) { + addEdge(previousId, nextNode.id, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + return + } + + // --- Regular FlowModule items --- + const module = moduleMap.get(item.id) + if (!module) return const localDisableMoveIds = [...disableMoveIds, module.id] - // Add the edge between the previous node and the current one + // Inter-module edge: connect previous → current (expanded subflows handle their own) if (index > 0 && previousId && expandedSubflows[module.id] == undefined) { addEdge(previousId, module.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -700,7 +885,7 @@ export function graphBuilder( ) processModules( - branch.modules, + item.branches[branchIndex]?.children ?? [], { rootId: module.id, branch: branchIndex }, startNode, endNode, @@ -722,7 +907,7 @@ export function graphBuilder( id: `${module.id}-start`, data: { id: module.id, - module: module, + module: moduleMap.get(module.id) ?? module, simplifiedTriggerView, eventHandlers: eventHandlers, editMode: extra.editMode, @@ -759,7 +944,7 @@ export function graphBuilder( const selectedIterIndex = extra.flowModuleStates?.[module.id]?.selectedForloopIndex processModules( - module.value.modules, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, startNode, endNode, @@ -798,7 +983,7 @@ export function graphBuilder( const selectedIterIndex = extra.flowModuleStates?.[module.id]?.selectedForloopIndex processModules( - module.value.modules, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, startNode, endNode, @@ -825,21 +1010,6 @@ export function graphBuilder( } nodes.push(endNode) - // // Add default branch - // const defaultBranch: NodeLayout = { - // id: `${module.id}-default`, - // data: { - // offset: 0, - // label: 'Default', - // id: module.id, - // branchIndex: -1, - // eventHandlers: eventHandlers, - // branchOne: true, - // ...extra - // }, - // type: 'noBranch' - // } - const defaultBranch: NodeLayout = { id: `${module.id}-branch-default`, data: { @@ -863,7 +1033,7 @@ export function graphBuilder( }) processModules( - module.value.default, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, defaultBranch, endNode, @@ -899,7 +1069,7 @@ export function graphBuilder( }) processModules( - branch.modules, + item.branches[branchIndex + 1]?.children ?? [], { rootId: module.id, branch: branchIndex + 1 }, startNode, endNode, @@ -912,9 +1082,9 @@ export function graphBuilder( previousId = endNode.id } else { - let expanded = expandedSubflows[module.id] - if (expanded) { - expanded = $state.snapshot(expanded) + const expandedData = expandedSubflows[module.id] + if (expandedData) { + const expandedMods = $state.snapshot(expandedData.modules) as FlowModule[] const startId = `${module.id}` const idWithoutPrefix = module.id.startsWith('subflow:') ? module.id.substring(8) @@ -936,12 +1106,12 @@ export function graphBuilder( if (previousId) { addEdge(previousId, startNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } else { addEdge(beforeNode.id, startNode.id, undefined, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -962,8 +1132,20 @@ export function graphBuilder( nodes.push(endNode) + // Register expanded subflow modules so prefix rewriting finds + // the inner modules (not the parent flow's modules with same IDs) + for (const em of getAllModules(expandedMods)) { + moduleMap.set(em.id, em) + } + + const expandedGroups = (expandedData.groups ?? []).map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: computeGroupModuleIds(g.start_id, g.end_id, getAllModules(expandedMods)) + })) + processModules( - expanded, + buildStructureTree(expandedMods, expandedGroups), undefined, startNode, endNode, @@ -981,15 +1163,15 @@ export function graphBuilder( if (index === 0 && expandedSubflows[module.id] == undefined) { addEdge(beforeNode.id, module.id, undefined, prefix, { - subModules: modules, + currentItems: items, disableMoveIds, disableInsert: simplifiedTriggerView }) } - if (index === modules.length - 1 && previousId && nextNode) { + if (index === items.length - 1 && previousId && nextNode) { addEdge(previousId, nextNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -997,10 +1179,12 @@ export function graphBuilder( } } + const topLevelItems = structureTree + if (simplifiableFlow?.simplifiedFlow === true && triggerNode) { - processModules(modules, undefined, triggerNode, undefined, true, undefined) + processModules(topLevelItems, undefined, triggerNode, undefined, true, undefined) } else { - processModules(modules, undefined, inputNode, resultNode, false, undefined) + processModules(topLevelItems, undefined, inputNode, resultNode, false, undefined) } if (failureModule) { diff --git a/frontend/src/lib/components/graph/graphContext.ts b/frontend/src/lib/components/graph/graphContext.ts index 8540982366..a6245769cf 100644 --- a/frontend/src/lib/components/graph/graphContext.ts +++ b/frontend/src/lib/components/graph/graphContext.ts @@ -4,6 +4,7 @@ import type { NoteManager } from './noteManager.svelte' import type { MoveManager } from './moveManager.svelte' import type { Writable } from 'svelte/store' import type { FlowDiffManager } from '../flows/flowDiffManager.svelte' +import type { GroupDisplayState } from './groupEditor.svelte' export type GraphContext = { selectionManager: SelectionManager @@ -14,6 +15,9 @@ export type GraphContext = { clearFlowSelection?: () => void yOffset?: number diffManager: FlowDiffManager + /** Current flow nodes for group validation (set by FlowGraphV2) */ + getFlowNodes?: () => { id: string; parentIds?: string[] }[] + groupDisplayState?: GroupDisplayState } const graphContextKey = 'FlowGraphContext' diff --git a/frontend/src/lib/components/graph/groupDetectionUtils.ts b/frontend/src/lib/components/graph/groupDetectionUtils.ts index e2c81c4d98..8dc6e3d8c2 100644 --- a/frontend/src/lib/components/graph/groupDetectionUtils.ts +++ b/frontend/src/lib/components/graph/groupDetectionUtils.ts @@ -1,7 +1,127 @@ +import { topologicalSort } from './graphBuilder.svelte' + +/** Node IDs synthesized by graphBuilder that are not real FlowModules */ +export const VIRTUAL_NODE_IDS = new Set(['Input', 'Result', 'Trigger']) + type FlowNode = { id: string; parentIds?: string[] } /** - * Use a simple algorithm to complete a group and split it into connected components + * Compute the set of module IDs that belong to a group defined by start_id and end_id. + * Uses the flattened module list (from getAllModules) and slices between start and end. + * Used for collapsed group icons, step count, and moduleToCollapsedGroup mapping. + */ +export function computeGroupModuleIds( + startId: string, + endId: string, + allModules: { id: string }[] +): string[] { + if (startId === endId) { + return allModules.some((m) => m.id === startId) ? [startId] : [] + } + + const startIdx = allModules.findIndex((m) => m.id === startId) + const endIdx = allModules.findIndex((m) => m.id === endId) + + if (startIdx === -1 || endIdx === -1 || startIdx > endIdx) { + if (startIdx > endIdx) { + console.warn( + `computeGroupModuleIds: inverted range for group ${startId}→${endId} (${startIdx} > ${endIdx})` + ) + } + return [] + } + + return allModules.slice(startIdx, endIdx + 1).map((m) => m.id) +} + +/** + * Check whether a set of selected node IDs can form a valid group. + * Normalizes marker IDs (branch/forloop) to parent module IDs, + * then uses topologicalSort to derive start and end boundaries. + */ +export function canFormValidGroup( + selectedIds: string[], + flowNodes: FlowNode[], + excludeIds?: Set +): { valid: true; startId: string; endId: string } | { valid: false } { + if (selectedIds.length === 0) return { valid: false } + + // Normalize marker IDs to parent module IDs. + // -start (forloop head) → parent ID. -end/-branch-* → skip if parent covered, else reject. + const rawSet = new Set(selectedIds) + const normalizedIds: string[] = [] + + for (const id of selectedIds) { + const parentId = id.replace(/-(end|start|branch-.*)$/, '') + if (parentId === id) { + normalizedIds.push(id) + continue + } + if (id.endsWith('-start')) { + normalizedIds.push(parentId) + continue + } + // -end or -branch-*: parent must be covered (directly or via -start) + if (!rawSet.has(parentId) && !rawSet.has(`${parentId}-start`)) { + return { valid: false } + } + } + + if (normalizedIds.length === 0) return { valid: false } + const normalizedSet = new Set(normalizedIds) + + // Topo sort full graph, filter to normalized selection. + // Include raw matches plus all markers (-start, -end, -branch-*) whose parent is selected. + const sorted = topologicalSort(flowNodes) + const selectedSorted = sorted.filter((n) => { + if (normalizedSet.has(n.id)) return true + const parentId = n.id.replace(/-(end|start|branch-.*)$/, '') + return parentId !== n.id && normalizedSet.has(parentId) + }) + + if (selectedSorted.length === 0) return { valid: false } + + // Reject virtual or excluded nodes + if (selectedSorted.some((n) => VIRTUAL_NODE_IDS.has(n.id) || excludeIds?.has(n.id))) { + return { valid: false } + } + + // Topo order is bottom-first: first = bottom (end), last = top (start). + // Use raw IDs for BFS traversal, normalize for the returned group boundaries. + const rawStartId = selectedSorted[selectedSorted.length - 1].id + const rawEndId = selectedSorted[0].id + const startId = rawStartId.replace(/-(end|start|branch-.*)$/, '') + const endId = rawEndId.replace(/-(end|start|branch-.*)$/, '') + + // Verify all selected nodes lie between start and end in the DAG. + // BFS backward from rawEndId to rawStartId to collect reachable nodes. + // Normalize collected IDs so container markers map to their parent module. + const between = new Set() + const queue = [rawEndId] + const visited = new Set() + const parentMap = new Map(flowNodes.map((n) => [n.id, n.parentIds ?? []])) + while (queue.length > 0) { + const cur = queue.shift()! + if (visited.has(cur)) continue + visited.add(cur) + const normalized = cur.replace(/-(end|start|branch-.*)$/, '') + between.add(cur) + between.add(normalized) + if (cur === rawStartId) continue + for (const p of parentMap.get(cur) ?? []) { + queue.push(p) + } + } + if (!normalizedIds.every((id) => between.has(id))) { + return { valid: false } + } + + return { valid: true, startId, endId } +} + +/** + * Legacy utility: complete a group and split it into connected components. + * Still used by NoteEditor for FlowNote group notes (contained_node_ids). */ export function completeAndSplitGroup(groupNodes: string[], flowNodes: FlowNode[]): string[][] { if (groupNodes.length <= 1) { diff --git a/frontend/src/lib/components/graph/groupEditor.svelte.ts b/frontend/src/lib/components/graph/groupEditor.svelte.ts new file mode 100644 index 0000000000..8405390b96 --- /dev/null +++ b/frontend/src/lib/components/graph/groupEditor.svelte.ts @@ -0,0 +1,325 @@ +import type { FlowModule } from '$lib/gen' +import type { StateStore } from '$lib/utils' +import type { ExtendedOpenFlow } from '../flows/types' + +import { canFormValidGroup } from './groupDetectionUtils' +import type { NoteColor } from './noteColors' +import { DEFAULT_GROUP_NOTE_COLOR, getNextAvailableColor } from './noteColors' +import { getContext, setContext } from 'svelte' + +/** + * Type for a flow group (matches the generated type from OpenAPI). + * Members are computed dynamically from all nodes on paths between start_id and end_id. + */ +export type FlowGroup = { + summary?: string + note?: string + autocollapse?: boolean + start_id: string + end_id: string + color?: string +} + +/** Derive a stable key from a group's boundaries. Used as ephemeral ID for graph nodes, runtime state, etc. */ +export function groupKey(g: { start_id: string; end_id: string }): string { + return `${g.start_id}:${g.end_id}` +} + +/** + * Display state for flow groups inside the graph. + * Handles runtime collapse state and note height tracking. + * Similar to NoteManager — instantiated inside FlowGraphV2. + */ +export class GroupDisplayState { + #getGroups: () => FlowGroup[] + #runtimeCollapsedIds = $state>(new Set()) + #runtimeInitialized = $state(false) + #noteHeights = $state>({}) + renderCount = $state(0) + + constructor(getGroups: () => FlowGroup[]) { + this.#getGroups = getGroups + } + + /** Initialize runtime state from autocollapse. Safe to call from event handlers. */ + private ensureRuntimeInitialized(): void { + if (this.#runtimeInitialized) return + const groups = this.#getGroups() + this.#runtimeCollapsedIds = new Set( + groups.filter((g) => g.autocollapse).map((g) => groupKey(g)) + ) + this.#runtimeInitialized = true + } + + /** Check if a group is currently collapsed (runtime). Safe to call from $derived. */ + isRuntimeCollapsed(groupId: string): boolean { + if (!this.#runtimeInitialized) { + return this.#getGroups().find((g) => groupKey(g) === groupId)?.autocollapse ?? false + } + return this.#runtimeCollapsedIds.has(groupId) + } + + /** Toggle runtime collapse (Minimize2 button) */ + toggleRuntimeCollapse(groupId: string): void { + this.ensureRuntimeInitialized() + const next = new Set(this.#runtimeCollapsedIds) + if (next.has(groupId)) next.delete(groupId) + else next.add(groupId) + this.#runtimeCollapsedIds = next + this.render() + } + + /** Expand a group at runtime (CollapsedGroupNode click) */ + expandGroup(groupId: string): void { + this.ensureRuntimeInitialized() + const next = new Set(this.#runtimeCollapsedIds) + next.delete(groupId) + this.#runtimeCollapsedIds = next + this.render() + } + + /** Set note height for a group (used for layout spacing) */ + setNoteHeight(groupId: string, height: number): void { + if (this.#noteHeights[groupId] !== height) { + this.#noteHeights[groupId] = height + this.render() + } + } + + /** Get all note heights */ + getNoteHeights(): Record { + return this.#noteHeights + } + + /** Bump render counter to trigger re-layout */ + render(): void { + this.renderCount++ + } + + /** Remap runtime state when a group's boundaries (and thus its key) change */ + remapGroupKey(oldKey: string, newKey: string): void { + if (this.#runtimeCollapsedIds.has(oldKey)) { + const next = new Set(this.#runtimeCollapsedIds) + next.delete(oldKey) + next.add(newKey) + this.#runtimeCollapsedIds = next + } + if (oldKey in this.#noteHeights) { + this.#noteHeights[newKey] = this.#noteHeights[oldKey] + delete this.#noteHeights[oldKey] + } + } + + /** Get currently collapsed groups for graph builder. Safe to call from $derived. */ + getCollapsedGroups(): FlowGroup[] { + if (!this.#runtimeInitialized) { + return this.#getGroups().filter((g) => g.autocollapse) + } + return this.#getGroups().filter((g) => this.#runtimeCollapsedIds.has(groupKey(g))) + } +} + +/** + * Utility class for editing flow groups via direct flowStore mutations. + * Follows the same pattern as NoteEditor. + */ +export class GroupEditor { + private flowStore: StateStore + + constructor(flowStore: StateStore) { + this.flowStore = flowStore + } + + getGroups(): FlowGroup[] { + return this.flowStore.val.value?.groups || [] + } + + private setGroups(groups: FlowGroup[]): void { + if (this.flowStore.val.value) { + this.flowStore.val.value.groups = groups + } + } + + /** IDs that cannot be part of a group (preprocessor, failure module) */ + getExcludeIds(): Set { + const excludeIds = new Set() + const pp = this.flowStore.val.value?.preprocessor_module?.id + if (pp) excludeIds.add(pp) + const fm = this.flowStore.val.value?.failure_module?.id + if (fm) excludeIds.add(fm) + return excludeIds + } + + /** Check whether the given selection can form a valid group */ + canCreateGroup( + selectedIds: string[], + flowNodes: { id: string; parentIds?: string[] }[] + ): boolean { + const result = canFormValidGroup(selectedIds, flowNodes, this.getExcludeIds()) + if (!result.valid) return false + // Reject if a group with the same boundaries already exists + return !this.getGroups().some((g) => g.start_id === result.startId && g.end_id === result.endId) + } + + /** + * Create a new group from selected node IDs. + * Uses canFormValidGroup to determine start_id and end_id. + * Returns the generated group ID. + */ + createGroup( + moduleIds: string[], + flowNodes: { id: string; parentIds?: string[] }[] + ): string | undefined { + // Filter subflow node IDs (same logic as NoteEditor.createGroupNote) + let filteredIds = [...moduleIds] + const subflowIds: string[] = [] + for (const id of moduleIds) { + if (id.startsWith('subflow:')) { + const match = id.match(/^subflow:([^:]+)/) + if (match) { + subflowIds.push(match[1]) + } + } + } + if (subflowIds.length > 0) { + filteredIds = filteredIds.filter((id) => !subflowIds.includes(id)) + filteredIds = [...filteredIds, ...subflowIds] + } + + const result = canFormValidGroup(filteredIds, flowNodes, this.getExcludeIds()) + if (!result.valid) return undefined + + const groups = this.getGroups() + + // Reject duplicate: a group with the same boundaries already exists + if (groups.some((g) => g.start_id === result.startId && g.end_id === result.endId)) { + return undefined + } + const usedColors = new Set() + for (const group of groups) { + if (group.color) { + usedColors.add(group.color as NoteColor) + } + } + const color = usedColors.size > 0 ? getNextAvailableColor(usedColors) : DEFAULT_GROUP_NOTE_COLOR + + const newGroup: FlowGroup = { + start_id: result.startId, + end_id: result.endId, + color + } + this.setGroups([...groups, newGroup]) + return groupKey(newGroup) + } + + deleteGroup(groupId: string): void { + const groups = this.getGroups() + this.setGroups(groups.filter((g) => groupKey(g) !== groupId)) + } + + updateColor(groupId: string, color: NoteColor): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, color } : g))) + } + + updateSummary(groupId: string, summary: string): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, summary } : g))) + } + + updateNote(groupId: string, note: string | undefined): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, note } : g))) + } + + /** Add a note to a group (sets note to empty string to trigger the placeholder UI) */ + addNote(groupId: string): void { + this.updateNote(groupId, '') + } + + /** Remove a note from a group */ + removeNote(groupId: string): void { + this.updateNote(groupId, undefined) + } + + updateAutocollapse(groupId: string, autocollapse: boolean): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, autocollapse } : g))) + } +} + +export type GroupEditorContext = { + groupEditor: GroupEditor + canCreateGroup: StateStore +} + +const CONTEXT_KEY = 'GroupEditorContext' + +export function setGroupEditorContext( + groupEditor: GroupEditor, + canCreateGroup: StateStore +): void { + setContext(CONTEXT_KEY, { groupEditor, canCreateGroup }) +} + +export function getGroupEditorContext(): GroupEditorContext | undefined { + return getContext(CONTEXT_KEY) +} + +/** Height of the group header bar */ +export const GROUP_HEADER_HEIGHT = 22 + +/** Extra margin between the header and the first node */ +export const GROUP_TOP_MARGIN = 30 + +export type GraphGroup = FlowGroup & { + id: string + moduleIds: string[] +} + +export type ContainerInnerArray = { + get: () => FlowModule[] + set: (v: any) => void + label?: string +} + +/** Get inner arrays from a container FlowModule with direct get/set accessors. */ +export function getContainerInnerArrays(mod: FlowModule): ContainerInnerArray[] { + const val = mod.value as any + if (val.type === 'forloopflow' || val.type === 'whileloopflow') { + return [ + { + get: () => val.modules, + set: (v) => { + val.modules = v + } + } + ] + } else if (val.type === 'branchone') { + return [ + { + get: () => val.default, + set: (v) => { + val.default = v + }, + label: 'Default' + }, + ...val.branches.map((b: any, i: number) => ({ + get: () => b.modules, + set: (v: any) => { + b.modules = v + }, + label: b.summary || `Branch ${i + 1}` + })) + ] + } else if (val.type === 'branchall') { + return val.branches.map((b: any, i: number) => ({ + get: () => b.modules, + set: (v: any) => { + b.modules = v + }, + label: b.summary || `Branch ${i + 1}` + })) + } + return [] +} diff --git a/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts b/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts new file mode 100644 index 0000000000..6965baf531 --- /dev/null +++ b/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts @@ -0,0 +1,181 @@ +import { untrack } from 'svelte' +import type { FlowModule } from '$lib/gen' +import { type FlowGroup, type GraphGroup, groupKey } from './groupEditor.svelte' +import type { StateStore } from '$lib/utils' +import { getAllModules } from '../flows/flowExplorer' +import { computeGroupModuleIds } from './groupDetectionUtils' +import { stateSnapshot } from '$lib/svelte5Utils.svelte' +import { + buildStructureTree, + deriveGroupsFromStructure, + applyStructureToModules, + removeEmptyGroups, + findDuplicateGroups, + removeDuplicateGroups, + flattenStructureIds, + type FlowStructureNode +} from './flowStructure' + +export type ExtendedOpenFlow = { + value: { + modules: FlowModule[] + groups?: FlowGroup[] + [key: string]: any + } + [key: string]: any +} + +/** + * Reactive read-only view of the flow structure tree. + * The tree is always derived from flowStore (single source of truth). + * Mutations go through prepareMutation: snapshot → mutate → clean empty groups → commit. + */ +export class GroupedModulesProxy { + #items = $state([]) + #error = $state(undefined) + #flowStore: StateStore + + constructor(flowStore: StateStore) { + this.#flowStore = flowStore + this.rebuild() + + // Rebuild tree whenever store changes (undo/load/mutation) + $effect(() => { + void flowStore.val.value.modules + void flowStore.val.value.groups + untrack(() => this.rebuild()) + }) + } + + /** Reactive access to the structure tree (read-only view) */ + get items(): FlowStructureNode[] { + return this.#items + } + + /** Reactive access to build errors */ + get error(): unknown { + return this.#error + } + + /** + * Prepare a structural mutation without writing to the store yet. + * Returns the list of groups that became empty (already removed from the snapshot) + * and a `commit` function that writes the result to the store. + * + * If no groups were emptied, the caller can commit immediately. + * If groups were emptied, the caller should show a confirmation modal + * and call commit() only on user confirmation. + */ + prepareMutation( + mutate: (tree: FlowStructureNode[]) => void, + opts?: { + extraModules?: FlowModule[] + displayState?: import('./groupEditor.svelte').GroupDisplayState + } + ): { + emptiedGroups: FlowGroup[] + duplicateGroups: FlowGroup[] + commit: (commitOpts?: { removeDuplicates?: boolean }) => void + } { + const snapshot = $state.snapshot(this.#items) as FlowStructureNode[] + mutate(snapshot) + + // Clean up empty groups and collect which ones were removed + const emptiedGroups = removeEmptyGroups(snapshot) + // Detect groups that became duplicates after the mutation + const duplicateGroups = findDuplicateGroups(snapshot) + + const commit = (commitOpts?: { removeDuplicates?: boolean }) => { + if (commitOpts?.removeDuplicates && duplicateGroups.length > 0) { + removeDuplicateGroups(snapshot) + } + + // Remap runtime state for groups whose boundaries shifted + if (opts?.displayState) { + this.#remapChangedGroupKeys(snapshot, opts.displayState) + } + + // Build moduleMap lazily at commit time so it reflects the latest store state + const moduleMap = new Map() + for (const m of getAllModules(this.#flowStore.val.value.modules)) { + moduleMap.set(m.id, m) + } + if (opts?.extraModules) { + for (const m of opts.extraModules) { + moduleMap.set(m.id, m) + } + } + this.#flowStore.val.value.modules = applyStructureToModules(snapshot, moduleMap) + this.#flowStore.val.value.groups = deriveGroupsFromStructure(snapshot) + } + + return { emptiedGroups, duplicateGroups, commit } + } + + /** + * Convenience: prepare + auto-commit. Only use for mutations that cannot + * empty groups (e.g. inserts). Throws if groups are unexpectedly emptied. + * For mutations that may empty groups, use prepareMutation() directly. + */ + applyTreeMutation( + mutate: (tree: FlowStructureNode[]) => void, + opts?: { + extraModules?: FlowModule[] + displayState?: import('./groupEditor.svelte').GroupDisplayState + } + ): void { + const { emptiedGroups, duplicateGroups, commit } = this.prepareMutation(mutate, opts) + if (emptiedGroups.length > 0) { + console.error('applyTreeMutation: unexpected empty groups', emptiedGroups) + } + if (duplicateGroups.length > 0) { + console.error('applyTreeMutation: unexpected duplicate groups', duplicateGroups) + } + commit() + } + + /** Remap runtime state for group nodes whose boundaries shifted after a mutation. */ + #remapChangedGroupKeys( + snapshot: FlowStructureNode[], + displayState: import('./groupEditor.svelte').GroupDisplayState + ): void { + const walk = (nodes: FlowStructureNode[]) => { + for (const node of nodes) { + if (node.kind === 'group') { + const oldKey = node.id + const flatIds = flattenStructureIds(node.branches[0].children) + const newKey = flatIds.length > 0 ? `${flatIds[0]}:${flatIds[flatIds.length - 1]}` : null + if (newKey && oldKey !== newKey) { + displayState.remapGroupKey(oldKey, newKey) + } + walk(node.branches[0].children) + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(snapshot) + } + + /** Rebuild from flowStore */ + private rebuild(): void { + const modules = stateSnapshot(this.#flowStore.val.value.modules) as FlowModule[] + const allGroups = this.#flowStore.val.value.groups ?? [] + const allModules = getAllModules(modules) + const graphGroups: GraphGroup[] = allGroups.map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: computeGroupModuleIds(g.start_id, g.end_id, allModules) + })) + try { + this.#items = buildStructureTree(modules, graphGroups) + this.#error = undefined + } catch (e) { + // Intentionally preserve last-known-good #items so the graph + // can still render while the error is surfaced to the user. + this.#error = e + } + } +} diff --git a/frontend/src/lib/components/graph/moveManager.svelte.ts b/frontend/src/lib/components/graph/moveManager.svelte.ts index 605c4b849a..1539cc0802 100644 --- a/frontend/src/lib/components/graph/moveManager.svelte.ts +++ b/frontend/src/lib/components/graph/moveManager.svelte.ts @@ -204,9 +204,6 @@ export class MoveManager { for (const [edgeId, zone] of this.#registeredDropZones) { if (zone.disableMoveIds.includes(draggedId)) continue - // Skip edges adjacent to the dragged node (no-op move) - if (zone.sourceId === draggedId || zone.targetId === draggedId) continue - const dx = Math.abs(flowPos.x - zone.centerX) const dy = Math.abs(flowPos.y - zone.centerY) diff --git a/frontend/src/lib/components/graph/nodeExtraSpace.ts b/frontend/src/lib/components/graph/nodeExtraSpace.ts new file mode 100644 index 0000000000..f51309a60b --- /dev/null +++ b/frontend/src/lib/components/graph/nodeExtraSpace.ts @@ -0,0 +1,153 @@ +import type { FlowNote } from '../../gen' +import type { AssetWithAltAccessType } from '../assets/lib' +import { + assetDisplaysAsInputInFlowGraph, + assetDisplaysAsOutputInFlowGraph, + NODE_WITH_READ_ASSET_Y_OFFSET, + NODE_WITH_WRITE_ASSET_Y_OFFSET +} from './renderers/nodes/AssetNode.svelte' +import { + AI_TOOL_BASE_OFFSET, + AI_TOOL_ROW_OFFSET, + BELOW_ADDITIONAL_OFFSET +} from './renderers/nodes/AIToolNode.svelte' +import { topologicalSort } from './graphBuilder.svelte' +import { GROUP_HEADER_HEIGHT } from './groupEditor.svelte' +import type { GroupDisplayState } from './groupEditor.svelte' +import type { GraphModuleState } from '.' + +type NodeDep = { + id: string + parentIds?: string[] + data?: { assets?: AssetWithAltAccessType[]; module?: any } +} + +type ExtraSpace = { top: number; bottom: number; left: number; right: number } + +const MAX_TOOLS_PER_ROW = 2 + +/** + * Pre-compute extra top/bottom space each node needs for decorations + * (assets, AI tools, group headers, group notes). + */ +export function computeNodeExtraSpace( + graphNodes: NodeDep[], + opts: { + showAssets: boolean + showNotes: boolean + notes: FlowNote[] | undefined + noteTextHeights: Record + groupDisplayState: GroupDisplayState + insertable: boolean + flowModuleStates: Record | undefined + } +): Map | undefined { + const extraSpace = new Map() + + // 1. Assets + if (opts.showAssets) { + for (const node of graphNodes) { + const assets = node.data?.assets ?? [] + if (!assets.length) continue + const hasRead = assets.some(assetDisplaysAsInputInFlowGraph) + const hasWrite = assets.some(assetDisplaysAsOutputInFlowGraph) + if (hasRead || hasWrite) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + top: prev.top + (hasRead ? NODE_WITH_READ_ASSET_Y_OFFSET : 0), + bottom: prev.bottom + (hasWrite ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0) + }) + } + } + } + + // 2. AI tools + for (const node of graphNodes) { + const mod = node.data?.module + if (!mod || mod.value?.type !== 'aiagent') continue + + const agentActions = !opts.insertable && opts.flowModuleStates?.[node.id]?.agent_actions + + if (agentActions) { + // Execution mode: tools below + const totalRows = Math.ceil(agentActions.length / MAX_TOOLS_PER_ROW) + const space = AI_TOOL_BASE_OFFSET + AI_TOOL_ROW_OFFSET * totalRows + BELOW_ADDITIONAL_OFFSET + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { ...prev, bottom: prev.bottom + space }) + } else { + // Edit mode: tools above + const tools = mod.value.tools ?? [] + const totalRows = Math.ceil(tools.length / MAX_TOOLS_PER_ROW) + (opts.insertable ? 1 : 0) + const space = AI_TOOL_BASE_OFFSET + AI_TOOL_ROW_OFFSET * totalRows + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { ...prev, top: prev.top + space }) + } + } + + // Topological sort (reversed: top-of-graph first) — shared by group notes and group headers + const sortedNodes = topologicalSort(graphNodes).reverse() + + // 3. Group notes (text above topmost node in each group note) + if (opts.showNotes) { + const groupNotes = (opts.notes ?? []).filter((n) => n.type === 'group') + if (groupNotes.length > 0) { + for (const groupNote of groupNotes) { + if (!groupNote.contained_node_ids?.length) continue + const topmostNodeId = sortedNodes.find((node) => + groupNote.contained_node_ids?.includes(node.id) + )?.id + if (topmostNodeId) { + const textHeight = opts.noteTextHeights[groupNote.id] || 60 + const spacing = textHeight + 16 // padding + const prev = extraSpace.get(topmostNodeId) ?? { + top: 0, + bottom: 0, + left: 0, + right: 0 + } + extraSpace.set(topmostNodeId, { + ...prev, + top: Math.max(prev.top, spacing + prev.top) + }) + } + } + } + } + + // 4. Collapsed group nodes are taller than regular nodes (header + module icons) + for (const node of graphNodes) { + if (node.id.startsWith('collapsed-group:')) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + bottom: prev.bottom + GROUP_HEADER_HEIGHT + }) + } + } + + // 5. Group nodes (expanded heads and collapsed) with notes need extra height + if (opts.showNotes) { + const noteHeights = opts.groupDisplayState.getNoteHeights() + for (const node of graphNodes) { + let groupId: string | undefined + if (node.id.startsWith('group:') && !node.id.endsWith('-end')) { + groupId = node.id.slice('group:'.length) + } else if (node.id.startsWith('collapsed-group:')) { + groupId = node.id.slice('collapsed-group:'.length) + } + if (groupId) { + const noteHeight = noteHeights[groupId] + if (noteHeight && noteHeight > 0) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + bottom: prev.bottom + noteHeight + }) + } + } + } + } + + return extraSpace.size > 0 ? extraSpace : undefined +} diff --git a/frontend/src/lib/components/graph/noteColors.ts b/frontend/src/lib/components/graph/noteColors.ts index f9024adf7e..2a82ed1f40 100644 --- a/frontend/src/lib/components/graph/noteColors.ts +++ b/frontend/src/lib/components/graph/noteColors.ts @@ -14,6 +14,7 @@ export enum NoteColor { export interface NoteColorConfig { background: string + backgroundLight: string outline: string outlineHover: string text: string @@ -24,70 +25,80 @@ export interface NoteColorConfig { export const NOTE_COLORS: Record = { [NoteColor.YELLOW]: { background: 'bg-yellow-200 dark:bg-yellow-900', - outline: 'outline-yellow-300 dark:outline-yellow-600', + backgroundLight: 'bg-yellow-400/5 dark:bg-yellow-600/5', + outline: 'outline-yellow-200 dark:outline-yellow-900', outlineHover: 'outline-yellow-300/60 dark:outline-yellow-600/60', text: 'text-yellow-900 dark:text-yellow-100', hover: 'hover:bg-yellow-200 dark:hover:bg-yellow-800' }, [NoteColor.BLUE]: { background: 'bg-blue-100 dark:bg-blue-950', - outline: 'outline-blue-300 dark:outline-blue-600', + backgroundLight: 'bg-blue-400/5 dark:bg-blue-600/5', + outline: 'outline-blue-100 dark:outline-blue-950', outlineHover: 'outline-blue-300/60 dark:outline-blue-600/60', text: 'text-blue-900 dark:text-blue-100', hover: 'hover:bg-blue-200 dark:hover:bg-blue-800' }, [NoteColor.GREEN]: { background: 'bg-green-200 dark:bg-green-900', - outline: 'outline-green-300 dark:outline-green-600', + backgroundLight: 'bg-green-400/5 dark:bg-green-600/5', + outline: 'outline-green-200 dark:outline-green-900', outlineHover: 'outline-green-300/60 dark:outline-green-600/60', text: 'text-green-900 dark:text-green-100', hover: 'hover:bg-green-200 dark:hover:bg-green-800' }, [NoteColor.PURPLE]: { background: 'bg-purple-200 dark:bg-purple-900', - outline: 'outline-purple-300 dark:outline-purple-600', + backgroundLight: 'bg-purple-400/5 dark:bg-purple-600/5', + outline: 'outline-purple-200 dark:outline-purple-900', outlineHover: 'outline-purple-300/60 dark:outline-purple-600/60', text: 'text-purple-900 dark:text-purple-100', hover: 'hover:bg-purple-200 dark:hover:bg-purple-800' }, [NoteColor.PINK]: { background: 'bg-pink-200 dark:bg-pink-900', - outline: 'outline-pink-300 dark:outline-pink-600', + backgroundLight: 'bg-pink-400/5 dark:bg-pink-600/5', + outline: 'outline-pink-200 dark:outline-pink-900', outlineHover: 'outline-pink-300/60 dark:outline-pink-600/60', text: 'text-pink-900 dark:text-pink-100', hover: 'hover:bg-pink-200 dark:hover:bg-pink-800' }, [NoteColor.ORANGE]: { background: 'bg-orange-200 dark:bg-orange-900', - outline: 'outline-orange-300 dark:outline-orange-600', + backgroundLight: 'bg-orange-400/5 dark:bg-orange-600/5', + outline: 'outline-orange-200 dark:outline-orange-900', outlineHover: 'outline-orange-300/60 dark:outline-orange-600/60', text: 'text-orange-900 dark:text-orange-100', hover: 'hover:bg-orange-200 dark:hover:bg-orange-800' }, [NoteColor.RED]: { background: 'bg-red-200 dark:bg-red-900', - outline: 'outline-red-300 dark:outline-red-600', + backgroundLight: 'bg-red-400/5 dark:bg-red-600/5', + outline: 'outline-red-200 dark:outline-red-900', outlineHover: 'outline-red-300/60 dark:outline-red-600/60', text: 'text-red-900 dark:text-red-100', hover: 'hover:bg-red-200 dark:hover:bg-red-800' }, [NoteColor.CYAN]: { background: 'bg-cyan-200 dark:bg-cyan-900', - outline: 'outline-cyan-300 dark:outline-cyan-600', + backgroundLight: 'bg-cyan-400/5 dark:bg-cyan-600/5', + outline: 'outline-cyan-200 dark:outline-cyan-900', outlineHover: 'outline-cyan-300/60 dark:outline-cyan-600/60', text: 'text-cyan-900 dark:text-cyan-100', hover: 'hover:bg-cyan-200 dark:hover:bg-cyan-800' }, [NoteColor.LIME]: { background: 'bg-lime-200 dark:bg-lime-900', - outline: 'outline-lime-300 dark:outline-lime-600', + backgroundLight: 'bg-lime-400/5 dark:bg-lime-600/5', + outline: 'outline-lime-200 dark:outline-lime-900', outlineHover: 'outline-lime-300/60 dark:outline-lime-600/60', text: 'text-lime-900 dark:text-lime-100', hover: 'hover:bg-lime-200 dark:hover:bg-lime-800' }, [NoteColor.GRAY]: { background: 'bg-gray-200 dark:bg-gray-800', - outline: 'outline-gray-300 dark:outline-gray-600', + backgroundLight: 'bg-gray-400/5 dark:bg-gray-600/5', + outline: 'outline-gray-200 dark:outline-gray-800', outlineHover: 'outline-gray-300/60 dark:outline-gray-600/60', text: 'text-gray-900 dark:text-gray-100', hover: 'hover:bg-gray-200 dark:hover:bg-gray-700' diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts index 3350e07ce8..bd7fc2ce05 100644 --- a/frontend/src/lib/components/graph/noteUtils.svelte.ts +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -19,20 +19,6 @@ export type NodeDep = { export type NoteComputeResult = { noteNodes: (Node & NodeLayout)[] - newNodePositions: Record -} - -export type AIToolSpacingInfo = { - toolNodes: (Node & NodeLayout)[] - toolEdges: any[] - newNodePositions: Record -} - -export interface GroupNoteBounds { - x: number - y: number - width: number - height: number } let computeNoteNodesCache: @@ -283,14 +269,9 @@ export function computeNoteNodes( const allNoteNodes: (Node & NodeLayout)[] = [] - // Build a map of Y positions that need extra spacing for group notes - const yPosMap: Record = {} // Y position -> spacing needed - - // Group notes that need spacing + // Find topmost node per group note for layout calculation const groupNotes = notes.filter((n) => n.type === 'group') - const topMostNodesMap: Record = {} - const sortedNodes = topologicalSort(nodes).reverse() for (const groupNote of groupNotes) { @@ -298,47 +279,12 @@ export function computeNoteNodes( const topmostNodeId = sortedNodes.find((node) => groupNote.contained_node_ids?.includes(node.id) )?.id - const topmostNode = nodes.find((node) => node.id === topmostNodeId) - if (topmostNode) { - const textHeight = noteTextHeights[groupNote.id] || 60 - const spacing = textHeight + 16 // padding - // Mark this Y position as needing spacing - yPosMap[topmostNode.position.y] = Math.max(yPosMap[topmostNode.position.y] || 0, spacing) - topMostNodesMap[groupNote.id] = topmostNode.id + if (topmostNodeId) { + topMostNodesMap[groupNote.id] = topmostNodeId } } } - // Calculate new positions for nodes (offset by group notes) - const sortedNewNodes = nodes - .map((n) => ({ position: { ...n.position }, id: n.id })) - .sort((a, b) => a.position.y - b.position.y) - - let currentYOffset = 0 - let prevYPos = NaN - - for (const node of sortedNewNodes) { - if (node.position.y !== prevYPos) { - // Add spacing for group notes at this Y level - if (yPosMap[node.position.y]) { - currentYOffset += yPosMap[node.position.y] - } - prevYPos = node.position.y - } - node.position.y += currentYOffset - } - - // Create note nodes AFTER calculating adjusted node positions - // For group notes, we need to use the adjusted node positions - const adjustedNodes = sortedNewNodes.map((n) => { - const origNode = nodes.find((orig) => orig.id === n.id) - return { - ...n, - data: origNode?.data, - type: origNode?.type - } - }) - // Calculate all z-indexes at once using hierarchy information const noteZIndexes = calculateAllNoteZIndexes(notes, nodes) @@ -346,11 +292,11 @@ export function computeNoteNodes( const isGroupNote = note.type === 'group' const zIndex = noteZIndexes[note.id] - // Calculate position and size using adjusted node positions for group notes + // Calculate position and size using node positions for group notes const { position, size } = isGroupNote ? calculateGroupNoteLayout( note, - adjustedNodes, + nodes, noteTextHeights[note.id] || 60, topMostNodesMap[note.id] ) @@ -375,13 +321,8 @@ export function computeNoteNodes( allNoteNodes.push(noteNode) } - const newNodePositions: Record = Object.fromEntries( - sortedNewNodes.map((n) => [n.id, n.position]) - ) - const result: NoteComputeResult = { - noteNodes: allNoteNodes, - newNodePositions + noteNodes: allNoteNodes } // Cache the result diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index 8f1ca0017f..5fbf084374 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -12,11 +12,14 @@ import type { GraphModuleState } from '../../model' import InsertModuleButton from '$lib/components/flows/map/InsertModuleButton.svelte' import { getGraphContext } from '../../graphContext' + import { GROUP_TOP_PADDING } from '$lib/components/graph/compoundLayout' const { useDataflow, showAssets, moveManager } = getGraphContext() let { id, + source, + target, sourceX, sourceY, sourcePosition, @@ -45,6 +48,13 @@ } } = $props() + // Derive group boundary from source/target node IDs + let groupBoundary: 'top' | 'bottom' | undefined = $derived.by(() => { + if (source.startsWith('group:') && !source.endsWith('-end')) return 'top' + if (target.startsWith('group:') && target.endsWith('-end')) return 'bottom' + return undefined + }) + let [edgePath] = $derived( getBezierPath({ sourceX, @@ -75,9 +85,15 @@ ) let centerY = $derived( - sourceY + - 32 + - (data.shouldOffsetInsertBtnDueToAssetNode && $showAssets ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0) + groupBoundary === 'bottom' + ? targetY + : groupBoundary === 'top' + ? sourceY + GROUP_TOP_PADDING / 2 + : sourceY + + 32 + + (data.shouldOffsetInsertBtnDueToAssetNode && $showAssets + ? NODE_WITH_WRITE_ASSET_Y_OFFSET + : 0) ) let isDragging = $derived(!!moveManager?.dragging) @@ -87,13 +103,13 @@ data?.insertable && draggedId !== undefined && !data.disableMoveIds?.includes(draggedId) && - data.sourceId !== draggedId && - data.targetId !== draggedId + source !== draggedId && + target !== draggedId ) - let isNearestDrop = $derived(isValidDropTarget && moveManager?.nearestDropZone?.edgeId === id) - let isAdjacentToDragged = $derived( - isDragging && (data?.sourceId === draggedId || data?.targetId === draggedId) + let isNearestDrop = $derived( + isValidDropTarget && moveManager?.nearestDropZone?.edgeId === id ? true : false ) + let isAdjacentToDragged = $derived(isDragging && (source === draggedId || target === draggedId)) // Register this edge's drop zone position with the drag manager so proximity // detection uses the actual xyflow-computed position rather than re-deriving it. @@ -161,7 +177,7 @@ {@render dropTargetIndicator(isNearestDrop)}
    - {:else if data?.insertable && !$useDataflow && !moveManager?.movingModuleId && !isDragging} + {:else if data?.insertable && !groupBoundary && !$useDataflow && !moveManager?.movingModuleId && !isDragging}
    - {#if !(moveManager.movingIds ?? [moveManager.movingModuleId]).some((id) => data.disableMoveIds?.includes(id))} + {#if !(moveManager.movingIds ?? [moveManager.movingModuleId]).some( (id) => data.disableMoveIds?.includes(id) )} - {/if} -
    +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte index 2ffa6a8d49..ccc5643981 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte @@ -53,7 +53,7 @@
    +
    + {#if hubSyncStatus === 'success'} +
    + + {hubSyncMessage} + +
    + {:else if hubSyncStatus === 'error'} +
    + + {hubSyncMessage} + +
    + {/if} + {/if} + + +{/if} diff --git a/frontend/src/lib/components/settings/AIPromptsModal.svelte b/frontend/src/lib/components/settings/AIPromptsModal.svelte index a0ea762c8d..b9c126d19c 100644 --- a/frontend/src/lib/components/settings/AIPromptsModal.svelte +++ b/frontend/src/lib/components/settings/AIPromptsModal.svelte @@ -13,7 +13,7 @@ onSave?: () => void onReset: () => void hasChanges: boolean - isWorkspaceSettings?: boolean + scope?: 'user' | 'workspace' | 'instance' } let { @@ -22,7 +22,7 @@ onSave, onReset, hasChanges, - isWorkspaceSettings = false + scope = 'user' }: Props = $props() const placeholders: Record = { @@ -63,9 +63,12 @@
    - {#if isWorkspaceSettings} + {#if scope === 'workspace'} Customize the system prompts for each AI mode. These prompts apply to all workspace members. + {:else if scope === 'instance'} + Customize the system prompts for each AI mode. These prompts apply to workspaces using + instance AI defaults. {:else} Customize the system prompts for each AI mode. These prompts are stored locally in your browser and apply in addition to workspace-level prompts. diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 2ed3432a3c..dd3cc6e0d0 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -1,5 +1,12 @@ - +
    - -
    - {#each Object.entries(AI_PROVIDERS) as [provider, details]} -
    -
    - { - if (e.detail) { - aiProviders = { - ...aiProviders, - [provider]: { - resource_path: '', - models: - availableAiModels[provider].length > 0 - ? [availableAiModels[provider][0]] - : [] - } - } - - if (availableAiModels[provider].length > 0 && !defaultModel) { - defaultModel = availableAiModels[provider][0] - } - } else { - aiProviders = Object.fromEntries( - Object.entries(aiProviders).filter(([key]) => key !== provider) - ) - if (defaultModel) { - const currentDefaultModel = Object.values(aiProviders).find( - (p) => defaultModel && p.models.includes(defaultModel) - ) - if (!currentDefaultModel) { - defaultModel = undefined - } - } - if (codeCompletionModel) { - const currentCodeCompletionModel = Object.values(aiProviders).find( - (p) => codeCompletionModel && p.models.includes(codeCompletionModel) - ) - if (!currentCodeCompletionModel) { - codeCompletionModel = undefined - } - } - } - }} - /> - {#if provider === 'anthropic'} - - Recommended - - Anthropic models handle tool calls better than other providers, which makes them a - better choice for AI chat. - - - {/if} -
    - - {#if aiProviders[provider]} -
    -
    - {/if} -
    + + {#key Object.keys(aiProviders).length} + + +
    {/if}
    - + + + + +
    + + {#if promptCount > 0} + ({promptCount} configured) + {/if} + {#if hasPromptsChanges} + Unsaved changes + {/if} +
    +     + {/if}
    - onDiscard?.()} - saveLabel="Save AI settings" - disabled={!Object.values(aiProviders).every((p) => p.resource_path) || - (codeCompletionModel != undefined && codeCompletionModel.length === 0) || - (Object.keys(aiProviders).length > 0 && !defaultModel)} -/> +{#if showWorkspaceOverrideEditor} + +{/if} diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte index 7f4b4e0f67..e02facb38e 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspace.svelte @@ -11,7 +11,8 @@ VariableService, WorkspaceService, type AIProvider, - type CompletedJob + type CompletedJob, + type GetCopilotInfoResponse } from '$lib/gen' import { validateUsername } from '$lib/utils' import { logoutWithRedirect } from '$lib/logoutKit' @@ -52,6 +53,10 @@ let aiKey = $state('') let codeCompletionEnabled = $state(true) let checking = $state(false) + let createLoading = $state(false) + let aiSetupLoading = $state(false) + let creationStep = $state<'details' | 'ai'>('details') + let createdWorkspaceId: string | undefined = $state(undefined) let workspaceColor: string | undefined = $state(undefined) let colorEnabled = $state(false) @@ -85,6 +90,64 @@ let errorMsgs: string[] = $state([]) let failedSyncJobs: string[] = $state([]) + function getErrorMessage(error: any): string { + return ( + error?.body?.error?.message || + error?.body?.message || + (typeof error?.body === 'string' ? error.body : null) || + error?.message || + 'Unknown error' + ) + } + + function hasEffectiveAi(copilotInfo: GetCopilotInfoResponse): boolean { + return Object.keys(copilotInfo.providers ?? {}).length > 0 + } + + async function finishWorkspaceSetup(workspaceId: string): Promise { + usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) + switchWorkspace(workspaceId) + goto(rd ?? '/') + } + + async function getWorkspaceUsername(workspaceId: string): Promise { + if (!automateUsernameCreation) { + return username + } + + const user = await UserService.whoami({ + workspace: workspaceId + }) + return user.username + } + + async function maybeShowAiSetupStep(workspaceId: string): Promise { + try { + const copilotInfo = await WorkspaceService.getCopilotInfo({ + workspace: workspaceId + }) + + if (hasEffectiveAi(copilotInfo)) { + await finishWorkspaceSetup(workspaceId) + return + } + } catch (error) { + console.error('Failed to check effective AI configuration for new workspace', error) + sendUserToast( + 'Workspace created, but Windmill AI availability could not be verified. You can configure it later in Workspace settings.', + true + ) + await finishWorkspaceSetup(workspaceId) + return + } + + createdWorkspaceId = workspaceId + creationStep = 'ai' + aiKey = '' + codeCompletionEnabled = true + selected = 'openai' + } + async function fetchFailedSyncJobs(jobs: string[]): Promise { let ret: CompletedJob[] = [] for (const job of jobs) { @@ -188,20 +251,22 @@ forkCreationLoading = false sendUserToast(`Successfully forked workspace ${$workspaceStore} as: wm-fork-${id}`) + await finishWorkspaceSetup(prefixed_id) } else { sendUserToast('No workspace selected, cannot fork non-existent workspace', true) } } else { - await createWorkspace() + createLoading = true + try { + const workspaceId = await createWorkspace() + await maybeShowAiSetupStep(workspaceId) + } finally { + createLoading = false + } } - - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) - switchWorkspace(isFork ? prefixed_id : id) - - goto(rd ?? '/') } - async function createWorkspace(): Promise { + async function createWorkspace(): Promise { await WorkspaceService.createWorkspace({ requestBody: { id, @@ -216,17 +281,23 @@ requestBody: { operator: operatorOnly, invite_all: !isCloudHosted(), auto_add: autoAdd } }) } - if (aiKey != '') { - let actualUsername = username - if (automateUsernameCreation) { - const user = await UserService.whoami({ - workspace: id - }) - actualUsername = user.username - } - let path = `u/${actualUsername}/${selected}_windmill_codegen` + + sendUserToast(`Created workspace id: ${id}`) + return id + } + + async function saveWorkspaceAiSetup(): Promise { + if (!createdWorkspaceId || !aiKey) { + return + } + + aiSetupLoading = true + try { + const actualUsername = await getWorkspaceUsername(createdWorkspaceId) + const path = `u/${actualUsername}/${selected}_windmill_codegen` + await VariableService.createVariable({ - workspace: id, + workspace: createdWorkspaceId, requestBody: { path, value: aiKey, @@ -235,7 +306,7 @@ } }) await ResourceService.createResource({ - workspace: id, + workspace: createdWorkspaceId, requestBody: { path, value: { @@ -245,40 +316,46 @@ } }) await WorkspaceService.editCopilotConfig({ - workspace: id, - requestBody: aiKey - ? { - providers: { - [selected]: { - resource_path: path, - models: [AI_PROVIDERS[selected].defaultModels[0]] - } - }, - default_model: { - model: AI_PROVIDERS[selected].defaultModels[0], - provider: selected - }, - code_completion_model: codeCompletionEnabled - ? { model: AI_PROVIDERS[selected].defaultModels[0], provider: selected } - : undefined + workspace: createdWorkspaceId, + requestBody: { + providers: { + [selected]: { + resource_path: path, + models: [AI_PROVIDERS[selected].defaultModels[0]] } - : {} + }, + default_model: { + model: AI_PROVIDERS[selected].defaultModels[0], + provider: selected + }, + code_completion_model: codeCompletionEnabled + ? { model: AI_PROVIDERS[selected].defaultModels[0], provider: selected } + : undefined + } }) + + sendUserToast('Windmill AI configured') + await finishWorkspaceSetup(createdWorkspaceId) + } catch (error) { + sendUserToast(`Failed to configure Windmill AI: ${getErrorMessage(error)}`, true) + } finally { + aiSetupLoading = false } - - sendUserToast(`Created workspace id: ${id}`) - - usersWorkspaceStore.set(await WorkspaceService.listUserWorkspaces()) - switchWorkspace(id) - - goto(rd ?? '/') } - function handleKeyUp(event: KeyboardEvent) { + function handleCreateKeyUp(event: KeyboardEvent) { const key = event.key if (key === 'Enter') { event.preventDefault() - createWorkspace() + createOrForkWorkspace() + } + } + + function handleAiKeyUp(event: KeyboardEvent) { + const key = event.key + if (key === 'Enter' && aiKey) { + event.preventDefault() + saveWorkspaceAiSetup() } } @@ -329,6 +406,9 @@ let operatorOnly = $state(false) let autoAdd = $state(true) let selected: Exclude = $state('openai') + let modalTitle = $derived( + isFork ? 'Fork Workspace' : creationStep === 'ai' ? 'Set up Windmill AI' : 'New Workspace' + ) run(() => { id = name.toLowerCase().replace(/\s/gi, '-') }) @@ -344,7 +424,7 @@ let domain = $derived($usersWorkspaceStore?.email.split('@')[1]) - +
    {#if isFork}
    @@ -410,88 +490,184 @@ {/if} {/if} - - - - {#if !automateUsernameCreation} + {#if isFork || creationStep === 'details'} + - {/if} - {#if !isFork} -
    + + {#if !automateUsernameCreation} + + {/if} + {#if !isFork} +
    + + + {#if isCloudHosted() && isDomainAllowed == false} +
    {domain} domain not allowed for auto-invite
    + {/if} + + {#if auto_invite} +
    + + {#if isCloudHosted()} + + {/if} + + +
    + {/if} +
    + {/if} + +
    + + {#if !forkCreationLoading} + + {:else} + + {/if} +
    + {:else} +
    - (optional but recommended) + + Windmill AI powers the chat, code generation, flow creation, and code completion. Set + it up now or configure it later in Workspace settings. + + Learn more + + - + {#snippet children({ item })} @@ -517,7 +704,7 @@ type="password" autocomplete="new-password" bind:value={aiKey} - onkeyup={handleKeyUp} + onkeyup={handleAiKeyUp} /> {#if aiKey} -
    +
    {/if}
    -
    - +
    - {/if} -
    - - {#if !forkCreationLoading} + Skip for now + - {:else} - - {/if} -
    +
    + {/if}
    diff --git a/frontend/src/lib/components/workspaceSettings/InstanceFallbackSettings.svelte b/frontend/src/lib/components/workspaceSettings/InstanceFallbackSettings.svelte new file mode 100644 index 0000000000..a44981ccbd --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/InstanceFallbackSettings.svelte @@ -0,0 +1,93 @@ + + +{#if instanceAiSummary} + +
    +

    + This workspace is currently using the instance AI defaults shown below. +

    + +
    + {#each sortedInstanceProviders as providerSummary} +
    +
    + + {getProviderLabel(providerSummary.provider)} + + Instance +
    +
    + {#each providerSummary.models as model} + {model} + {/each} +
    +
    + {/each} +
    + + {#if instanceAiSummary.default_model} +
    + Default chat model: + {instanceAiSummary.default_model.model} + + ({getProviderLabel(instanceAiSummary.default_model.provider)}) + +
    + {/if} + + {#if instanceAiSummary.code_completion_model} +
    + Code completion model: + + {instanceAiSummary.code_completion_model.model} + + + ({getProviderLabel(instanceAiSummary.code_completion_model.provider)}) + +
    + {/if} +
    +     +{/if} + + +
    +

    + Create workspace-specific AI settings only if this workspace needs to override the active + instance defaults. +

    +
    + +
    +
    +     diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte index 66d9568c09..36f9be8faa 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte @@ -25,13 +25,20 @@ import SettingsPageHeader from '$lib/components/settings/SettingsPageHeader.svelte' import SettingCard from '$lib/components/instanceSettings/SettingCard.svelte' import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' + import InstanceAISettings from '$lib/components/instanceSettings/InstanceAISettings.svelte' const settingsSteps = [ { id: 'Core', label: 'Core' }, { id: 'Auth/OAuth/SAML', label: 'Authentication' } ] as const - const wizardStepLabels = [...settingsSteps.map((s) => s.label), 'Root login & Resource Types'] + const AI_STEP_INDEX = settingsSteps.length + + const wizardStepLabels = [ + ...settingsSteps.map((s) => s.label), + 'AI', + 'Root login & Resource Types' + ] const fullStepLabels = ['Settings', 'Root login & Resource Types'] @@ -67,6 +74,7 @@ }) let instanceSettings: InstanceSettings | undefined = $state() + let instanceAiSettings: InstanceAISettings | undefined = $state() function isSettingsStep(step: number): boolean { return step < settingsSteps.length @@ -148,6 +156,9 @@ let passwordValid = $derived(newPassword.length >= 2) let accountFormValid = $derived(emailValid && passwordValid) + // --- AI step state --- + let aiHasUnsavedChanges = $state(false) + // --- EE license key warning --- let showLicenseKeyWarning = $state(false) let pendingNextCallback: (() => void) | undefined = $state(undefined) @@ -168,9 +179,20 @@ let authSubTab: 'sso' | 'oauth' | 'scim' = $derived(tabToAuthSubTab[fullTab] ?? 'sso') let yamlMode = $state(false) - function handleNavigate(newTab: string) { - if (newTab === fullTab) return + function isAiStepActive(): boolean { + return ( + (mode === 'wizard' && wizardStep === AI_STEP_INDEX) || + (mode === 'full' && fullStep === 0 && fullTab === 'ai' && !yamlMode) + ) + } + + async function handleNavigate(newTab: string): Promise { + if (newTab === fullTab) return true + if (isAiStepActive() && !((await instanceAiSettings?.persistBeforeExit()) ?? true)) { + return false + } fullTab = newTab + return true } // --- Settings search (full mode) --- @@ -180,7 +202,10 @@ let highlightTimeout: ReturnType | undefined async function handleSearchSelect(item: SearchableSettingItem) { - handleNavigate(item.tabId) + const didNavigate = await handleNavigate(item.tabId) + if (!didNavigate) { + return + } if (item.settingKey) { clearTimeout(scrollTimeout) clearTimeout(highlightTimeout) @@ -202,7 +227,7 @@ }) /** Check if we need to warn about missing EE license key before proceeding */ - function proceedFromCore(callback: () => void) { + async function proceedFromCore(callback: () => void) { const leavingSettings = (mode === 'wizard' && wizardStep === 0) || (mode === 'full' && fullStep === 0) if (leavingSettings && isEeImage() && isLicenseKeyEmpty()) { @@ -210,12 +235,16 @@ showLicenseKeyWarning = true return } - saveAndProceed(callback) + await saveAndProceed(callback) } /** Auto-save dirty settings, then run the callback */ async function saveAndProceed(callback: () => void) { - if (yamlMode) { + if (isAiStepActive()) { + if (!((await instanceAiSettings?.persistBeforeExit()) ?? true)) { + return + } + } else if (yamlMode) { // In YAML mode, sync editor → form, then bulk-save everything if (!instanceSettings?.syncBeforeDiff()) return await instanceSettings.saveSettings() @@ -231,11 +260,14 @@ callback() } - function switchToFullMode() { + async function switchToFullMode() { mode = 'full' } - function switchToWizardMode() { + async function switchToWizardMode() { + if (isAiStepActive() && !((await instanceAiSettings?.persistBeforeExit()) ?? true)) { + return + } yamlMode = false fullStep = 0 mode = 'wizard' @@ -461,6 +493,13 @@ tab={settingsSteps[wizardStep].id} /> {/key} + {:else if wizardStep === AI_STEP_INDEX} + {:else} {@render accountSetupContent()} {/if} @@ -505,19 +544,28 @@ {/if}
    - { - const targetTab = categoryToTabMap[category] - if (targetTab) { - handleNavigate(targetTab) - } - }} - /> + {#if fullTab === 'ai' && !yamlMode} + + {:else} + { + const targetTab = categoryToTabMap[category] + if (targetTab) { + handleNavigate(targetTab) + } + }} + /> + {/if}
    {:else} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index d481346af2..db3a140c0b 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -19,10 +19,12 @@ import { OauthService, WorkspaceService, - ResourceService, SettingService, type AIConfig, - type ErrorHandler + type ErrorHandler, + type GetCopilotSettingsStateResponse, + type InstanceAISummary, + type GetSettingsResponse } from '$lib/gen' import { enterpriseLicense, @@ -60,7 +62,6 @@ convertDucklakeSettingsFromBackend, type DucklakeSettingsType } from '$lib/components/workspaceSettings/DucklakeSettings.svelte' - import { AIMode } from '$lib/components/copilot/chat/AIChatManager.svelte' import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import CollapseLink from '$lib/components/CollapseLink.svelte' @@ -112,19 +113,12 @@ let publicAppRateLimitPerMinute: number | undefined = $state(undefined) let initialPublicAppRateLimitPerMinute: number | undefined = $state(undefined) - let aiProviders: Exclude = $state({}) - let codeCompletionModel: string | undefined = $state(undefined) - let defaultModel: string | undefined = $state(undefined) - let customPrompts: Record = $state({}) - let maxTokensPerModel: Record = $state({}) - - // Track initial AI config for unsaved changes detection - let initialAiProviders: Exclude = $state({}) - let initialCodeCompletionModel: string | undefined = $state(undefined) - let initialDefaultModel: string | undefined = $state(undefined) - let initialCustomPrompts: Record = $state({}) - let initialMaxTokensPerModel: Record = $state({}) - + let hasInstanceAiConfig = $state(false) + let usesInstanceAiConfig = $state(false) + let instanceAiSummary: InstanceAISummary | undefined = $state(undefined) + let aiInitialConfig: AIConfig | undefined = $state(undefined) + let aiSettingsComponent: AISettings | undefined = $state(undefined) + let hasAiSettingsChanges = $state(false) // Track initial deploy settings for unsaved changes detection let initialWorkspaceToDeployTo: string | undefined = $state(undefined) let initialDeployUiSettings: { @@ -227,14 +221,6 @@ return currentValue !== initialValue }) - // Derived state for checking unsaved changes in AI settings - let hasAiSettingsChanges = $derived.by(() => { - if (tab !== 'ai') return false - const changes = getAiSettingsInitialAndModifiedValues() - if (!changes.savedValue || !changes.modifiedValue) return false - return hasUnsavedChanges(changes.savedValue, changes.modifiedValue) - }) - // Derived state for checking unsaved changes in deployment settings let hasDeploySettingsChanges = $derived.by(() => { if (tab !== 'deploy_to') return false @@ -320,8 +306,6 @@ $page.url.searchParams.get('tab') === 'teams' ? 'teams_commands' : 'slack_commands' ) - let usingOpenaiClientCredentialsOauth = $state(false) - let loadedSettings = $state(false) let oauths: Record = $state({}) @@ -489,7 +473,17 @@ } async function loadSettings(): Promise { - const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) + const [settings, copilotSettingsState]: [ + GetSettingsResponse, + GetCopilotSettingsStateResponse + ] = await Promise.all([ + WorkspaceService.getSettings({ + workspace: $workspaceStore! + }), + WorkspaceService.getCopilotSettingsState({ + workspace: $workspaceStore! + }) + ]) slack_team_name = settings.slack_name teams_team_id = settings.teams_team_id teams_team_name = settings.teams_team_name @@ -508,23 +502,10 @@ workspaceToDeployTo = settings.deploy_to webhook = settings.webhook - aiProviders = settings.ai_config?.providers ?? {} - defaultModel = settings.ai_config?.default_model?.model - codeCompletionModel = settings.ai_config?.code_completion_model?.model - customPrompts = settings.ai_config?.custom_prompts ?? {} - maxTokensPerModel = settings.ai_config?.max_tokens_per_model ?? {} - for (const mode of Object.values(AIMode)) { - if (!(mode in customPrompts)) { - customPrompts[mode] = '' - } - } - - // Store initial AI config state for unsaved changes detection - initialAiProviders = clone(aiProviders) - initialDefaultModel = defaultModel - initialCodeCompletionModel = codeCompletionModel - initialCustomPrompts = clone(customPrompts) - initialMaxTokensPerModel = clone(maxTokensPerModel) + aiInitialConfig = settings.ai_config ?? {} + hasInstanceAiConfig = copilotSettingsState.has_instance_ai_config + usesInstanceAiConfig = copilotSettingsState.uses_instance_ai_config + instanceAiSummary = copilotSettingsState.instance_ai_summary const errorHandler = settings.error_handler as | { path?: string; extra_args?: any; muted_on_cancel?: boolean; muted_on_user_path?: boolean } | undefined @@ -600,12 +581,6 @@ // Store initial success handler state for unsaved changes detection initialSuccessHandlerScriptPath = successHandlerScriptPath - // check openai_client_credentials_oauth - usingOpenaiClientCredentialsOauth = await ResourceService.existsResourceType({ - workspace: $workspaceStore!, - path: 'openai_client_credentials_oauth' - }) - loadedSettings = true } @@ -816,36 +791,6 @@ ) } - // Function to check if there are unsaved changes in AI settings - function getAiSettingsInitialAndModifiedValues() { - const savedValue = { - aiProviders: initialAiProviders, - defaultModel: initialDefaultModel, - codeCompletionModel: initialCodeCompletionModel, - customPrompts: initialCustomPrompts, - maxTokensPerModel: initialMaxTokensPerModel - } - - const modifiedValue = { - aiProviders: aiProviders, - defaultModel: defaultModel, - codeCompletionModel: codeCompletionModel, - customPrompts: customPrompts, - maxTokensPerModel: maxTokensPerModel - } - - return { savedValue, modifiedValue } - } - - // Function to discard unsaved AI settings changes - function discardAiSettingsChanges() { - aiProviders = clone(initialAiProviders) - defaultModel = initialDefaultModel - codeCompletionModel = initialCodeCompletionModel - customPrompts = clone(initialCustomPrompts) - maxTokensPerModel = clone(initialMaxTokensPerModel) - } - // Function to check if there are unsaved changes in storage settings function getStorageSettingsInitialAndModifiedValues() { return { @@ -1017,7 +962,9 @@ case 'windmill_data_tables': return dataTableSettingsComponent?.unsavedChanges() ?? { savedValue: {}, modifiedValue: {} } case 'ai': - return getAiSettingsInitialAndModifiedValues() + return hasAiSettingsChanges + ? { savedValue: { changed: false }, modifiedValue: { changed: true } } + : { savedValue: {}, modifiedValue: {} } case 'windmill_lfs': return getStorageSettingsInitialAndModifiedValues() case 'volume_storage': @@ -1059,7 +1006,7 @@ function discardAllChanges() { switch (tab) { case 'ai': - discardAiSettingsChanges() + aiSettingsComponent?.discard() break case 'windmill_lfs': discardStorageSettingsChanges() @@ -1830,21 +1777,19 @@ export async function main( /> {:else if tab == 'ai'} { - // Update initial state after successful save - initialAiProviders = clone(aiProviders) - initialDefaultModel = defaultModel - initialCodeCompletionModel = codeCompletionModel - initialCustomPrompts = clone(customPrompts) - initialMaxTokensPerModel = clone(maxTokensPerModel) + bind:this={aiSettingsComponent} + initialConfig={aiInitialConfig} + bind:hasUnsavedChanges={hasAiSettingsChanges} + {hasInstanceAiConfig} + {usesInstanceAiConfig} + {instanceAiSummary} + onSave={(copilotSettingsState) => { + if (!copilotSettingsState) { + return + } + hasInstanceAiConfig = copilotSettingsState.has_instance_ai_config + usesInstanceAiConfig = copilotSettingsState.uses_instance_ai_config + instanceAiSummary = copilotSettingsState.instance_ai_summary }} /> {:else if tab == 'windmill_data_tables'} From d578e40101a838d3dffda14157cf72ee4d5a93c0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 21:22:35 +0000 Subject: [PATCH 27/48] feat: add selfApproval option to WAC + inline approval buttons (#8440) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add selfApproval option to WAC waitForApproval + inline approval buttons Add self-approval configuration to WAC workflows and inline approve/reject buttons in WorkflowTimeline. - TS SDK: add selfApproval option to waitForApproval() - Python SDK: add self_approval param to wait_for_approval() - Backend: store approval_conditions in flow_status for WAC, enforce self-approval checks on resume endpoints - Frontend: show Approve/Reject buttons in timeline with form support (EE), gated by user permissions Co-Authored-By: Claude Opus 4.6 (1M context) * fix: revert sqlx query change + regenerate system prompts - Revert get_suspended_flow_info to use original sqlx::query_as! with COALESCE to avoid sqlx offline cache mismatch in CI - Detect WAC by checking if FlowStatus parsing fails + suspend > 0 - Re-fetch flow_status column separately for WAC approval conditions - Regenerate auto-generated system prompt files for SDK changes Co-Authored-By: Claude Opus 4.6 (1M context) * feat: use resume URLs for WAC inline approval buttons - Backend generates HMAC-signed resume/cancel URLs when creating WAC approval, stores them in timeline entry and approval meta - Frontend uses anonymous resume endpoint (like classic flows) with fallback to resumeSuspendedFlowAsOwner for admins - Buttons show for everyone when URLs are present; server-side self_approval_disabled check enforces restrictions - Show warning for admins/owners when self-approval is disabled - selfApproval: false requires EE (errors at dispatch on CE) - self_approval_disabled check moved outside user_auth_required gate so it works independently - WAC detection no longer requires task import Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add resume_suspended and approval_info endpoints - New approval_token DB table for token-based approval access - New POST /jobs_u/flow/resume_suspended/{job_id} endpoint: - OptAuthed: works with login or approval_token - Checks approval_conditions (self_approval, groups, auth) - Admins/owners bypass rules - New GET /jobs_u/flow/approval_info/{job_id} endpoint: - Returns form, rules, can_approve status - HMAC anonymous endpoint now bypasses all approval_conditions (secret = full capability) - getResumeUrls approvalPage URL now uses token format - WAC approval dispatch generates and stores approval tokens - Mark resumeSuspendedFlowAsOwner as legacy Co-Authored-By: Claude Opus 4.6 (1M context) * feat: simplify frontend to use resume_suspended endpoint - OpenAPI spec updated with resume_suspended and approval_info endpoints - WorkflowTimeline: removed URL parsing, now calls single resumeSuspended endpoint for both approve and reject - Buttons show for any logged-in user viewing the job (backend enforces authorization rules) - Kept self-approval warning for admins Co-Authored-By: Claude Opus 4.6 (1M context) * feat: stateless approval tokens, new approval page, FlowStatusWaitingForEvents update - Replace DB-stored approval tokens with stateless HMAC derivation: token = HMAC(workspace_key, job_id + "approval_token") Verifiable without DB lookup, not reversible to resume secret - Drop approval_token migration (no DB table needed) - FlowStatusWaitingForEvents: use resumeSuspended endpoint instead of URL parsing + resumeSuspendedFlowAsOwner - New approval page route /approve/{ws}/{job}?token= that uses approval_info and resume_suspended endpoints - Old approval page route kept for back-compat Co-Authored-By: Claude Opus 4.6 (1M context) * feat: match old approval page content in new approval page - Add FlowMetadata, JobArgs, FlowGraphV2, DisplayResult - Add approvers with tooltips, flow arguments section - Add admin self-approval bypass warning - Add "Open run details" link - Fetch full job alongside approval_info for all UI data Co-Authored-By: Claude Opus 4.6 (1M context) * fix: filter _MODULES from args, show 'workflow' for WAC approvals Co-Authored-By: Claude Opus 4.6 (1M context) * chore: remove deno template from approval/prompt SuspendDrawer Co-Authored-By: Claude Opus 4.6 (1M context) * fix: approval page form display + hide deno from approval script picker - Fix form schema rendering on new approval page by wrapping flat WAC form schemas in { properties, order } for SchemaForm - Hide deno from the approval step language picker in flow editor Co-Authored-By: Claude Opus 4.6 (1M context) * fix: remove deno from canHaveApproval in script_helpers.ts The insert menu uses canHaveApproval() from script_helpers.ts via FlowInputsQuick, not the displayLang function in FlowInputs.svelte. Revert the unnecessary FlowInputs.svelte change. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: return form schema and description in approval_info for classic flows The approval_info endpoint was returning None for form_schema on classic flows. Now fetches raw_flow to get suspend.resume_form schema, hide_cancel, and the step's completed result for description. Co-Authored-By: Claude Opus 4.6 (1M context) * feat: inline Login component on approval page instead of redirect Show the Login component directly on the approval page when authentication is required. On successful login, reloads user and approval info without navigating away. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: show resume buttons for all users, not just owners The resume_suspended endpoint handles authorization server-side, so the frontend should always show the buttons. Remove isOwner gate and the "cannot resume" message. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: prevent layout shift on resume by removing spinner from cancel button Co-Authored-By: Claude Opus 4.6 (1M context) * fix: prevent resume button expansion by using disabled instead of loading The loading prop adds a Loader2 spinner that expands the button width. Use disabled={loading} instead to prevent layout shift. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: approval page login redirects back with full page reload Set rd to the full URL (starts with http) so Login.redirectUser() uses window.location.href instead of goto(), triggering a full page reload after login. This ensures the approval page re-fetches data as an authenticated user. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: fetch flow definition from flow_version when raw_flow is null Deployed flows don't store raw_flow on the job. Fall back to flow_version table using runnable_id to get suspend settings (form schema, hide_cancel) for the approval_info endpoint. Co-Authored-By: Claude Opus 4.6 (1M context) * feat: show specific reasons when user cannot approve Display whether denial is due to self-approval being disabled, required group membership, or both. Co-Authored-By: Claude Opus 4.6 (1M context) * feat: support both nested and flat form schema in waitForApproval Users can now pass either: waitForApproval({ form: { schema: { name: { type: "string" } } } }) or: waitForApproval({ form: { name: { type: "string" } } }) Both WorkflowTimeline and approval page handle both formats. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: convert sqlx query macros to non-macro for CI offline cache Replace sqlx::query! and sqlx::query_scalar! with sqlx::query and sqlx::query_as to avoid SQLX_OFFLINE cache misses in CI. Also remove unused LogIn import from approval page. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: suppress dead code warning + unused isOwner variable - Add #[allow(dead_code)] to without_flow method (CI -D warnings) - Rename isOwner to _isOwner in FlowStatusWaitingForEvents (unused) Co-Authored-By: Claude Opus 4.6 (1M context) * fix: security and robustness fixes from PR review - Add workspace_id verification in resume_suspended to prevent cross-workspace approval (#3) - Fix token leakage: use relative path for login redirect instead of full URL with token (#4) - Handle getJob failure independently from approval_info so the page works for unauthenticated users (#7) - Clear error state on successful data load (#13) Co-Authored-By: Claude Opus 4.6 (1M context) * fix: address review feedback — shared token gen, rand resume_id, UX - Move generate_approval_token to windmill-common::variables (shared between windmill-api and windmill-worker, eliminates duplicate HMAC) - Use rand::random::() for resume_id instead of DefaultHasher - Stop polling after approve/reject on approval page - Add cancelLoading state to WorkflowTimeline Reject button Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- backend/Cargo.lock | 1 + backend/windmill-api/openapi.yaml | 123 ++++ backend/windmill-api/src/jobs.rs | 530 ++++++++++++++++-- backend/windmill-common/src/variables.rs | 18 + backend/windmill-worker/Cargo.toml | 1 + backend/windmill-worker/src/bun_executor.rs | 89 ++- backend/windmill-worker/src/wac_executor.rs | 24 +- cli/src/guidance/skills.ts | 15 +- .../components/FlowStatusViewerInner.svelte | 1 + .../FlowStatusWaitingForEvents.svelte | 150 ++--- .../lib/components/WorkflowTimeline.svelte | 145 ++++- .../flows/content/SuspendDrawer.svelte | 18 - .../lib/components/runs/JobRunsPreview.svelte | 1 + .../components/scriptEditor/LogPanel.svelte | 1 + frontend/src/lib/script_helpers.ts | 2 +- .../(root)/(logged)/run/[...run]/+page.svelte | 1 + .../approve/[workspace]/[job]/+page.svelte | 359 ++++++++++++ python-client/wmill/wmill/client.py | 11 +- system_prompts/auto-generated/prompts.ts | 9 +- system_prompts/auto-generated/script.md | 9 +- system_prompts/auto-generated/sdks/python.md | 7 +- .../auto-generated/sdks/typescript.md | 2 +- .../skills/write-script-bun/SKILL.md | 2 +- .../skills/write-script-bunnative/SKILL.md | 2 +- .../skills/write-script-deno/SKILL.md | 2 +- .../skills/write-script-nativets/SKILL.md | 2 +- .../skills/write-script-python3/SKILL.md | 7 +- typescript-client/client.ts | 3 + 28 files changed, 1313 insertions(+), 222 deletions(-) create mode 100644 frontend/src/routes/approve/[workspace]/[job]/+page.svelte diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f044478b25..30138085c9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -17500,6 +17500,7 @@ dependencies = [ "gcp_auth", "git-version", "hex", + "hmac", "hudsucker", "hyper-http-proxy", "hyper-tls", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d55596bc72..6576633319 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11088,6 +11088,129 @@ paths: "200": description: Interactive slack approval message sent successfully + /w/{workspace}/jobs_u/flow/resume_suspended/{job_id}: + post: + summary: resume or cancel a suspended flow/WAC job + description: > + Resume or cancel a suspended flow/WAC job. Uses approval rules to + determine authorization. Either a valid approval_token or an + authenticated session is required. + operationId: resumeSuspended + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: job_id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + payload: + description: payload to send to the resumed job + approval_token: + type: string + description: approval token for unauthenticated access + approved: + type: boolean + description: whether to approve (true) or cancel (false) the job + default: true + responses: + "201": + description: job resumed + content: + text/plain: + schema: + type: string + + /w/{workspace}/jobs_u/flow/approval_info/{job_id}: + get: + summary: get approval info for a suspended flow/WAC job + description: > + Get approval info for a suspended flow/WAC job. Returns form schema, + approval rules, and whether the current user can approve. Either a + valid token query parameter or an authenticated session is required. + operationId: getApprovalInfo + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: job_id + in: path + required: true + schema: + type: string + format: uuid + - name: token + in: query + required: false + schema: + type: string + description: approval token for unauthenticated access + responses: + "200": + description: approval info + content: + application/json: + schema: + type: object + required: + - flow_id + - can_approve + - user_auth_required + - approvers + properties: + flow_id: + type: string + format: uuid + form_schema: + description: form schema for the approval step + description: + description: description of the approval step + approval_conditions: + type: object + properties: + user_auth_required: + type: boolean + user_groups_required: + type: array + items: + type: string + self_approval_disabled: + type: boolean + required: + - user_auth_required + - user_groups_required + - self_approval_disabled + can_approve: + type: boolean + description: whether the current user/token holder can approve + user_auth_required: + type: boolean + description: whether user authentication is required to approve + hide_cancel: + type: boolean + description: whether to hide the cancel button in the UI + approvers: + type: array + items: + type: object + required: + - resume_id + - approver + properties: + resume_id: + type: integer + approver: + type: string + /w/{workspace}/jobs_u/resume/{id}/{resume_id}/{signature}: get: summary: resume a job for a suspended flow diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index a99b2737a8..bde3b81a14 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -103,7 +103,7 @@ use windmill_common::{ cache, db::UserDB, error::{self, to_anyhow, Error}, - flow_status::{Approval, FlowStatus, FlowStatusModule}, + flow_status::{Approval, ApprovalConditions, FlowStatus, FlowStatusModule}, flows::{add_virtual_items_if_necessary, resolve_maybe_value, FlowValue}, jobs::{script_path_to_payload, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode}, oauth2::HmacSha256, @@ -401,6 +401,8 @@ pub fn workspace_unauthed_service() -> Router { post(cancel_persistent_script_api), ) .route("/queue/force_cancel/:id", post(force_cancel)) + .route("/flow/resume_suspended/:job_id", post(resume_suspended)) + .route("/flow/approval_info/:job_id", get(get_approval_info)) } pub fn global_root_service() -> Router { @@ -1058,6 +1060,7 @@ impl<'a> GetQuery<'a> { Self { with_code: false, ..self } } + #[allow(dead_code)] fn without_flow(self) -> Self { Self { with_flow: false, ..self } } @@ -2181,7 +2184,7 @@ pub async fn resume_suspended_flow_as_owner( ) -> error::Result { let mut tx = db.begin().await?; - let (flow, job_id) = get_suspended_flow_info(flow_id, &mut tx).await?; + let (flow, job_id, is_wac) = get_suspended_flow_info(flow_id, &mut tx).await?; let flow_path = flow.script_path.as_deref().unwrap_or_else(|| ""); require_owner_of_path(&authed, flow_path)?; @@ -2189,10 +2192,17 @@ pub async fn resume_suspended_flow_as_owner( // Check approval conditions (self-approval, required groups, etc.) if let Some(ref flow_status_value) = flow.flow_status { - if let Ok(flow_status) = serde_json::from_value::(flow_status_value.clone()) { - let trigger_email = flow.email.as_deref().unwrap_or(""); - conditionally_require_authed_user(Some(authed.clone()), flow_status, trigger_email)?; - } + let trigger_email = flow.email.as_deref().unwrap_or(""); + let ac = serde_json::from_value::(flow_status_value.clone()) + .ok() + .and_then(|fs| fs.approval_conditions) + .or_else(|| { + // WAC flows store approval_conditions directly in flow_status JSONB + flow_status_value + .get("approval_conditions") + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + }); + conditionally_require_authed_user(Some(authed.clone()), ac, trigger_email)?; } let value = value.unwrap_or(serde_json::Value::Null); @@ -2208,12 +2218,426 @@ pub async fn resume_suspended_flow_as_owner( ) .await?; - resume_immediately_if_relevant(flow, job_id, &mut tx).await?; + if is_wac { + // WAC: directly decrement suspend counter + if flow.suspend > 0 { + sqlx::query!( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + flow.id, + ) + .execute(&mut *tx) + .await?; + } + } else { + resume_immediately_if_relevant(flow, job_id, &mut tx).await?; + } tx.commit().await?; Ok(StatusCode::CREATED) } +// --- New approval system endpoints --- + +use windmill_common::variables::generate_approval_token; + +/// Verify an approval token against the workspace key + job_id. +async fn validate_approval_token( + db: &DB, + token: &str, + job_id: Uuid, + workspace_id: &str, +) -> error::Result<()> { + let expected = generate_approval_token(workspace_id, job_id, db).await?; + if token != expected { + return Err(Error::NotAuthorized("Invalid approval token".to_string())); + } + Ok(()) +} + +#[derive(Deserialize)] +struct ResumeSuspendedBody { + payload: Option, + approval_token: Option, + approved: Option, +} + +async fn resume_suspended( + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Json(body): Json, +) -> error::Result { + let approved = body.approved.unwrap_or(true); + let value = body.payload.unwrap_or(serde_json::Value::Null); + + // Determine if we have a valid authed user or token + let has_token = if let Some(ref token) = body.approval_token { + validate_approval_token(&db, token, job_id, &w_id) + .await + .is_ok() + } else { + false + }; + + if opt_authed.is_none() && !has_token { + return Err(Error::NotAuthorized( + "Must be logged in or provide a valid approval token".to_string(), + )); + } + + let mut tx = db.begin().await?; + + // Resolve the suspended flow (works for both WAC and classic flows) + let (flow, resume_job_id, is_wac) = get_suspended_flow_info(job_id, &mut tx).await?; + + // Verify the job belongs to this workspace + let job_workspace: Option = + sqlx::query_scalar("SELECT workspace_id FROM v2_job WHERE id = $1") + .bind(&flow.id) + .fetch_optional(&mut *tx) + .await?; + if job_workspace.as_deref() != Some(w_id.as_str()) { + return Err(Error::NotFound( + "Job not found in this workspace".to_string(), + )); + } + + // Check approval conditions + let approval_conditions = if is_wac { + flow.flow_status + .as_ref() + .and_then(|v| v.get("approval_conditions")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + } else { + flow.flow_status + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()) + .and_then(|fs| fs.approval_conditions) + }; + + if let Some(ref ac) = approval_conditions { + if ac.user_auth_required && opt_authed.is_none() { + return Err(Error::NotAuthorized( + "This approval requires a logged-in user. Please sign in.".to_string(), + )); + } + } + + // If logged in, check authorization rules + if let Some(ref authed) = opt_authed { + let is_admin = authed.is_admin; + let is_owner = flow + .script_path + .as_deref() + .map(|p| require_owner_of_path(authed, p).is_ok()) + .unwrap_or(false); + + if !is_admin && !is_owner { + let trigger_email = flow.email.as_deref().unwrap_or(""); + conditionally_require_authed_user( + Some(authed.clone()), + approval_conditions.clone(), + trigger_email, + )?; + } + } else if !has_token { + return Err(Error::NotAuthorized( + "Must be logged in or provide a valid approval token".to_string(), + )); + } + + // Generate a unique resume_id + let resume_id: u32 = rand::random(); + + // Check for duplicate + let exists: bool = sqlx::query_scalar("SELECT EXISTS (SELECT 1 FROM resume_job WHERE id = $1)") + .bind(Uuid::from_u128(resume_job_id.as_u128() ^ resume_id as u128)) + .fetch_one(&mut *tx) + .await?; + + if exists { + return Err(Error::BadRequest("Resume request already sent".to_string())); + } + + let approver_value = opt_authed.as_ref().map(|a| a.username.clone()); + + insert_resume_job( + resume_id, + resume_job_id, + &flow, + value, + approver_value.clone(), + approved, + &mut tx, + ) + .await?; + + if !approved { + sqlx::query("UPDATE v2_job_queue SET suspend = 0 WHERE id = $1") + .bind(&flow.id) + .execute(&mut *tx) + .await?; + } else if is_wac { + if flow.suspend > 0 { + sqlx::query("UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1") + .bind(&flow.id) + .execute(&mut *tx) + .await?; + } + } else { + resume_immediately_if_relevant(flow, resume_job_id, &mut tx).await?; + } + + let approver = approver_value.unwrap_or_else(|| "anonymous".to_string()); + let audit_author = if let Some(ref authed) = opt_authed { + AuditAuthor::from(authed) + } else { + AuditAuthor { + email: approver.clone(), + username: approver.clone(), + username_override: None, + token_prefix: None, + } + }; + + audit_log( + &mut *tx, + &audit_author, + "jobs.suspend_resume", + ActionKind::Update, + &w_id, + Some( + &serde_json::json!({ + "approved": approved, + "job_id": job_id, + "details": if approved { + format!("Approved by {}", &approver) + } else { + format!("Cancelled by {}", &approver) + } + }) + .to_string(), + ), + None, + ) + .await?; + + tx.commit().await?; + Ok(StatusCode::CREATED) +} + +#[derive(Deserialize)] +struct ApprovalInfoQuery { + token: Option, +} + +#[derive(Serialize)] +struct ApprovalInfo { + flow_id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + form_schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + approval_conditions: Option, + can_approve: bool, + user_auth_required: bool, + #[serde(skip_serializing_if = "Option::is_none")] + hide_cancel: Option, + approvers: Vec, +} + +async fn get_approval_info( + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Path((w_id, job_id)): Path<(String, Uuid)>, + Query(query): Query, +) -> error::Result> { + // Validate access: either logged in or valid token + let has_token = if let Some(ref token) = query.token { + validate_approval_token(&db, token, job_id, &w_id) + .await + .is_ok() + } else { + false + }; + + if opt_authed.is_none() && !has_token { + return Err(Error::NotAuthorized( + "Must be logged in or provide a valid approval token".to_string(), + )); + } + + // Fetch job info + #[derive(sqlx::FromRow)] + struct ApprovalJobRow { + id: Uuid, + script_path: Option, + email: String, + flow_status: Option, + workflow_as_code_status: Option, + } + let row = sqlx::query_as::<_, ApprovalJobRow>( + "SELECT j.id, j.runnable_path as script_path, j.permissioned_as_email as email, + s.flow_status, s.workflow_as_code_status + FROM v2_job j + LEFT JOIN v2_job_status s ON s.id = j.id + WHERE j.id = $1 AND j.workspace_id = $2", + ) + .bind(&job_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job {job_id} not found")))?; + + let is_wac = row.workflow_as_code_status.is_some(); + + // Extract approval info based on WAC vs classic flow + let (form_schema, description, approval_conditions, hide_cancel) = if is_wac { + let approval_meta = row + .workflow_as_code_status + .as_ref() + .and_then(|v| v.get("_approval")); + let form = approval_meta.and_then(|m| m.get("form").cloned()); + let ac = row + .flow_status + .as_ref() + .and_then(|v| v.get("approval_conditions")) + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + (form, None, ac, None) + } else { + let fs = row + .flow_status + .as_ref() + .and_then(|v| serde_json::from_value::(v.clone()).ok()); + let ac = fs.as_ref().and_then(|s| s.approval_conditions.clone()); + + // For classic flows, form/description come from the flow definition and step result + let approval_step = fs.as_ref().map(|s| (s.step as usize).saturating_sub(1)); + + // Fetch flow definition to get suspend settings (form schema, hide_cancel). + // Try raw_flow on the job first, fall back to flow_version for deployed flows. + let raw_flow: Option = { + let from_job: Option = sqlx::query_scalar( + "SELECT raw_flow FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + + if let Some(v) = from_job { + serde_json::from_value(v).ok() + } else { + // Deployed flow: fetch from flow_version using runnable_id + let from_version: Option = sqlx::query_scalar( + "SELECT fv.value FROM v2_job j JOIN flow_version fv ON fv.id = j.runnable_id \ + WHERE j.id = $1 AND j.workspace_id = $2", + ) + .bind(&job_id) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + from_version.and_then(|v| serde_json::from_value(v).ok()) + } + }; + + let suspend_module = raw_flow + .as_ref() + .and_then(|rf| approval_step.and_then(|s| rf.modules.get(s))); + let suspend_settings = suspend_module.and_then(|m| m.suspend.as_ref()); + + let form = suspend_settings + .and_then(|s| s.resume_form.as_ref()) + .map(|rf| serde_json::json!(rf)); + let hc = suspend_settings.map(|s| s.hide_cancel.unwrap_or(false)); + + // Fetch description and default_args from the step's completed job result + let step_job_id = fs + .as_ref() + .and_then(|s| approval_step.and_then(|step| s.modules.get(step))) + .and_then(|m| m.job()); + let (desc, _default_args) = if let Some(sjid) = step_job_id { + let result: Option = sqlx::query_scalar( + "SELECT result FROM v2_job_completed WHERE id = $1 AND workspace_id = $2", + ) + .bind(sjid) + .bind(&w_id) + .fetch_optional(&db) + .await? + .flatten(); + let desc = result.as_ref().and_then(|r| r.get("description").cloned()); + let da = result.as_ref().and_then(|r| r.get("default_args").cloned()); + (desc, da) + } else { + (None, None) + }; + + (form, desc, ac, hc) + }; + + let user_auth_required = approval_conditions + .as_ref() + .map(|ac| ac.user_auth_required) + .unwrap_or(false); + + // Determine if current user can approve + let can_approve = if let Some(ref authed) = opt_authed { + if authed.is_admin { + true + } else { + let is_owner = row + .script_path + .as_deref() + .map(|p| require_owner_of_path(authed, p).is_ok()) + .unwrap_or(false); + if is_owner { + true + } else { + let trigger_email = row.email.as_str(); + conditionally_require_authed_user( + Some(authed.clone()), + approval_conditions.clone(), + trigger_email, + ) + .is_ok() + } + } + } else { + // Not logged in — can approve only if no auth required + !user_auth_required + }; + + // Get existing approvers + let approvers: Vec = sqlx::query_as::<_, (i32, Option)>( + "SELECT resume_id, approver FROM resume_job WHERE flow = $1", + ) + .bind(&job_id) + .fetch_all(&db) + .await? + .into_iter() + .map(|(rid, approver)| Approval { + resume_id: rid as u16, + approver: approver.unwrap_or_else(|| "anonymous".to_string()), + }) + .collect(); + + Ok(Json(ApprovalInfo { + flow_id: row.id, + form_schema, + description, + approval_conditions, + can_approve, + user_auth_required, + hide_cancel, + approvers, + })) +} + +// --- End new approval system endpoints --- + pub async fn resume_suspended_job( authed: Option, opt_tokened: OptTokened, @@ -2255,26 +2679,8 @@ async fn resume_suspended_job_internal( // Get flow info - works for step-level, flow-level, and WAC approval let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; - // For step-level resumes, verify user auth and flow status - // For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet - // For WAC approvals, skip flow status checks (there is no flow) - if !is_flow_level && !is_wac { - let parent_flow = GetQuery::new() - .without_logs() - .without_code() - .without_flow() - .fetch(&db, &flow_info.id, &w_id) - .await?; - let flow_status = parent_flow - .flow_status() - .ok_or_else(|| anyhow::anyhow!("unable to find the flow status in the flow job"))?; - - let trigger_email = match &parent_flow { - Job::CompletedJob(job) => &job.email, - Job::QueuedJob(job) => &job.email, - }; - conditionally_require_authed_user(authed.clone(), flow_status, trigger_email)?; - } + // HMAC secret = full capability. Skip approval_conditions checks. + // Authorization rules are enforced by the new resume_suspended endpoint instead. let exists = sqlx::query_scalar!( r#" @@ -2540,7 +2946,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI async fn get_suspended_flow_info<'c>( job_id: Uuid, tx: &mut Transaction<'c, Postgres>, -) -> error::Result<(FlowInfo, Uuid)> { +) -> error::Result<(FlowInfo, Uuid, bool)> { let flow = sqlx::query_as!( FlowInfo, r#" @@ -2553,7 +2959,9 @@ async fn get_suspended_flow_info<'c>( .fetch_optional(&mut **tx) .await? .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; - let job_id = flow + + // Try to extract step job_id from FlowStatus modules (classic flow path) + let step_job_id = flow .flow_status .as_ref() .and_then(|v| serde_json::from_value::(v.clone()).ok()) @@ -2562,8 +2970,31 @@ async fn get_suspended_flow_info<'c>( _ => None, }); - if let Some(job_id) = job_id { - Ok((flow, job_id)) + if let Some(step_job_id) = step_job_id { + // Classic flow + Ok((flow, step_job_id, false)) + } else if flow.suspend > 0 { + // WAC approval: no FlowStatus modules, but the job is suspended + // The flow_status here comes from COALESCE(flow_status, workflow_as_code_status), + // so for WAC it may contain approval_conditions from flow_status column + // or the WAC checkpoint from workflow_as_code_status column. + // We need the approval_conditions which are in flow_status column. + // Re-fetch just flow_status (without COALESCE fallback) for the auth check. + let flow_status_only: Option = + sqlx::query_scalar("SELECT flow_status FROM v2_job_status WHERE id = $1") + .bind(&job_id) + .fetch_optional(&mut **tx) + .await? + .flatten(); + + let flow = FlowInfo { + id: flow.id, + flow_status: flow_status_only, + suspend: flow.suspend, + script_path: flow.script_path, + email: flow.email, + }; + Ok((flow, job_id, true)) } else { Err(anyhow::anyhow!("the flow is not in a suspended state anymore").into()) } @@ -2640,7 +3071,11 @@ pub async fn get_suspended_job_flow( Job::CompletedJob(job) => &job.email, Job::QueuedJob(job) => &job.email, }; - conditionally_require_authed_user(authed.clone(), flow_status.clone(), trigger_email)?; + conditionally_require_authed_user( + authed.clone(), + flow_status.approval_conditions.clone(), + trigger_email, + )?; let approvers_from_status = match flow_module_status { FlowStatusModule::Success { approvers, .. } => approvers.to_owned(), @@ -2681,16 +3116,25 @@ pub async fn get_suspended_job_flow( fn conditionally_require_authed_user( _authed: Option, - flow_status: FlowStatus, + approval_conditions_opt: Option, _trigger_email: &str, ) -> error::Result<()> { - let approval_conditions_opt = flow_status.approval_conditions; - if approval_conditions_opt.is_none() { return Ok(()); } let approval_conditions = approval_conditions_opt.unwrap(); + // Check self-approval independently of user_auth_required + if approval_conditions.self_approval_disabled { + if let Some(ref authed) = _authed { + if !authed.is_admin && authed.email.eq(_trigger_email) { + return Err(Error::PermissionDenied( + "Self-approval is disabled for this flow step".to_string(), + )); + } + } + } + if approval_conditions.user_auth_required { { #[cfg(not(feature = "enterprise"))] @@ -2708,13 +3152,6 @@ fn conditionally_require_authed_user( let authed = _authed.unwrap(); if !authed.is_admin { - if approval_conditions.self_approval_disabled && authed.email.eq(_trigger_email) - { - return Err(Error::PermissionDenied( - "Self-approval is disabled for this flow step".to_string(), - )); - } - if !approval_conditions.user_groups_required.is_empty() { #[cfg(feature = "enterprise")] { @@ -2860,11 +3297,18 @@ pub async fn get_resume_urls_internal( .map(|x| format!("?approver={}", encode(x))) .unwrap_or_else(String::new); + // Generate approval token for the new approval page URL. + // The token targets the parent flow/WAC job for proper resolution. + let approval_target_id = get_flow_id_for_job(&db, job_id) + .await + .unwrap_or(target_job_id); + let approval_token = generate_approval_token(&w_id, approval_target_id, &db).await?; + let base_url_str = BASE_URL.read().await.clone(); let base_url = base_url_str.as_str(); let res = ResumeUrls { approvalPage: format!( - "{base_url}/approve/{w_id}/{target_job_id}/{resume_id}/{signature}{approver_query}" + "{base_url}/approve/{w_id}/{approval_target_id}?token={approval_token}" ), cancel: build_resume_url( "cancel", diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 8129f39b3f..e7595f9d1b 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -140,6 +140,24 @@ pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result crate::error::Result { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + let key = get_workspace_key(w_id, db).await?; + let mut mac = Hmac::::new_from_slice(key.as_bytes()) + .map_err(|e| crate::Error::internal_err(format!("HMAC key error: {e}")))?; + mac.update(job_id.as_bytes()); + mac.update(b"approval_token"); + Ok(hex::encode(mac.finalize().into_bytes())) +} + pub async fn get_secret_value_as_admin( db: &DB, w_id: &str, diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index ed82298038..c1e2a4927a 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -110,6 +110,7 @@ gcp_auth = { workspace = true, optional = true } rust_decimal.workspace = true jsonwebtoken.workspace = true sha2.workspace = true +hmac.workspace = true pem = { workspace = true, optional = true } urlencoding.workspace = true nix.workspace = true diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f0ed8793a7..7dbaa6723d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1504,7 +1504,7 @@ async function run() {{ return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null, started_at: dispatch.started_at, duration_ms: dispatch.duration_ms }}; }} if (dispatch.mode === "approval") {{ - return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form }}; + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form, self_approval_disabled: dispatch.self_approval_disabled }}; }} if (dispatch.mode === "sleep") {{ return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; @@ -2634,7 +2634,7 @@ pub async fn handle_wac_v2_output( job.id, num_steps ))) } - WacOutput::Approval { key, timeout, form } => { + WacOutput::Approval { key, timeout, form, self_approval_disabled } => { let db = match conn { Connection::Sql(db) => db, _ => { @@ -2676,11 +2676,91 @@ pub async fn handle_wac_v2_output( .await .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + // Store approval_conditions in flow_status for resume endpoint auth checks + let sad = self_approval_disabled.unwrap_or(false); + if sad { + #[cfg(not(feature = "enterprise"))] + return Err(error::Error::ExecutionErr( + "Disabling self-approval is an enterprise only feature".to_string(), + )); + + #[cfg(feature = "enterprise")] + { + use windmill_common::flow_status::ApprovalConditions; + let approval_conditions = ApprovalConditions { + user_auth_required: true, + user_groups_required: vec![], + self_approval_disabled: true, + }; + sqlx::query( + "UPDATE v2_job_status SET flow_status = JSONB_SET( + COALESCE(flow_status, '{}'::jsonb), + '{approval_conditions}', + $2::jsonb + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&serde_json::json!(approval_conditions)) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to save approval conditions: {e}" + )) + })?; + } + } + + // Generate resume URLs for the inline approval buttons. + // Use a hash of the step key as resume_id so each waitForApproval() + // in the same workflow gets a unique resume_job record. + let resume_id: u32 = { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + key.hash(&mut hasher); + (hasher.finish() & 0xFFFF_FFFF) as u32 + }; + // Generate stateless approval token using shared utility + let approval_token = + windmill_common::variables::generate_approval_token(&job.workspace_id, job.id, db) + .await?; + + let (resume_url, cancel_url, approval_page_url) = { + use hmac::{Hmac, Mac}; + use sha2::Sha256; + use windmill_common::variables::get_workspace_key; + + let wkey = get_workspace_key(&job.workspace_id, db).await?; + let mut mac = Hmac::::new_from_slice(wkey.as_bytes()) + .map_err(|e| error::Error::internal_err(format!("HMAC key error: {e}")))?; + mac.update(job.id.as_bytes()); + mac.update(resume_id.to_be_bytes().as_ref()); + let signature = hex::encode(mac.finalize().into_bytes()); + + let base_url = windmill_common::BASE_URL.read().await.clone(); + let w_id = &job.workspace_id; + let job_id = &job.id; + + let resume = format!( + "{base_url}/api/w/{w_id}/jobs_u/resume/{job_id}/{resume_id}/{signature}" + ); + let cancel = format!( + "{base_url}/api/w/{w_id}/jobs_u/cancel/{job_id}/{resume_id}/{signature}" + ); + let approval_page = + format!("{base_url}/approve/{w_id}/{job_id}?token={approval_token}"); + (resume, cancel, approval_page) + }; + // Store approval form metadata for the approval page endpoint let approval_meta = serde_json::json!({ "key": key, "form": form, "timeout": timeout_secs as u32, + "self_approval_disabled": sad, + "resume": resume_url, + "cancel": cancel_url, + "approvalPage": approval_page_url, }); sqlx::query( "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( @@ -2705,6 +2785,11 @@ pub async fn handle_wac_v2_output( "started_at": &now_str, "name": key, "approval": true, + "self_approval_disabled": sad, + "form": form, + "resume": &resume_url, + "cancel": &cancel_url, + "approvalPage": &approval_page_url, }); let step_timeline_key = format!("_step/{}", key); sqlx::query( diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 208012ccce..9b4ba3d92a 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -59,7 +59,13 @@ pub enum WacOutput { /// No child job is dispatched — the parent suspends directly and resumes /// when a user hits the resume/cancel endpoint. #[serde(rename = "approval")] - Approval { key: String, timeout: Option, form: Option }, + Approval { + key: String, + timeout: Option, + form: Option, + #[serde(default)] + self_approval_disabled: Option, + }, /// Server-side sleep — suspend the workflow for a duration without holding a worker. #[serde(rename = "sleep")] Sleep { key: String, seconds: u32 }, @@ -306,15 +312,13 @@ pub async fn prepare_checkpoint_for_resume( } /// Detect WAC v2 patterns in TypeScript/Bun code. -/// Checks for `import ... from "windmill-client"` containing workflow/task, +/// Checks for `import ... from "windmill-client"` containing workflow, /// skipping comment lines. Handles both single-line and multi-line imports. pub fn is_wac_v2_ts(code: &str) -> bool { let mut has_wac_import = false; let mut has_workflow = false; - let mut has_task = false; let mut in_import_block = false; let mut import_block_has_workflow = false; - let mut import_block_has_task = false; for line in code.lines() { let trimmed = line.trim(); if trimmed.starts_with("//") { @@ -328,34 +332,24 @@ pub fn is_wac_v2_ts(code: &str) -> bool { if trimmed.contains("workflow") { has_workflow = true; } - if trimmed.contains("task") { - has_task = true; - } in_import_block = false; } // Start of multi-line import: import { else if trimmed.starts_with("import") && trimmed.contains("{") && !trimmed.contains("}") { in_import_block = true; import_block_has_workflow = trimmed.contains("workflow"); - import_block_has_task = trimmed.contains("task"); } // Inside multi-line import block else if in_import_block { if trimmed.contains("workflow") { import_block_has_workflow = true; } - if trimmed.contains("task") { - import_block_has_task = true; - } // End of multi-line import: } from "windmill-client" if trimmed.contains("windmill-client") { has_wac_import = true; if import_block_has_workflow { has_workflow = true; } - if import_block_has_task { - has_task = true; - } in_import_block = false; } // End of import block but not windmill-client @@ -367,7 +361,7 @@ pub fn is_wac_v2_ts(code: &str) -> bool { has_workflow = true; } } - has_wac_import && has_workflow && has_task + has_wac_import && has_workflow } /// Detect WAC v2 patterns in Python code. diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index e1354302e6..662c55eb8e 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -740,7 +740,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -1403,7 +1403,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -2129,7 +2129,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -3069,7 +3069,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -4078,12 +4078,17 @@ async def sleep(seconds: int) # # Returns a dict with \`\`value\`\` (form data), \`\`approver\`\`, and \`\`approved\`\`. # +# Args: +# timeout: Approval timeout in seconds (default 1800). +# form: Optional form schema for the approval page. +# self_approval: Whether the user who triggered the flow can approve it (default True). +# # Example:: # # urls = await step("urls", lambda: get_resume_urls()) # await step("notify", lambda: send_email(urls["approvalPage"])) # result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict # Process items in parallel with optional concurrency control. # diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index e9f2361629..2c580bb02b 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -2084,6 +2084,7 @@ stepResults={getStepResults(node.workflow_as_code_status)} result={node.result} success={node.type === 'Success'} + jobId={node.job_id} />
    {/if} diff --git a/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte b/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte index 60f30b1adc..195a1d6fad 100644 --- a/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte +++ b/frontend/src/lib/components/FlowStatusWaitingForEvents.svelte @@ -18,11 +18,9 @@ light?: boolean } - let { isOwner, workspaceId, job, light = false }: Props = $props() + let { isOwner: _isOwner, workspaceId, job, light = false }: Props = $props() let default_payload: object = $state({}) - let resumeUrl: string | undefined = $state(undefined) - let cancelUrl: string | undefined = $state(undefined) let description: any = $state(undefined) let hide_cancel = $state(false) @@ -49,8 +47,6 @@ defaultValues = JSON.parse(JSON.stringify(args)) default_payload = args - resumeUrl = job_result?.['resume'] - cancelUrl = job_result?.['cancel'] hide_cancel = job?.raw_flow?.modules?.[approvalStep]?.suspend?.hide_cancel ?? false schema = mergeSchema( job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema ?? {}, @@ -61,61 +57,19 @@ let loading = $state(false) async function continu(approve: boolean) { loading = true - if ((resumeUrl && approve) || (cancelUrl && !approve)) { - let split = (approve ? resumeUrl : cancelUrl)!.split('/') - let signatureUrl = split.pop() ?? '' - const regex = /([^?]+)(?:\?[^=]+=(\w+))?/ - - const matches = signatureUrl.match(regex) - - const signature = matches?.[1] - if (!signature) { - sendUserToast(`Could not parse signature: ${signatureUrl}`, true) - return - } - const approver = matches?.[2] || undefined - - let resumeId = -1 - let parsedResumeId = split.pop() ?? '' - try { - resumeId = new Number(parsedResumeId).valueOf() - } catch (e) { - console.error(`Could not parse resume id: ${parsedResumeId}`) - } - let jobId = split.pop() ?? '' - if (approve) { - await JobService.resumeSuspendedJobPost({ - workspace: workspaceId ?? $workspaceStore ?? '', - id: jobId, - requestBody: default_payload as any, - resumeId, - signature, - approver - }) - } else { - await JobService.cancelSuspendedJobPost({ - workspace: workspaceId ?? $workspaceStore ?? '', - id: jobId, - resumeId, - signature, - approver, - requestBody: {} - }) - } - } else { - if (approve) { - await JobService.resumeSuspendedFlowAsOwner({ - workspace: workspaceId ?? $workspaceStore ?? '', - id: job?.id ?? '', - requestBody: default_payload as any - }) - } else { - await JobService.cancelQueuedJob({ - workspace: workspaceId ?? $workspaceStore ?? '', - id: job?.id ?? '', - requestBody: {} - }) - } + try { + await JobService.resumeSuspended({ + workspace: workspaceId ?? $workspaceStore ?? '', + jobId: job?.id ?? '', + requestBody: { + payload: approve ? (default_payload as any) : undefined, + approved: approve + } + }) + } catch (e: any) { + sendUserToast(e?.body ?? e?.message ?? 'Failed', true) + } finally { + loading = false } } let approvalStep = $derived((job?.flow_status?.step ?? 1) - 1) @@ -130,51 +84,41 @@
    {/if}
    - {#if isOwner || resumeUrl} -
    - {#if !hide_cancel} -
    -
    - {/if} +
    + {#if !hide_cancel}
    - +
    - - {#if job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema} -
    - -
    - - The payload is optional, it is passed to the following step through the `resume` - variable - - {/if} + {/if} +
    +
    - {:else} - You cannot resume the flow yourself without receiving the resume secret since you are not an - owner of {job.script_path} and the approval step did not contain the resume url at key `resume` - {/if} + + {#if job?.raw_flow?.modules?.[approvalStep]?.suspend?.resume_form?.schema} +
    + +
    + + The payload is optional, it is passed to the following step through the `resume` variable + + {/if} +
    diff --git a/frontend/src/lib/components/WorkflowTimeline.svelte b/frontend/src/lib/components/WorkflowTimeline.svelte index 3dea7279d6..2ea5ebba50 100644 --- a/frontend/src/lib/components/WorkflowTimeline.svelte +++ b/frontend/src/lib/components/WorkflowTimeline.svelte @@ -1,6 +1,6 @@ {#if flow_status} @@ -167,22 +216,74 @@ sleep ({(v as any).sleep_duration_s}s)
    {:else if isApproval} -
    -
    - - - {v.name ?? stepKey(k)} - - {#if !isDone} - - - waiting + {@const selfApprovalDisabled = (v as any).self_approval_disabled === true} + {@const formSchema = (v as any).form?.schema ?? (v as any).form} + {@const hasForm = + formSchema && typeof formSchema === 'object' && Object.keys(formSchema).length > 0} + {@const canApprove = !isDone && jobId} +
    +
    +
    + + + {v.name ?? stepKey(k)} - {:else} - {msToSec(v.duration_ms ?? 0)}s + {#if !isDone} + + + waiting + + {#if canApprove} +
    + + +
    + {/if} + {:else} + {msToSec(v.duration_ms ?? 0)}s + {/if} +
    + {#if canApprove && selfApprovalDisabled && $userStore?.is_admin} +
    + Self-approval is disabled but allowed because you are an admin/owner +
    + {/if} + {#if canApprove && hasForm} +
    + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} +
    {/if}
    {:else} @@ -275,13 +376,13 @@ {@const result = stepResults[stepKey(k)]} {#if isDone && result !== undefined}
    -
    Result
    +
    Result
    {:else} -
    Step completed (no result)
    +
    Step completed (no result)
    {/if} {:else if loadingJobs[k] && !childJobs[k]}
    @@ -293,7 +394,7 @@ {#if job.logs || isRunning}
    -
    Logs
    +
    Logs
    {#if isDone && job.result !== undefined}
    -
    Result
    +
    Result
    diff --git a/frontend/src/lib/components/flows/content/SuspendDrawer.svelte b/frontend/src/lib/components/flows/content/SuspendDrawer.svelte index ee22349cac..3ac2afe001 100644 --- a/frontend/src/lib/components/flows/content/SuspendDrawer.svelte +++ b/frontend/src/lib/components/flows/content/SuspendDrawer.svelte @@ -39,27 +39,9 @@ render a cancel button, providing the operator with an option to cancel the step. e.g: - {#snippet content()} - - -
    {/if} diff --git a/frontend/src/lib/components/scriptEditor/LogPanel.svelte b/frontend/src/lib/components/scriptEditor/LogPanel.svelte index 271e2a765c..82845c4a88 100644 --- a/frontend/src/lib/components/scriptEditor/LogPanel.svelte +++ b/frontend/src/lib/components/scriptEditor/LogPanel.svelte @@ -158,6 +158,7 @@ result={previewJob?.result} success={previewJob?.success !== false} autoExpandResult + jobId={previewJob?.id} />
    {:else} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index d05d78f6ef..6865637ba7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -1601,7 +1601,7 @@ export function canHaveApproval(language: SupportedLanguage | undefined): boolea return false } - return ['python3', 'bun', 'deno'].includes(language) + return ['python3', 'bun'].includes(language) } export function canHaveFailure(language: SupportedLanguage | undefined): boolean { diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 5bf8f21ac5..5f7e426fe9 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -797,6 +797,7 @@ stepResults={getStepResults(job.workflow_as_code_status)} result={job.result} success={(job as any).success !== false} + jobId={job.id} />
    diff --git a/frontend/src/routes/approve/[workspace]/[job]/+page.svelte b/frontend/src/routes/approve/[workspace]/[job]/+page.svelte new file mode 100644 index 0000000000..6c9638a948 --- /dev/null +++ b/frontend/src/routes/approve/[workspace]/[job]/+page.svelte @@ -0,0 +1,359 @@ + + + + + + {#if error} +
    + {#if error.includes('logged in') || error.includes('sign in') || error.includes('Not authorized')} +
    + +

    Not Authorized

    +
    +

    {error}

    + + {:else if error.includes('Permission denied') || error.includes('Self-approval')} +
    + +

    Permission denied

    +
    +

    {error}

    + {:else} +
    + +

    Error

    +
    +

    {error}

    + {/if} +
    + {:else if approvalInfo} +
    +
    +

    Approvers

    +
    + {#if approvalInfo.approvers?.length > 0} +
      + {#each approvalInfo.approvers as a} +
    • +

      + {a.approver} + Unique id of approval: {a.resume_id} +

      +
      • + {/each} +
    + {:else} +

    + No current approvers for this step (approval steps can require more than one approval) +

    + {/if} +
    +
    +
    + {#if job && job.raw_flow} + + {/if} +
    +
    + + {#if !completed} +

    + {isWac ? 'Workflow' : 'Flow'} arguments +

    + + {/if} + +
    + +
    + {#if completed} + + The flow is not running anymore. You cannot cancel or resume it. + + {/if} + + {#if approvalInfo.description != undefined} + + {/if} + + {#if hasForm && !completed} + {#if emptyString($enterpriseLicense)} + + {:else} + + {/if} + {/if} + + {#if !completed && approvalInfo.can_approve} +
    + {#if approvalInfo.hide_cancel !== true} + + {:else} +
    + {/if} + +
    + {:else if !completed && !approvalInfo.can_approve} + {#if approvalInfo.user_auth_required && !$userStore} + + {:else} +
    +

    You are not authorized to approve this flow.

    + {#if approvalInfo.approval_conditions?.self_approval_disabled && $userStore && $userStore.email === (job as any)?.email} +

    Self-approval is disabled for this step.

    + {/if} + {#if approvalInfo.approval_conditions?.user_groups_required?.length > 0} +

    Only members of the following groups can approve: {approvalInfo.approval_conditions.user_groups_required.join(', ')}

    + {/if} +
    + {/if} + {:else if completed} + + {/if} + + {#if !completed && isSelfApprovalBypass} +
    + + As an administrator, by resuming or cancelling this stage of the flow, you bypass the + self-approval interdiction. + +
    + {/if} +
    + +
    + + Open run details (require auth) + +
    + + {#if job && job.raw_flow && !completed} +

    Flow details

    +
    + +
    + {/if} + {:else} +

    Loading...

    + {/if} +     diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index cdd3342771..d952584e5a 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2463,7 +2463,7 @@ class WorkflowCtx: ) async def _wait_for_approval( - self, timeout: int = 1800, form: dict | None = None + self, timeout: int = 1800, form: dict | None = None, self_approval: bool = True ): key = self._alloc_key("approval") @@ -2479,6 +2479,7 @@ class WorkflowCtx: "key": key, "timeout": timeout, "form": form, + "self_approval_disabled": not self_approval, "steps": [], }) @@ -2762,6 +2763,7 @@ async def sleep(seconds: int): async def wait_for_approval( timeout: int = 1800, form: dict | None = None, + self_approval: bool = True, ) -> dict: """Suspend the workflow and wait for an external approval. @@ -2770,6 +2772,11 @@ async def wait_for_approval( Returns a dict with ``value`` (form data), ``approver``, and ``approved``. + Args: + timeout: Approval timeout in seconds (default 1800). + form: Optional form schema for the approval page. + self_approval: Whether the user who triggered the flow can approve it (default True). + Example:: urls = await step("urls", lambda: get_resume_urls()) @@ -2778,7 +2785,7 @@ async def wait_for_approval( """ ctx: WorkflowCtx | None = _workflow_ctx.get(None) if ctx is not None: - return await ctx._wait_for_approval(timeout=timeout, form=form) + return await ctx._wait_for_approval(timeout=timeout, form=form, self_approval=self_approval) raise RuntimeError("wait_for_approval can only be called inside a @workflow") diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 0170d7edca..d30a5be3eb 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -632,7 +632,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -1336,12 +1336,17 @@ async def sleep(seconds: int) # # Returns a dict with \`\`value\`\` (form data), \`\`approver\`\`, and \`\`approved\`\`. # +# Args: +# timeout: Approval timeout in seconds (default 1800). +# form: Optional form schema for the approval page. +# self_approval: Whether the user who triggered the flow can approve it (default True). +# # Example:: # # urls = await step("urls", lambda: get_resume_urls()) # await step("notify", lambda: send_email(urls["approvalPage"])) # result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index ec776de97e..674c9986b9 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1605,7 +1605,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. @@ -2309,12 +2309,17 @@ async def sleep(seconds: int) # # Returns a dict with ``value`` (form data), ``approver``, and ``approved``. # +# Args: +# timeout: Approval timeout in seconds (default 1800). +# form: Optional form schema for the approval page. +# self_approval: Whether the user who triggered the flow can approve it (default True). +# # Example:: # # urls = await step("urls", lambda: get_resume_urls()) # await step("notify", lambda: send_email(urls["approvalPage"])) # result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 7163e76a4a..241d438f58 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -648,12 +648,17 @@ async def sleep(seconds: int) # # Returns a dict with ``value`` (form data), ``approver``, and ``approved``. # +# Args: +# timeout: Approval timeout in seconds (default 1800). +# form: Optional form schema for the approval page. +# self_approval: Whether the user who triggered the flow can approve it (default True). +# # Example:: # # urls = await step("urls", lambda: get_resume_urls()) # await step("notify", lambda: send_email(urls["approvalPage"])) # result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict # Process items in parallel with optional concurrency control. # diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index f38ba274c1..8d96473313 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -481,7 +481,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index ba40a2d624..b4db20ae80 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -610,7 +610,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index cdd015863a..ecf7fe2103 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -608,7 +608,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index fddae85f6e..563d01ed48 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -614,7 +614,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. diff --git a/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md b/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md index 4687be55e4..1d52290283 100644 --- a/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-nativets/SKILL.md @@ -575,7 +575,7 @@ workflow(fn: (...args: any[]) => Promise): void * await step("notify", () => sendEmail(urls.approvalPage)); * const { value, approver } = await waitForApproval({ timeout: 3600 }); */ -waitForApproval(options?: { timeout?: number; form?: object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> +waitForApproval(options?: { timeout?: number; form?: object; selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> /** * Process items in parallel with optional concurrency control. diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index c860ee696c..e6aa3b848c 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -783,12 +783,17 @@ async def sleep(seconds: int) # # Returns a dict with ``value`` (form data), ``approver``, and ``approved``. # +# Args: +# timeout: Approval timeout in seconds (default 1800). +# form: Optional form schema for the approval page. +# self_approval: Whether the user who triggered the flow can approve it (default True). +# # Example:: # # urls = await step("urls", lambda: get_resume_urls()) # await step("notify", lambda: send_email(urls["approvalPage"])) # result = await wait_for_approval(timeout=3600) -async def wait_for_approval(timeout: int = 1800, form: dict | None = None) -> dict +async def wait_for_approval(timeout: int = 1800, form: dict | None = None, self_approval: bool = True) -> dict # Process items in parallel with optional concurrency control. # diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 49087a8b60..1aded4a720 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1577,6 +1577,7 @@ export class WorkflowCtx { _waitForApproval(options?: { timeout?: number; form?: object; + selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { const key = this._allocKey("approval"); @@ -1597,6 +1598,7 @@ export class WorkflowCtx { key, timeout: options?.timeout ?? 1800, form: options?.form, + self_approval_disabled: !(options?.selfApproval ?? true), steps: [], }); } @@ -1842,6 +1844,7 @@ export function workflow(fn: (...args: any[]) => Promise) { export function waitForApproval(options?: { timeout?: number; form?: object; + selfApproval?: boolean; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); if (!ctx) { From 6060ac3adc0afd94d62ec233f5d7282238d3ffc9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 21:40:26 +0000 Subject: [PATCH 28/48] chore(main): release 1.664.0 (#8498) * chore(main): release 1.664.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 15 ++ backend/Cargo.lock | 175 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 113 insertions(+), 107 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c8aa1a9648..e08a1cf181 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## [1.664.0](https://github.com/windmill-labs/windmill/compare/v1.663.0...v1.664.0) (2026-03-24) + + +### Features + +* add instance-level AI settings ([#8453](https://github.com/windmill-labs/windmill/issues/8453)) ([db5e036](https://github.com/windmill-labs/windmill/commit/db5e03610da325288d53afdbca94b9cbfc7ceace)) +* add selfApproval option to WAC + inline approval buttons ([#8440](https://github.com/windmill-labs/windmill/issues/8440)) ([d578e40](https://github.com/windmill-labs/windmill/commit/d578e40101a838d3dffda14157cf72ee4d5a93c0)) +* flow group nodes with collapsible groups ([#8075](https://github.com/windmill-labs/windmill/issues/8075)) ([81eb446](https://github.com/windmill-labs/windmill/commit/81eb446eee359f44374b81320690e5345fd08c15)) + + +### Bug Fixes + +* add GIT_SSL_CAINFO to tracing proxy env vars ([#8502](https://github.com/windmill-labs/windmill/issues/8502)) ([bdfd5d5](https://github.com/windmill-labs/windmill/commit/bdfd5d57261a4bb760fc57ad41ee56aff9b9c0af)) +* create parent dirs and accept 'python' alias in script bootstrap ([#8497](https://github.com/windmill-labs/windmill/issues/8497)) ([7f27d99](https://github.com/windmill-labs/windmill/commit/7f27d996accb3c3b471d1c50df397867d89c738a)) + ## [1.663.0](https://github.com/windmill-labs/windmill/compare/v1.662.0...v1.663.0) (2026-03-24) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 30138085c9..1871fda815 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7401,14 +7401,15 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d72a21f6a71a6c4c3160e095e8925861f5119dd26ef71acee1b9146f74f76c8" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" dependencies = [ "socket2 0.6.3", "widestring", + "windows-registry", + "windows-result 0.4.1", "windows-sys 0.61.2", - "winreg", ] [[package]] @@ -8063,9 +8064,9 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08" dependencies = [ "bitflags 2.9.4", "libc", @@ -15044,9 +15045,9 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-segmentation" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +checksum = "da36089a805484bcccfffe0739803392c8298778a2d2f09febf76fac5ad9025b" [[package]] name = "unicode-width" @@ -15746,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-nats", @@ -15822,7 +15823,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15835,7 +15836,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "argon2", @@ -15976,7 +15977,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15999,7 +16000,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16012,7 +16013,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16038,7 +16039,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.663.0" +version = "1.664.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16048,7 +16049,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16065,7 +16066,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16088,7 +16089,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16111,7 +16112,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16127,7 +16128,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16147,7 +16148,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16167,7 +16168,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16181,7 +16182,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-nats", @@ -16209,7 +16210,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16234,7 +16235,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16252,7 +16253,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16274,7 +16275,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16294,7 +16295,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16324,7 +16325,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16351,7 +16352,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.663.0" +version = "1.664.0" dependencies = [ "lazy_static", "serde", @@ -16363,7 +16364,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.663.0" +version = "1.664.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16386,7 +16387,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16400,7 +16401,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.663.0" +version = "1.664.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16431,7 +16432,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.663.0" +version = "1.664.0" dependencies = [ "chrono", "lazy_static", @@ -16445,7 +16446,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16464,7 +16465,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.663.0" +version = "1.664.0" dependencies = [ "aes-gcm", "anyhow", @@ -16564,7 +16565,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.663.0" +version = "1.664.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16583,7 +16584,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.663.0" +version = "1.664.0" dependencies = [ "regex", "serde", @@ -16598,7 +16599,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16622,7 +16623,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "futures", @@ -16639,7 +16640,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.663.0" +version = "1.664.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16655,7 +16656,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -16676,7 +16677,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -16707,7 +16708,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-oauth2", @@ -16731,7 +16732,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-stream", @@ -16765,7 +16766,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "futures", @@ -16783,7 +16784,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.663.0" +version = "1.664.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16792,7 +16793,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "lazy_static", @@ -16804,7 +16805,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "serde_json", @@ -16816,7 +16817,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "gosyn", @@ -16828,7 +16829,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "lazy_static", @@ -16840,7 +16841,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "serde_json", @@ -16852,7 +16853,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "nu-parser", @@ -16863,7 +16864,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16874,7 +16875,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16886,7 +16887,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16897,7 +16898,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-recursion", @@ -16919,7 +16920,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "lazy_static", @@ -16933,7 +16934,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16950,7 +16951,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "lazy_static", @@ -16963,7 +16964,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "serde", @@ -16975,7 +16976,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "lazy_static", @@ -16993,7 +16994,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17009,7 +17010,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17025,7 +17026,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "serde", @@ -17036,7 +17037,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-recursion", @@ -17073,7 +17074,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "const_format", @@ -17111,7 +17112,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.663.0" +version = "1.664.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17122,7 +17123,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-recursion", @@ -17151,7 +17152,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17174,7 +17175,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17207,7 +17208,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17227,7 +17228,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17261,7 +17262,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17296,7 +17297,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17319,7 +17320,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17343,7 +17344,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-nats", @@ -17367,7 +17368,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17402,7 +17403,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17430,7 +17431,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-trait", @@ -17453,7 +17454,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17472,7 +17473,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.663.0" +version = "1.664.0" dependencies = [ "anyhow", "async-once-cell", @@ -17580,7 +17581,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.663.0" +version = "1.664.0" dependencies = [ "bytes", "futures", @@ -18193,16 +18194,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" -[[package]] -name = "winreg" -version = "0.55.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" -dependencies = [ - "cfg-if", - "windows-sys 0.59.0", -] - [[package]] name = "winsafe" version = "0.0.19" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6dc821689c..3b44e3d87a 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.663.0" +version = "1.664.0" authors.workspace = true edition.workspace = true @@ -82,7 +82,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.663.0" +version = "1.664.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6576633319..993569e0f0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.663.0 + version: 1.664.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f93b61c6ee..e79e14588d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.663.0"; +export const VERSION = "v1.664.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 83e9c1bbc8..8901d5f979 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { workspaceAdd, }; -export const VERSION = "1.663.0"; +export const VERSION = "1.664.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 792e303b6e..3cd78bf5a6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.663.0", + "version": "1.664.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.663.0", + "version": "1.664.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 39bb3b6365..19eb2b69b5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.663.0", + "version": "1.664.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 130d95a820..13775e4a5d 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.663.0" +wmill = ">=1.664.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 9a0983b663..7480b43a1e 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.663.0 + version: 1.664.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index c84a6d49f7..6629cc8034 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.663.0' + ModuleVersion = '1.664.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 92fbaf56d1..8cfc22d3a6 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.663.0" +version = "1.664.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 5435328c88..9632ec19a2 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.663.0", + "version": "1.664.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index bd66c04c51..d4023c7e69 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.663.0", + "version": "1.664.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index eb4feec596..694b27ca91 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.663.0 +1.664.0 From 85c52e2cded10606cc895d0d3b717e13c69bc9b3 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 06:40:20 +0000 Subject: [PATCH 29/48] fix: use /apps_raw/get/ redirect URL for raw apps set as workspace default (#8508) * fix: use /apps_raw/get/ redirect URL for raw apps set as workspace default Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update sqlx cache for default_app query Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...c3592deb61d1111d1430ddd2879b72e6424ef.json | 28 +++++++++++++++++++ ...3f3be67b6160cd258c86b8e8f22a6d601afd0.json | 22 --------------- .../windmill-api-workspaces/src/workspaces.rs | 19 +++++++++---- backend/windmill-api/openapi.yaml | 2 ++ frontend/src/lib/components/Login.svelte | 3 +- .../(logged)/user/(user)/login/+page.svelte | 3 +- .../user/(user)/workspaces/+page.svelte | 3 +- 7 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 backend/.sqlx/query-1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef.json delete mode 100644 backend/.sqlx/query-ed1a053c7b22d9cb69767be40d33f3be67b6160cd258c86b8e8f22a6d601afd0.json diff --git a/backend/.sqlx/query-1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef.json b/backend/.sqlx/query-1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef.json new file mode 100644 index 0000000000..0930bdf1b8 --- /dev/null +++ b/backend/.sqlx/query-1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ws.default_app AS default_app_path, av.raw_app AS \"default_app_raw: Option\"\n FROM workspace_settings ws\n LEFT JOIN app ON app.path = ws.default_app AND app.workspace_id = ws.workspace_id\n LEFT JOIN app_version av ON av.id = app.versions[array_upper(app.versions, 1)]\n WHERE ws.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "default_app_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "default_app_raw: Option", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "1bc77ad29b9c68b1d339b85158bc3592deb61d1111d1430ddd2879b72e6424ef" +} diff --git a/backend/.sqlx/query-ed1a053c7b22d9cb69767be40d33f3be67b6160cd258c86b8e8f22a6d601afd0.json b/backend/.sqlx/query-ed1a053c7b22d9cb69767be40d33f3be67b6160cd258c86b8e8f22a6d601afd0.json deleted file mode 100644 index b7c642ef12..0000000000 --- a/backend/.sqlx/query-ed1a053c7b22d9cb69767be40d33f3be67b6160cd258c86b8e8f22a6d601afd0.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT default_app FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "default_app", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "ed1a053c7b22d9cb69767be40d33f3be67b6160cd258c86b8e8f22a6d601afd0" -} diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8cd8f27d9b..55ad777cbf 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2233,22 +2233,29 @@ async fn edit_default_app( #[derive(Serialize)] struct WorkspaceDefaultApp { pub default_app_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_app_raw: Option, } async fn get_default_app( Extension(db): Extension, Path(w_id): Path, ) -> JsonResult { - let mut tx = db.begin().await?; - let default_app_path = sqlx::query_scalar!( - "SELECT default_app FROM workspace_settings WHERE workspace_id = $1", + let row = sqlx::query!( + "SELECT ws.default_app AS default_app_path, av.raw_app AS \"default_app_raw: Option\" + FROM workspace_settings ws + LEFT JOIN app ON app.path = ws.default_app AND app.workspace_id = ws.workspace_id + LEFT JOIN app_version av ON av.id = app.versions[array_upper(app.versions, 1)] + WHERE ws.workspace_id = $1", &w_id ) - .fetch_one(&mut *tx) + .fetch_one(&db) .await .map_err(|err| Error::internal_err(format!("getting default_app: {err}")))?; - tx.commit().await?; - Ok(Json(WorkspaceDefaultApp { default_app_path })) + Ok(Json(WorkspaceDefaultApp { + default_app_path: row.default_app_path, + default_app_raw: row.default_app_raw, + })) } async fn edit_error_handler( diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 993569e0f0..bdd70229e9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3702,6 +3702,8 @@ paths: properties: default_app_path: type: string + default_app_raw: + type: boolean /w/{workspace}/workspaces/usage: get: diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 1e375cb53d..c1a2a7ebdb 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -164,7 +164,8 @@ workspace: $workspaceStore! }) if (!emptyString(defaultApp.default_app_path)) { - goto(`/apps/get/${defaultApp.default_app_path}`) + const prefix = defaultApp.default_app_raw ? '/apps_raw/get' : '/apps/get' + goto(`${prefix}/${defaultApp.default_app_path}`) } else { goto(rd ?? '/') } diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index 63843578c9..3e7ef57bf6 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -87,7 +87,8 @@ workspace: $workspaceStore! }) if (!emptyString(defaultApp.default_app_path)) { - goto(`/apps/get/${defaultApp.default_app_path}`) + const prefix = defaultApp.default_app_raw ? '/apps_raw/get' : '/apps/get' + goto(`${prefix}/${defaultApp.default_app_path}`) } else { goto(rd ?? '/') } diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte index 0ddfd92914..2cc2a65fbf 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/workspaces/+page.svelte @@ -151,7 +151,8 @@ workspace: $workspaceStore! }) if (!emptyString(defaultApp.default_app_path)) { - await goto(`/apps/get/${defaultApp.default_app_path}`) + const prefix = defaultApp.default_app_raw ? '/apps_raw/get' : '/apps/get' + await goto(`${prefix}/${defaultApp.default_app_path}`) } else { if (rd?.startsWith('http')) { window.location.href = rd From 1341a1321da3ab7c5ce24df27fe6b028887d6a0b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 07:11:57 +0000 Subject: [PATCH 30/48] chore: update tantivy from 0.24 to 0.26 (#8510) * [ee] chore: update tantivy from 0.24 to 0.26 - Rebase windmill-labs/tantivy fork onto upstream 0.26 - Bump serde pin from 1.0.219 to 1.0.220 (required by tantivy 0.26's time dependency) Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to ec613f2db9e72e32e9131181546dcd679405a782 This commit updates the EE repository reference after PR #479 was merged in windmill-ee-private. Previous ee-repo-ref: 920cf601b0651b7ba94493668ea051e00f3e74bf New ee-repo-ref: ec613f2db9e72e32e9131181546dcd679405a782 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 149 +++++++++++++++++++++++----------------- backend/Cargo.toml | 4 +- backend/ee-repo-ref.txt | 2 +- 3 files changed, 89 insertions(+), 66 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1871fda815..fcb56bb81c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -405,7 +405,7 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", - "lz4_flex", + "lz4_flex 0.11.6", ] [[package]] @@ -2124,7 +2124,7 @@ dependencies = [ "num-traits", "num_cpus", "rand 0.9.0", - "rand_distr 0.5.1", + "rand_distr", "rayon", "safetensors", "thiserror 2.0.18", @@ -3640,6 +3640,12 @@ dependencies = [ "sqlparser 0.55.0", ] +[[package]] +name = "datasketches" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745" + [[package]] name = "debug-helper" version = "0.3.13" @@ -5678,7 +5684,7 @@ dependencies = [ "half", "num-traits", "rand 0.9.0", - "rand_distr 0.5.1", + "rand_distr", ] [[package]] @@ -6490,7 +6496,7 @@ dependencies = [ "crunchy", "num-traits", "rand 0.9.0", - "rand_distr 0.5.1", + "rand_distr", "zerocopy", ] @@ -7109,15 +7115,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyperloglogplus" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" -dependencies = [ - "serde", -] - [[package]] name = "iana-time-zone" version = "0.1.65" @@ -8197,6 +8194,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.0", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -8221,6 +8227,12 @@ dependencies = [ "twox-hash 2.1.2", ] +[[package]] +name = "lz4_flex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" + [[package]] name = "lzma-sys" version = "0.1.20" @@ -9186,9 +9198,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" [[package]] name = "num-format" @@ -9713,6 +9725,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d" +dependencies = [ + "num-traits", +] + [[package]] name = "os_pipe" version = "1.1.5" @@ -9738,7 +9759,7 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "ownedbytes" version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "stable_deref_trait", ] @@ -9850,7 +9871,7 @@ dependencies = [ "futures", "half", "hashbrown 0.15.5", - "lz4_flex", + "lz4_flex 0.11.6", "num", "num-bigint", "object_store", @@ -10842,16 +10863,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand 0.8.5", -] - [[package]] name = "rand_distr" version = "0.5.1" @@ -12191,10 +12202,11 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.220" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "ceecad4c782e936ac90ecfd6b56532322e3262b14320abf30ce89a92ffdbfe22" dependencies = [ + "serde_core", "serde_derive", ] @@ -12216,7 +12228,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" dependencies = [ - "ordered-float", + "ordered-float 2.10.1", "serde", ] @@ -12241,10 +12253,19 @@ dependencies = [ ] [[package]] -name = "serde_derive" -version = "1.0.219" +name = "serde_core" +version = "1.0.220" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "ddba47394f3b862d6ff6efdbd26ca4673e3566a307880a0ffb98f274bbe0ec32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.220" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e1f3b1761e96def5ec6d04a6e7421c0404fa3cf5c0155f1e2848fae3d8cc08" dependencies = [ "proc-macro2", "quote", @@ -12608,9 +12629,9 @@ checksum = "1b6709c7b6754dca1311b3c73e79fcce40dd414c782c66d88e8823030093b02b" [[package]] name = "sketches-ddsketch" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" +checksum = "05e40b6cf54d988dc1a2223531b969c9a9e30906ad90ef64890c27b4bfbb46ea" dependencies = [ "serde", ] @@ -13705,8 +13726,8 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" [[package]] name = "tantivy" -version = "0.24.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.26.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "aho-corasick", "arc-swap", @@ -13717,17 +13738,17 @@ dependencies = [ "census", "crc32fast", "crossbeam-channel", + "datasketches", "downcast-rs", "fastdivide", "fnv", "fs4", "htmlescape", - "hyperloglogplus", "itertools 0.14.0", "levenshtein_automata", "log", - "lru 0.12.5", - "lz4_flex", + "lru 0.16.3", + "lz4_flex 0.13.0", "measure_time", "memmap2 0.9.10", "once_cell", @@ -13750,22 +13771,23 @@ dependencies = [ "tempfile", "thiserror 2.0.18", "time", + "typetag", "uuid", "winapi", ] [[package]] name = "tantivy-bitpacker" -version = "0.8.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.9.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "bitpacking", ] [[package]] name = "tantivy-columnar" -version = "0.5.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.6.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "downcast-rs", "fastdivide", @@ -13779,8 +13801,8 @@ dependencies = [ [[package]] name = "tantivy-common" -version = "0.9.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.10.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "async-trait", "byteorder", @@ -13802,18 +13824,20 @@ dependencies = [ [[package]] name = "tantivy-query-grammar" -version = "0.24.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.25.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ + "fnv", "nom 7.1.3", + "ordered-float 5.1.0", "serde", "serde_json", ] [[package]] name = "tantivy-sstable" -version = "0.5.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.6.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "futures-util", "itertools 0.14.0", @@ -13825,18 +13849,17 @@ dependencies = [ [[package]] name = "tantivy-stacker" -version = "0.5.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.6.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "murmurhash32", - "rand_distr 0.4.3", "tantivy-common", ] [[package]] name = "tantivy-tokenizer-api" -version = "0.5.0" -source = "git+https://github.com/windmill-labs/tantivy?rev=6a24621231202ccd77bec90d8787e2281fb94e4e#6a24621231202ccd77bec90d8787e2281fb94e4e" +version = "0.6.0" +source = "git+https://github.com/windmill-labs/tantivy?rev=6ae7c70bc603b8e69e27f3240e08bd00a93fb12c#6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" dependencies = [ "serde", ] @@ -13966,7 +13989,7 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float", + "ordered-float 2.10.1", ] [[package]] @@ -14043,30 +14066,30 @@ dependencies = [ [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.24" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 3b44e3d87a..927a6f6ba3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -373,7 +373,7 @@ tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } tower-cookies = "^0.10" #stuck because of swc for now -serde = "=1.0.219" +serde = "=1.0.220" serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } serde_yml = "0.0.12" uuid = { version = "^1", features = ["serde", "v4", "js"] } @@ -587,7 +587,7 @@ tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" pin-project-lite = "^0" -tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6a24621231202ccd77bec90d8787e2281fb94e4e" } +tantivy = { git="https://github.com/windmill-labs/tantivy", rev="6ae7c70bc603b8e69e27f3240e08bd00a93fb12c" } backon = "1.3.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7ef2ef46db..c5ca6a15cf 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -faeaa43bbe2ba4804f80b828b85fd4d6daef096c +ec613f2db9e72e32e9131181546dcd679405a782 From fe223bffa32c17815988ff4210d89f4f01d486e2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 07:34:24 +0000 Subject: [PATCH 31/48] chore: update samael from 0.0.14 to 0.0.20 (#8512) Co-authored-by: Claude Opus 4.6 (1M context) --- backend/Cargo.lock | 105 +++++++++++++++++++-------------------------- backend/Cargo.toml | 2 +- 2 files changed, 46 insertions(+), 61 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index fcb56bb81c..af95c7a85e 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1594,29 +1594,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.9.4", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.117", - "which 4.4.2", -] - [[package]] name = "bindgen" version = "0.70.1" @@ -1667,6 +1644,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease", "proc-macro2", "quote", "regex", @@ -4862,7 +4841,16 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" dependencies = [ - "derive_builder_macro", + "derive_builder_macro 0.12.0", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro 0.20.2", ] [[package]] @@ -4877,16 +4865,38 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_builder_macro" version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" dependencies = [ - "derive_builder_core", + "derive_builder_core 0.12.0", "syn 1.0.109", ] +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core 0.20.2", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "1.0.0" @@ -7476,15 +7486,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -7900,12 +7901,6 @@ dependencies = [ "spin 0.9.8", ] -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "leb128fmt" version = "0.1.0" @@ -9349,7 +9344,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml 0.37.5", + "quick-xml", "rand 0.9.0", "reqwest 0.12.28", "ring 0.17.14", @@ -10646,16 +10641,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "quick-xml" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick-xml" version = "0.37.5" @@ -11950,15 +11935,15 @@ dependencies = [ [[package]] name = "samael" -version = "0.0.14" +version = "0.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b75583aad4a51c50fc0af69c230d18078c9d5a69a98d0f6013d01053acf744f4" +checksum = "8b010d88b2c7b2c3fc9e49f6fffa086d4c350ec50538a8082f88e446ea16c670" dependencies = [ - "base64 0.21.7", - "bindgen 0.69.5", + "base64 0.22.1", + "bindgen 0.72.1", "chrono", "data-encoding", - "derive_builder", + "derive_builder 0.20.2", "flate2", "lazy_static", "libc", @@ -11967,10 +11952,10 @@ dependencies = [ "openssl-probe 0.1.6", "openssl-sys", "pkg-config", - "quick-xml 0.30.0", - "rand 0.8.5", + "quick-xml", + "rand 0.9.0", "serde", - "thiserror 1.0.69", + "thiserror 2.0.18", "url", "uuid", ] @@ -14152,7 +14137,7 @@ checksum = "d9be88c795d8b9f9c4002b3a8f26a6d0876103a6f523b32ea3bac52d8560c17c" dependencies = [ "aho-corasick", "clap", - "derive_builder", + "derive_builder 0.12.0", "esaxx-rs", "getrandom 0.2.17", "indicatif", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 927a6f6ba3..e98d4b793e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -510,7 +510,7 @@ native-tls = ">=0.2, <0.2.17" # samael will break compilation on MacOS. Use this fork instead to make it work # samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] } libxml = { version = "=0.3.3" } -samael = { version="0.0.14", features = ["xmlsec"] } +samael = { version="0.0.20", features = ["xmlsec"] } gcp_auth = "0.9.0" rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} jsonwebtoken = "8.3.0" From 0db21aa6b7c5b557ad53a1a74493c4fd80a53b49 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 07:44:49 +0000 Subject: [PATCH 32/48] samael bump --- backend/Cargo.lock | 3 +-- backend/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index af95c7a85e..a5e2e8fc98 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -11936,8 +11936,7 @@ dependencies = [ [[package]] name = "samael" version = "0.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b010d88b2c7b2c3fc9e49f6fffa086d4c350ec50538a8082f88e446ea16c670" +source = "git+https://github.com/njaremko/samael?rev=f879f1942ec1b34b6d3027ce7e4724ad95d15dfa#f879f1942ec1b34b6d3027ce7e4724ad95d15dfa" dependencies = [ "base64 0.22.1", "bindgen 0.72.1", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e98d4b793e..e1d6dbe9d2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -510,7 +510,7 @@ native-tls = ">=0.2, <0.2.17" # samael will break compilation on MacOS. Use this fork instead to make it work # samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] } libxml = { version = "=0.3.3" } -samael = { version="0.0.20", features = ["xmlsec"] } +samael = { git="https://github.com/njaremko/samael", rev="f879f1942ec1b34b6d3027ce7e4724ad95d15dfa", features = ["xmlsec"] } gcp_auth = "0.9.0" rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} jsonwebtoken = "8.3.0" From e3620e074e1bdb46b2b8d732f35a91d300589663 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 07:56:45 +0000 Subject: [PATCH 33/48] fix: serve index disk storage sizes from /srch/ endpoint (#8511) * [ee] fix: serve index disk storage sizes from /srch/ endpoint On multi-container deployments, the API server doesn't have the index files on its local disk, so disk size was always reported as 0.0B. Added a new GET /srch/index/storage/disk endpoint that calculates disk sizes on the indexer process (which owns the files). The frontend now fetches disk sizes from this endpoint in parallel with the status call. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to 71aab648925f31cde37efd31d79a7f3a977fd42a This commit updates the EE repository reference after PR #480 was merged in windmill-ee-private. Previous ee-repo-ref: b3e0000e2528809302c18f36930aebf3d004747a New ee-repo-ref: 71aab648925f31cde37efd31d79a7f3a977fd42a Automated by sync-ee-ref workflow. * chore: update ee-repo-ref to indexer-disk-storage-zero branch Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update sqlx metadata and ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...3e69e4ef8821c6cbf3b4f296b3853d95692af.json | 22 +++++++++ ...ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json | 28 ----------- ...08cb1ca21fbdba3373af54fadf1f4af324073.json | 35 -------------- ...59f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json | 46 ------------------- ...6e8a4f8f3a9bf04238b33e9caf46836df73d9.json | 35 -------------- ...91688f3ed0efd3a43e81f4ea296255248092c.json | 16 ------- ...32e97ebefb46be9e58bd3da9067748075311b.json | 35 -------------- ...8903bec93ef79a71053c00227e17c6f0415a2.json | 23 ---------- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 21 +++++++++ .../IndexerMemorySettings.svelte | 29 ++++++++---- 11 files changed, 64 insertions(+), 228 deletions(-) create mode 100644 backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json delete mode 100644 backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json delete mode 100644 backend/.sqlx/query-79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073.json delete mode 100644 backend/.sqlx/query-950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json delete mode 100644 backend/.sqlx/query-98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9.json delete mode 100644 backend/.sqlx/query-b1979a8249557d29e9055fde06191688f3ed0efd3a43e81f4ea296255248092c.json delete mode 100644 backend/.sqlx/query-c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b.json delete mode 100644 backend/.sqlx/query-ecf67b08d327c351909b7ba80218903bec93ef79a71053c00227e17c6f0415a2.json diff --git a/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json b/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json new file mode 100644 index 0000000000..a78e67067f --- /dev/null +++ b/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af" +} diff --git a/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json b/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json deleted file mode 100644 index 0a2976f868..0000000000 --- a/backend/.sqlx/query-32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT parent_job, flow_step_id FROM v2_job WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "flow_step_id", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "32ca7941db013dacd2479962fa9ed5c8c64daec45ba820a6c8f7d7ab76cc40c9" -} diff --git a/backend/.sqlx/query-79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073.json b/backend/.sqlx/query-79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073.json deleted file mode 100644 index f990932367..0000000000 --- a/backend/.sqlx/query-79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "debounced_times", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "first_started_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "job_id_to_debounce", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "79b82ae996fba2e2ab53fcf84c108cb1ca21fbdba3373af54fadf1f4af324073" -} diff --git a/backend/.sqlx/query-950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json b/backend/.sqlx/query-950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json deleted file mode 100644 index 9ec97dbc82..0000000000 --- a/backend/.sqlx/query-950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH RECURSIVE chain AS (\n SELECT\n j.id,\n j.parent_job,\n j.flow_step_id,\n 1 AS depth\n FROM v2_job j\n WHERE j.id = $1\n UNION ALL\n SELECT\n pj.id,\n pj.parent_job,\n pj.flow_step_id,\n c.depth + 1\n FROM chain c\n JOIN v2_job pj ON pj.id = c.parent_job\n WHERE c.parent_job IS NOT NULL\n )\n SELECT\n c.id,\n c.parent_job,\n c.flow_step_id,\n EXISTS(SELECT 1 FROM v2_job_queue q WHERE q.id = c.parent_job) AS \"parent_in_queue!\",\n EXISTS(\n SELECT 1 FROM v2_job sib\n WHERE sib.parent_job = c.parent_job\n AND sib.id != c.id\n AND sib.id IN (SELECT sq.id FROM v2_job_queue sq)\n ) AS \"has_other_active_siblings!\"\n FROM chain c\n WHERE c.depth >= 1\n ORDER BY c.depth ASC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 2, - "name": "flow_step_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parent_in_queue!", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "has_other_active_siblings!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null, - null, - null, - null, - null - ] - }, - "hash": "950f364c9fa3c680eea895558a559f29220c08e94e1822e3bcb5c6ed6aa7d2bb" -} diff --git a/backend/.sqlx/query-98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9.json b/backend/.sqlx/query-98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9.json deleted file mode 100644 index 7dd6e9ac5d..0000000000 --- a/backend/.sqlx/query-98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "debounced_times", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "first_started_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "job_id_to_debounce", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "98033aae3182bde22d5b2ff08ef6e8a4f8f3a9bf04238b33e9caf46836df73d9" -} diff --git a/backend/.sqlx/query-b1979a8249557d29e9055fde06191688f3ed0efd3a43e81f4ea296255248092c.json b/backend/.sqlx/query-b1979a8249557d29e9055fde06191688f3ed0efd3a43e81f4ea296255248092c.json deleted file mode 100644 index 5fb12bed16..0000000000 --- a/backend/.sqlx/query-b1979a8249557d29e9055fde06191688f3ed0efd3a43e81f4ea296255248092c.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET flow_status = (\n SELECT jsonb_set(\n flow_status,\n ARRAY['modules', (idx - 1)::text],\n $2::jsonb\n )\n FROM jsonb_array_elements(flow_status->'modules')\n WITH ORDINALITY arr(elem, idx)\n WHERE elem->>'id' = $3\n LIMIT 1\n ) WHERE id = $1 AND (\n SELECT COUNT(*) FROM jsonb_array_elements(flow_status->'modules')\n WITH ORDINALITY arr(elem, idx)\n WHERE elem->>'id' = $3\n ) > 0", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Jsonb", - "Text" - ] - }, - "nullable": [] - }, - "hash": "b1979a8249557d29e9055fde06191688f3ed0efd3a43e81f4ea296255248092c" -} diff --git a/backend/.sqlx/query-c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b.json b/backend/.sqlx/query-c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b.json deleted file mode 100644 index 7d7842d7f4..0000000000 --- a/backend/.sqlx/query-c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH dk AS (\n INSERT INTO debounce_key (job_id, key)\n VALUES ($1, $2)\n ON CONFLICT (key)\n DO UPDATE SET\n previous_job_id = debounce_key.job_id,\n job_id = EXCLUDED.job_id,\n debounced_times = debounce_key.debounced_times + 1\n RETURNING\n debounced_times,\n first_started_at,\n previous_job_id AS job_id_to_debounce\n ), _batch AS (\n INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT\n $1,\n COALESCE(\n (SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = dk.job_id_to_debounce LIMIT 1),\n nextval('debounce_batch_seq')\n )\n FROM dk\n )\n SELECT debounced_times, first_started_at, job_id_to_debounce FROM dk\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "debounced_times", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "first_started_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 2, - "name": "job_id_to_debounce", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "c2347460b73ae9d3167031c263032e97ebefb46be9e58bd3da9067748075311b" -} diff --git a/backend/.sqlx/query-ecf67b08d327c351909b7ba80218903bec93ef79a71053c00227e17c6f0415a2.json b/backend/.sqlx/query-ecf67b08d327c351909b7ba80218903bec93ef79a71053c00227e17c6f0415a2.json deleted file mode 100644 index c43a2bcd30..0000000000 --- a/backend/.sqlx/query-ecf67b08d327c351909b7ba80218903bec93ef79a71053c00227e17c6f0415a2.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(\n SELECT 1 FROM v2_job\n WHERE parent_job = $1 AND id != $2\n AND id IN (SELECT id FROM v2_job_queue)\n ) as has", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "has", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ecf67b08d327c351909b7ba80218903bec93ef79a71053c00227e17c6f0415a2" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c5ca6a15cf..40b16c0b62 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ec613f2db9e72e32e9131181546dcd679405a782 +414202845a45e2a7c6a2d3e154bd8dfd0273cc14 \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bdd70229e9..5f350f2ce5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -17429,6 +17429,27 @@ paths: description: count of log lines that matched the query per hostname type: object + /srch/index/storage/disk: + get: + summary: Get index disk storage sizes from the indexer. + operationId: getIndexDiskStorageSizes + tags: + - indexSearch + responses: + "200": + description: disk storage sizes for each index + content: + application/json: + schema: + type: object + properties: + job_index_disk_size_bytes: + type: integer + nullable: true + log_index_disk_size_bytes: + type: integer + nullable: true + /indexer/delete/{idx_name}: delete: summary: Clear an index and restart the indexer. diff --git a/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte b/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte index 52bc3cd9bc..eb37ac8a0c 100644 --- a/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte +++ b/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte @@ -2,7 +2,7 @@ import { Button } from '$lib/components/common' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { IndexSearchService } from '$lib/gen' - import type { GetIndexerStatusResponse } from '$lib/gen' + import type { GetIndexerStatusResponse, GetIndexDiskStorageSizesResponse } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { displaySize } from '$lib/utils' import Tooltip from '../Tooltip.svelte' @@ -24,6 +24,7 @@ let clearServiceLogsIndexModalOpen = $state(false) let status: GetIndexerStatusResponse | undefined = $state(undefined) + let diskSizes: GetIndexDiskStorageSizesResponse | undefined = $state(undefined) let statusLoading = $state(true) let statusError = $state(false) @@ -41,9 +42,15 @@ statusLoading = true statusError = false try { - status = await IndexSearchService.getIndexerStatus() + const [statusRes, diskRes] = await Promise.all([ + IndexSearchService.getIndexerStatus(), + IndexSearchService.getIndexDiskStorageSizes().catch(() => undefined) + ]) + status = statusRes + diskSizes = diskRes } catch (e) { status = undefined + diskSizes = undefined statusError = true } finally { statusLoading = false @@ -139,7 +146,11 @@ : 'bg-red-500'}" > {label}: - + {entry?.is_alive ? 'Running' : 'Stopped'} {#if entry?.last_locked_at} @@ -161,21 +172,21 @@
    Jobs index: - {#if status.job_indexer?.storage?.disk_size_bytes != null} - Disk: {displaySize(status.job_indexer.storage.disk_size_bytes) ?? 'N/A'} + {#if diskSizes?.job_index_disk_size_bytes != null} + Disk: {displaySize(diskSizes.job_index_disk_size_bytes) ?? 'N/A'} {/if} {#if status.job_indexer?.storage?.s3_size_bytes != null} - {#if status.job_indexer?.storage?.disk_size_bytes != null}·{/if} + {#if diskSizes?.job_index_disk_size_bytes != null}·{/if} S3: {displaySize(status.job_indexer.storage.s3_size_bytes) ?? 'N/A'} {/if} Service logs index: - {#if status.log_indexer?.storage?.disk_size_bytes != null} - Disk: {displaySize(status.log_indexer.storage.disk_size_bytes) ?? 'N/A'} + {#if diskSizes?.log_index_disk_size_bytes != null} + Disk: {displaySize(diskSizes.log_index_disk_size_bytes) ?? 'N/A'} {/if} {#if status.log_indexer?.storage?.s3_size_bytes != null} - {#if status.log_indexer?.storage?.disk_size_bytes != null}·{/if} + {#if diskSizes?.log_index_disk_size_bytes != null}·{/if} S3: {displaySize(status.log_indexer.storage.s3_size_bytes) ?? 'N/A'} {/if} From 79d2bd51a00654162754046308d7670242120df6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 08:41:29 +0000 Subject: [PATCH 34/48] feat: move basic git sync from EE to CE with runtime user count gating (#8493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: move basic git sync from EE to CE with runtime user count gating Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt for git sync CE migration Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: keep git sync impl in private repo, revert oss to stub Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt after merge Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use LICENSE_KEY check instead of get_license_plan for runtime gating Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * fix: improve git sync CE UX — use "Community Edition" wording, mention user limit Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use "workspace members" instead of "users" in git sync messaging Co-Authored-By: Claude Opus 4.6 (1M context) * fix: lower CE git sync limit from 3 to 2 workspace members Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * fix: simplify git sync CE alerts to warn about EE feature with member limit Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add EE feature restrictions detail to CE git sync warning Co-Authored-By: Claude Opus 4.6 (1M context) * fix: show git sync settings even when >2 members, with disabled warning Co-Authored-By: Claude Opus 4.6 (1M context) * fix: show error alert when git sync settings exist but members exceed CE limit Co-Authored-By: Claude Opus 4.6 (1M context) * fix: mention CE git sync limit is for testing and hobbyist use Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to 79eeacccc0438010d7dfa60207a5cbdaf2eda08d This commit updates the EE repository reference after PR #476 was merged in windmill-ee-private. Previous ee-repo-ref: c4d69c6e700c16d44f909d9c7b6738b07043db98 New ee-repo-ref: 79eeacccc0438010d7dfa60207a5cbdaf2eda08d Automated by sync-ee-ref workflow. * chore: update sqlx cache Co-Authored-By: Claude Opus 4.6 (1M context) * chore: regenerate full sqlx cache after main merge Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update sqlx cache Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref and regenerate sqlx cache with private feature Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use LICENSE_KEY_VALID for EE check, allow delete without access check, extract helpers Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: use compile-time cfg(enterprise) gating instead of runtime license checks Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to 6171a91da38d6d16a88aeb1a3a4f4df78f995383 This commit updates the EE repository reference after PR #481 was merged in windmill-ee-private. Previous ee-repo-ref: 52681940cda6d70f65aeeb7144288f060b4d736e New ee-repo-ref: 6171a91da38d6d16a88aeb1a3a4f4df78f995383 Automated by sync-ee-ref workflow. * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to b5c8af4df9ba2c39fdd494d7a40f9a92fbff8abc This commit updates the EE repository reference after PR #482 was merged in windmill-ee-private. Previous ee-repo-ref: 6e5b2741831468a7b30b26c0df1241e6141c6833 New ee-repo-ref: b5c8af4df9ba2c39fdd494d7a40f9a92fbff8abc Automated by sync-ee-ref workflow. * fix: gate CE_GIT_SYNC_MAX_USERS behind cfg(not(enterprise)) Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...aacf6af2c284ae446860113c82bc4e1da08ab.json | 12 - ...bc47caebc25215a430d6b301b35e265888159.json | 12 - ...fb7cf5f2b76f013c274245af13d7d727ebf1f.json | 12 - ...e2e60e3183fa81a411622891caea6dc03fa90.json | 15 -- ...960ffc33da5f31bf780e8fd6a66d5150b8027.json | 12 - ...69c87a9d29370ec985d2c8c28633cd078ffaf.json | 12 - ...74da8c73120b3e16194904575f79a4e055002.json | 12 - ...437ab3e02d8c3c10c53decc664533b8d04bc0.json | 22 -- backend/ee-repo-ref.txt | 2 +- .../windmill-api-workspaces/src/workspaces.rs | 131 +++++++--- backend/windmill-api/openapi.yaml | 31 +++ backend/windmill-common/src/ee_oss.rs | 1 + backend/windmill-git-sync/Cargo.toml | 2 +- backend/windmill-git-sync/src/lib.rs | 20 +- cli/package-lock.json | 14 +- .../git_sync/GitSyncContext.svelte.ts | 11 + .../components/git_sync/GitSyncSection.svelte | 245 +++++++++++------- 17 files changed, 298 insertions(+), 268 deletions(-) delete mode 100644 backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json delete mode 100644 backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json delete mode 100644 backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json delete mode 100644 backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json delete mode 100644 backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json delete mode 100644 backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json delete mode 100644 backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json delete mode 100644 backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json diff --git a/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json b/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json deleted file mode 100644 index 0ad1fe4367..0000000000 --- a/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_version (id, app_id, value, created_by, created_at)\n VALUES (3001, 3001, '{\"grid\": []}', 'admin', NOW())", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab" -} diff --git a/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json b/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json deleted file mode 100644 index 24d3c8929a..0000000000 --- a/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE app SET versions = ARRAY[3001::bigint] WHERE id = 3001", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159" -} diff --git a/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json b/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json deleted file mode 100644 index 10cab9117a..0000000000 --- a/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema, extra_perms)\n VALUES ('test-workspace', 'u/operator/existing_flow', 'Existing flow', '', '{\"modules\": []}', 'admin', NOW(), '{}', '{}')", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f" -} diff --git a/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json b/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json deleted file mode 100644 index 27d46b27ed..0000000000 --- a/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO token (token_hash, token_prefix, email, label, super_admin, owner, workspace_id)\n VALUES ($1, $2, 'charlie@windmill.dev', 'Charlie new token', false, 'u/charlie', 'test-workspace')", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90" -} diff --git a/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json b/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json deleted file mode 100644 index 8e558fe67b..0000000000 --- a/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app (id, workspace_id, path, summary, versions, policy, extra_perms)\n VALUES (3001, 'test-workspace', 'u/operator/existing_app', 'Existing app', '{}',\n '{\"on_behalf_of\": \"u/admin\", \"on_behalf_of_email\": \"admin@windmill.dev\", \"execution_mode\": \"viewer\"}', '{}')", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027" -} diff --git a/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json b/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json deleted file mode 100644 index 6da123cbc4..0000000000 --- a/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usr_to_group (workspace_id, group_, usr) VALUES ('test-workspace', 'editors', 'charlie')", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf" -} diff --git a/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json b/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json deleted file mode 100644 index d7cc49fe3f..0000000000 --- a/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, extra_perms)\n VALUES ('test-workspace', 3001, 'u/operator/existing_script', 'export function main() { return \"original\"; }', 'deno', 'script', 'admin', '{}', 'Existing script', '', '', '{}')", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002" -} diff --git a/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json b/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json deleted file mode 100644 index d9b7688eba..0000000000 --- a/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id,\n script_path, is_flow, workspace_id, edited_by, permissioned_as\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 40b16c0b62..7d86a6114e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -414202845a45e2a7c6a2d3e154bd8dfd0273cc14 \ No newline at end of file +b5c8af4df9ba2c39fdd494d7a40f9a92fbff8abc diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 55ad777cbf..fb234a13a3 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -35,7 +35,6 @@ use windmill_common::variables::{ build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, }; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; -#[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; @@ -115,6 +114,7 @@ pub fn workspaced_service() -> Router { .route("/list_datatables", get(list_datatables)) .route("/list_datatable_schemas", get(list_datatable_schemas)) .route("/edit_datatable_config", post(edit_datatable_config)) + .route("/git_sync_enabled", get(get_git_sync_enabled)) .route("/edit_git_sync_config", post(edit_git_sync_config)) .route("/edit_git_sync_repository", post(edit_git_sync_repository)) .route( @@ -1595,24 +1595,20 @@ async fn edit_datatable_config( #[derive(Deserialize)] pub struct EditGitSyncConfig { - #[cfg(feature = "enterprise")] pub git_sync_settings: Option, } -#[cfg(feature = "enterprise")] #[derive(Deserialize, Debug)] pub struct EditGitSyncRepository { pub git_repo_resource_path: String, pub repository: GitRepositorySettings, } -#[cfg(feature = "enterprise")] #[derive(Deserialize, Debug)] pub struct DeleteGitSyncRepositoryRequest { pub git_repo_resource_path: String, } -#[cfg(feature = "enterprise")] fn validate_git_repo_resource_path(path: &str) -> Result<()> { // Resource paths should follow the pattern: $res:f// or $res:u// if path.is_empty() { @@ -1661,7 +1657,6 @@ fn validate_git_repo_resource_path(path: &str) -> Result<()> { Ok(()) } -#[cfg(feature = "enterprise")] fn cleanup_legacy_git_sync_settings_in_memory( git_sync_settings: &mut windmill_common::workspaces::WorkspaceGitSyncSettings, workspace_id: &str, @@ -1688,18 +1683,72 @@ fn cleanup_legacy_git_sync_settings_in_memory( } #[cfg(not(feature = "enterprise"))] -async fn edit_git_sync_config( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_new_config): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); +const CE_GIT_SYNC_MAX_USERS: i64 = 2; + +#[cfg(feature = "enterprise")] +async fn check_git_sync_access(_db: &DB, _w_id: &str) -> Result<()> { + Ok(()) +} + +#[cfg(not(feature = "enterprise"))] +async fn check_git_sync_access(db: &DB, w_id: &str) -> Result<()> { + let user_count: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false", + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + if user_count > CE_GIT_SYNC_MAX_USERS { + return Err(Error::BadRequest(format!( + "Git sync is available for workspaces with up to {} members. \ + Upgrade to Windmill Enterprise Edition for unlimited workspace members.", + CE_GIT_SYNC_MAX_USERS + ))); + } + Ok(()) } #[cfg(feature = "enterprise")] +async fn get_git_sync_enabled( + _authed: ApiAuthed, + Extension(_db): Extension, + Path(_w_id): Path, +) -> JsonResult { + Ok(Json(serde_json::json!({ + "enabled": true, + "reason": "enterprise", + "max_repos": null, + "user_count": null, + "max_users": null, + }))) +} + +#[cfg(not(feature = "enterprise"))] +async fn get_git_sync_enabled( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult { + let user_count: i64 = sqlx::query_scalar!( + "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false", + &w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(0); + + let enabled = user_count <= CE_GIT_SYNC_MAX_USERS; + Ok(Json(serde_json::json!({ + "enabled": enabled, + "reason": if enabled { Some("free_tier") } else { None::<&str> }, + "max_repos": if enabled { Some(1) } else { None:: }, + "user_count": user_count, + "max_users": CE_GIT_SYNC_MAX_USERS, + }))) +} + async fn edit_git_sync_config( authed: ApiAuthed, Extension(db): Extension, @@ -1708,6 +1757,7 @@ async fn edit_git_sync_config( Json(new_config): Json, ) -> Result { require_admin(is_admin, &username)?; + check_git_sync_access(&db, &w_id).await?; let mut tx = db.begin().await?; @@ -1764,19 +1814,6 @@ async fn edit_git_sync_config( Ok(format!("Edit git sync config for workspace {}", &w_id)) } -#[cfg(not(feature = "enterprise"))] -async fn edit_git_sync_repository( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_new_config): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] async fn edit_git_sync_repository( authed: ApiAuthed, Extension(db): Extension, @@ -1785,10 +1822,19 @@ async fn edit_git_sync_repository( Json(new_config): Json, ) -> Result { require_admin(is_admin, &username)?; + check_git_sync_access(&db, &w_id).await?; // Validate the resource path format validate_git_repo_resource_path(&new_config.git_repo_resource_path)?; + // Promotion mode: EE only + #[cfg(not(feature = "enterprise"))] + if new_config.repository.use_individual_branch.unwrap_or(false) { + return Err(Error::BadRequest( + "Promotion mode is an Enterprise Edition feature".to_string(), + )); + } + let mut tx = db.begin().await?; // First, get the current git sync settings @@ -1810,6 +1856,20 @@ async fn edit_git_sync_repository( WorkspaceGitSyncSettings::default() }; + // Multi-repo: EE only + #[cfg(not(feature = "enterprise"))] + { + let is_new = !git_sync_settings + .repositories + .iter() + .any(|r| r.git_repo_resource_path == new_config.git_repo_resource_path); + if is_new && !git_sync_settings.repositories.is_empty() { + return Err(Error::BadRequest( + "Multiple git sync repositories is an Enterprise Edition feature".to_string(), + )); + } + } + // Audit log before we move the repository audit_log( &mut *tx, @@ -1893,19 +1953,6 @@ async fn edit_git_sync_repository( )) } -#[cfg(not(feature = "enterprise"))] -async fn delete_git_sync_repository( - _authed: ApiAuthed, - Extension(_db): Extension, - Path(_w_id): Path, - Json(_request): Json, -) -> Result { - return Err(Error::BadRequest( - "Git sync is only available on Windmill Enterprise Edition".to_string(), - )); -} - -#[cfg(feature = "enterprise")] async fn delete_git_sync_repository( authed: ApiAuthed, Extension(db): Extension, @@ -1915,7 +1962,7 @@ async fn delete_git_sync_repository( ) -> Result { require_admin(is_admin, &username)?; - // For deletion, only validate that path is not empty to allow cleanup of malformed entries + // No check_git_sync_access here — admins should always be able to delete/clean up repos if request.git_repo_resource_path.is_empty() { return Err(Error::BadRequest( "Resource path cannot be empty".to_string(), diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5f350f2ce5..a8f83c368a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3430,6 +3430,37 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/git_sync_enabled: + get: + summary: Check if git sync is available for this workspace + operationId: getGitSyncEnabled + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: Git sync availability status + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + reason: + type: string + nullable: true + max_repos: + type: integer + nullable: true + user_count: + type: integer + nullable: true + max_users: + type: integer + nullable: true + /w/{workspace}/workspaces/edit_git_sync_config: post: summary: edit workspace git sync settings diff --git a/backend/windmill-common/src/ee_oss.rs b/backend/windmill-common/src/ee_oss.rs index 51b1efd2e2..93d0061ade 100644 --- a/backend/windmill-common/src/ee_oss.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -23,6 +23,7 @@ lazy_static::lazy_static! { } #[cfg(not(feature = "private"))] +#[derive(PartialEq, Eq)] pub enum LicensePlan { Community, Pro, diff --git a/backend/windmill-git-sync/Cargo.toml b/backend/windmill-git-sync/Cargo.toml index 063e1ce54a..148746dcca 100644 --- a/backend/windmill-git-sync/Cargo.toml +++ b/backend/windmill-git-sync/Cargo.toml @@ -9,7 +9,7 @@ name = "windmill_git_sync" path = "./src/lib.rs" [features] -private = [] +private = ["windmill-common/private"] enterprise = ["windmill-queue/enterprise", "windmill-common/enterprise"] all_sqlx_features = ["enterprise"] default = [] diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index f9cecd46ce..dcbcd5bcb2 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -272,7 +272,10 @@ mod tests { path: "f/folder/script".to_string(), parent_path: Some("f/folder/old_script".to_string()), }; - assert_eq!(obj.get_parent_path(), Some("f/folder/old_script".to_string())); + assert_eq!( + obj.get_parent_path(), + Some("f/folder/old_script".to_string()) + ); } #[test] @@ -313,21 +316,13 @@ mod tests { #[test] fn test_get_kind_flow() { - let obj = DeployedObject::Flow { - path: "test".to_string(), - parent_path: None, - version: 1, - }; + let obj = DeployedObject::Flow { path: "test".to_string(), parent_path: None, version: 1 }; assert_eq!(obj.get_kind(), "flow"); } #[test] fn test_get_kind_app() { - let obj = DeployedObject::App { - path: "test".to_string(), - version: 1, - parent_path: None, - }; + let obj = DeployedObject::App { path: "test".to_string(), version: 1, parent_path: None }; assert_eq!(obj.get_kind(), "app"); } @@ -346,7 +341,8 @@ mod tests { "http_trigger" ); assert_eq!( - DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None }.get_kind(), + DeployedObject::WebsocketTrigger { path: "t".to_string(), parent_path: None } + .get_kind(), "websocket_trigger" ); assert_eq!( diff --git a/cli/package-lock.json b/cli/package-lock.json index 0e86b9d2b7..ae46c240c9 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -25,10 +25,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", @@ -1414,6 +1415,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz", "integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw==" }, + "node_modules/windmill-parser-wasm-py-imports": { + "version": "1.659.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py-imports/-/windmill-parser-wasm-py-imports-1.659.1.tgz", + "integrity": "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ==" + }, "node_modules/windmill-parser-wasm-regex": { "version": "1.639.0", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.639.0.tgz", @@ -1430,9 +1436,9 @@ "integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.647.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.647.1.tgz", - "integrity": "sha512-64iSAUMU5W/WtePqE1vtDvglDqtkiZVndyieYBVDX0nl7UuovS+wPgH/P3TEoKbR+FwAPacki0CX3DsEzZ/Yxw==" + "version": "1.659.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.659.1.tgz", + "integrity": "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.593.0", diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index 834982c3f3..ba919fa16b 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -1,4 +1,7 @@ import { getContext, setContext } from 'svelte' +import { enterpriseLicense } from '$lib/stores' +import { get } from 'svelte/store' +import { sendUserToast } from '$lib/toast' import { JobService, WorkspaceService, ResourceService } from '$lib/gen' import type { GitRepositorySettings as BackendGitRepositorySettings, @@ -646,6 +649,10 @@ export function createGitSyncContext(workspace: string) { } function addSyncRepository() { + if (!get(enterpriseLicense) && repositories && repositories.length >= 1) { + sendUserToast('Multiple repositories requires Enterprise Edition', true) + return + } repositories.push({ git_repo_resource_path: '', script_path: undefined, @@ -669,6 +676,10 @@ export function createGitSyncContext(workspace: string) { } function addPromotionRepository() { + if (!get(enterpriseLicense)) { + sendUserToast('Promotion mode requires Enterprise Edition', true) + return + } repositories.push({ git_repo_resource_path: '', script_path: undefined, diff --git a/frontend/src/lib/components/git_sync/GitSyncSection.svelte b/frontend/src/lib/components/git_sync/GitSyncSection.svelte index 893e782b19..9a03e43b27 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSection.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSection.svelte @@ -6,12 +6,46 @@ import GitSyncRepositoryCard from './GitSyncRepositoryCard.svelte' import GitSyncModalManager from './GitSyncModalManager.svelte' import { enterpriseLicense, workspaceStore } from '$lib/stores' + import { WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { untrack } from 'svelte' // Create context reactively based on workspaceStore const gitSyncContext = $derived($workspaceStore ? setGitSyncContext($workspaceStore) : null) + // Fetch git sync eligibility + let gitSyncStatus = $state<{ + enabled: boolean + reason: string | null + max_repos: number | null + user_count: number | null + max_users: number | null + }>({ enabled: false, reason: null, max_repos: null, user_count: null, max_users: null }) + + $effect(() => { + if ($workspaceStore) { + WorkspaceService.getGitSyncEnabled({ workspace: $workspaceStore }) + .then((status) => { + gitSyncStatus = status as typeof gitSyncStatus + }) + .catch(() => { + gitSyncStatus = { + enabled: false, + reason: null, + max_repos: null, + user_count: null, + max_users: null + } + }) + } + }) + + const gitSyncAllowed = $derived(gitSyncStatus.enabled) + const isFreeTier = $derived(gitSyncAllowed && !$enterpriseLicense) + const hasConfiguredRepos = $derived( + gitSyncContext?.repositories?.some((r) => r.git_repo_resource_path) ?? false + ) + // Load settings when workspace context changes $effect(() => { if (gitSyncContext) { @@ -58,7 +92,7 @@ link="https://www.windmill.dev/docs/advanced/git_sync" > {#snippet actions()} - {#if $enterpriseLicense && gitSyncContext.repositories != undefined} + {#if (gitSyncAllowed || gitSyncStatus.user_count != null) && gitSyncContext?.repositories != undefined} - - {#if secondarySyncExpanded} -
    - {#if secondarySync.length === 0} -
    - No secondary sync repositories configured -
    - {:else} - {#each secondarySync as { repo, idx } (repo.git_repo_resource_path)} -
    - -
    - {/each} - {/if} - - {#if !hasUnsavedSecondary} -
    - -
    - {/if} -
    - {/if} -
    - {:else} - - {#if !hasUnsavedSecondary} -
    - -
    - {/if} - {/if} - {/if} - - -
    - gitSyncContext.addPromotionRepository()} - isCollapsible={false} - showEmptyState={primaryPromotion?.repo === null} - /> - - - {#if primaryPromotion && !primaryPromotion.repo?.isUnsavedConnection} - {#if secondaryPromotion.length > 0 || secondaryPromotionExpanded} + {#if $enterpriseLicense} + + {#if primarySync && !primarySync.repo?.isUnsavedConnection} + {#if secondarySync.length > 0 || secondarySyncExpanded}
    - {#if secondaryPromotionExpanded} + {#if secondarySyncExpanded}
    - {#if secondaryPromotion.length === 0} + {#if secondarySync.length === 0}
    - No secondary promotion repositories configured + No secondary sync repositories configured
    {:else} - {#each secondaryPromotion as { repo, idx } (repo.git_repo_resource_path)} + {#each secondarySync as { repo, idx } (repo.git_repo_resource_path)}
    {/each} {/if} - {#if !hasUnsavedSecondaryPromotion} + {#if !hasUnsavedSecondary}
    {/if} @@ -216,23 +187,99 @@ {/if}
    {:else} - - {#if !hasUnsavedSecondaryPromotion} + + {#if !hasUnsavedSecondary}
    {/if} {/if} {/if} -
    + + +
    + gitSyncContext.addPromotionRepository()} + isCollapsible={false} + showEmptyState={primaryPromotion?.repo === null} + /> + + + {#if primaryPromotion && !primaryPromotion.repo?.isUnsavedConnection} + {#if secondaryPromotion.length > 0 || secondaryPromotionExpanded} +
    + + + {#if secondaryPromotionExpanded} +
    + {#if secondaryPromotion.length === 0} +
    + No secondary promotion repositories configured +
    + {:else} + {#each secondaryPromotion as { repo, idx } (repo.git_repo_resource_path)} +
    + +
    + {/each} + {/if} + + {#if !hasUnsavedSecondaryPromotion} +
    + +
    + {/if} +
    + {/if} +
    + {:else} + + {#if !hasUnsavedSecondaryPromotion} +
    + +
    + {/if} + {/if} + {/if} +
    + {/if}
    From 10c5c97d3723dc317ed0a30098d248d91777d01a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 08:48:05 +0000 Subject: [PATCH 35/48] nit frontend --- frontend/src/lib/components/git_sync/GitSyncSection.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/git_sync/GitSyncSection.svelte b/frontend/src/lib/components/git_sync/GitSyncSection.svelte index 9a03e43b27..8041c734da 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSection.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSection.svelte @@ -120,7 +120,7 @@ {:else if isFreeTier}
    - + Git sync is an EE feature provided in CE for testing and hobbyist use when workspace members ≤ {gitSyncStatus.max_users}. Limited to a single repository. Upgrade to EE for multiple repositories, promotion mode, and GitHub App authentication. From 60804a96c630087958e3dc8b8ea0c87cbb690bce Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 25 Mar 2026 09:50:12 +0100 Subject: [PATCH 36/48] refactor: unify eval pipeline with production chat code path (#8504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: unify eval pipeline with production chat code path Extract a shared headless runChatLoop() that both AIChatManager (production) and the eval runner use, with injectable SDK clients. Drop OpenRouter — evals now use direct provider APIs (OpenAI SDK, Anthropic SDK) with streaming, matching production behavior. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: re-read tools/helpers/systemMessage/model on each loop iteration The old chatRequest() re-read this.tools, this.helpers, this.systemMessage, and getCurrentModel() on every iteration. This matters because changeModeTool (Navigator → Script/Flow) reassigns all of these mid-loop. Use JS getters in the config object so runChatLoop picks up changes each iteration. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- .../copilot/chat/AIChatManager.svelte.ts | 163 ++++--------- .../chat/__tests__/app/appChat.eval.test.ts | 165 +++++++++---- .../chat/__tests__/app/appEvalComparison.ts | 31 +-- .../chat/__tests__/app/appEvalRunner.ts | 39 ++-- .../chat/__tests__/flow/flowChat.eval.test.ts | 151 ++++++++---- .../chat/__tests__/flow/flowEvalComparison.ts | 6 +- .../chat/__tests__/flow/flowEvalRunner.ts | 39 ++-- .../chat/__tests__/shared/baseEvalRunner.ts | 216 ++++++++---------- .../chat/__tests__/shared/baseLLMEvaluator.ts | 37 +-- .../copilot/chat/__tests__/shared/types.ts | 3 + .../lib/components/copilot/chat/anthropic.ts | 24 +- .../lib/components/copilot/chat/chatLoop.ts | 211 +++++++++++++++++ .../copilot/chat/openai-responses.ts | 23 +- .../src/lib/components/copilot/chat/shared.ts | 6 +- frontend/src/lib/components/copilot/lib.ts | 26 ++- 15 files changed, 743 insertions(+), 397 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/chatLoop.ts diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 00a1a11fdf..140bbf4f14 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -23,8 +23,7 @@ import { } from './shared' import type { ChatCompletionMessageParam, - ChatCompletionSystemMessageParam, - ChatCompletionUserMessageParam + ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs' import { prepareInlineChatSystemPrompt, @@ -37,7 +36,7 @@ import { loadApiTools } from './api/apiTools' import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' -import { getCompletion, getModelContextWindow, parseOpenAICompletion } from '../lib' +import { getModelContextWindow, workspaceAIClients } from '../lib' import { dfs } from '$lib/components/flows/previousResults' import { getStringError } from './utils' import type { FlowModuleState, FlowState } from '$lib/components/flows/flowState' @@ -56,8 +55,7 @@ import type { import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' -import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' -import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses' +import { runChatLoop } from './chatLoop' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, tryGetCurrentModel, getCombinedCustomPrompt } from '$lib/aiStore' @@ -413,130 +411,63 @@ class AIChatManager { systemMessage?: ChatCompletionSystemMessageParam }) => { try { - let addedMessages: ChatCompletionMessageParam[] = [] - while (true) { - const systemMessage = systemMessageOverride ?? this.systemMessage - const helpers = this.helpers - const tools = this.tools - for (const tool of tools) { - if (tool.setSchema) { - await tool.setSchema(helpers) - } - } - - let pendingPrompt = this.pendingPrompt - let pendingUserMessage: ChatCompletionUserMessageParam | undefined = undefined - if (pendingPrompt) { + // Use JS getters so runChatLoop re-reads tools/helpers/systemMessage/modelProvider + // on each iteration. This is critical for changeModeTool (Navigator → Script/Flow) + // which reassigns this.tools, this.helpers, this.systemMessage mid-loop. + const self = this + const result = await runChatLoop({ + messages, + get systemMessage() { + return systemMessageOverride ?? self.systemMessage + }, + get tools() { + return self.tools + }, + get helpers() { + return self.helpers + }, + abortController, + callbacks, + get modelProvider() { + return getCurrentModel() + }, + clients: { + openai: workspaceAIClients.getOpenaiClient(), + anthropic: workspaceAIClients.getAnthropicClient() + }, + workspace: get(workspaceStore) ?? '', + skipResponsesApi: this.skipResponsesApi, + onSkipResponsesApi: () => { + this.skipResponsesApi = true + }, + getPendingUserMessage: () => { + const pendingPrompt = this.pendingPrompt + if (!pendingPrompt) return undefined + this.pendingPrompt = '' if (this.mode === AIMode.SCRIPT) { - pendingUserMessage = prepareScriptUserMessage( + return prepareScriptUserMessage( pendingPrompt, this.contextManager.getSelectedContext() ) } else if (this.mode === AIMode.FLOW) { - pendingUserMessage = prepareFlowUserMessage( + return prepareFlowUserMessage( pendingPrompt, this.flowAiChatHelpers!.getFlowAndSelectedId() ) } else if (this.mode === AIMode.NAVIGATOR) { - pendingUserMessage = prepareNavigatorUserMessage(pendingPrompt) + return prepareNavigatorUserMessage(pendingPrompt) } - this.pendingPrompt = '' - } - - const model = getCurrentModel() - const isOpenAI = model.provider === 'openai' || model.provider === 'azure_openai' - const isAnthropic = model.provider === 'anthropic' - - const messageParams = [ - systemMessage, - ...messages, - ...(pendingUserMessage ? [pendingUserMessage] : []) - ] - const toolDefs = tools.map((t) => t.def) - - // For OpenAI/Azure, try Responses API first, fallback to Completions API - if (isOpenAI) { - let useCompletionsApi = this.skipResponsesApi - if (!this.skipResponsesApi) { - try { - const completion = await getOpenAIResponsesCompletion( - messageParams, - abortController, - toolDefs - ) - const continueCompletion = await parseOpenAIResponsesCompletion( - completion, - callbacks, - messages, - addedMessages, - tools, - helpers - ) - if (!continueCompletion) { - break - } - } catch (err) { - console.warn('OpenAI Responses API failed, falling back to Completions API:', err) - // If the error indicates Responses API is not available in this region, skip it for future requests - const errorMessage = err instanceof Error ? err.message : String(err) - if (errorMessage.includes('Responses API is not enabled')) { - this.skipResponsesApi = true - } - useCompletionsApi = true - } - } - - // Use Completions API if Responses API is not available or failed - if (useCompletionsApi) { - const completion = await getCompletion(messageParams, abortController, toolDefs, { - forceCompletions: true - }) - const continueCompletion = await parseOpenAICompletion( - completion, - callbacks, - messages, - addedMessages, - tools, - helpers - ) - if (!continueCompletion) { - break - } - } - } else if (isAnthropic) { - const completion = await getAnthropicCompletion(messageParams, abortController, toolDefs) - if (completion) { - const continueCompletion = await parseAnthropicCompletion( - completion, - callbacks, - messages, - addedMessages, - tools, - helpers, - abortController - ) - if (!continueCompletion) { - break - } - } - } else { - const completion = await getCompletion(messageParams, abortController, toolDefs) - if (completion) { - const continueCompletion = await parseOpenAICompletion( - completion, - callbacks, - messages, - addedMessages, - tools, - helpers - ) - if (!continueCompletion) { - break + return undefined + }, + onBeforeIteration: async (tools) => { + for (const tool of tools) { + if (tool.setSchema) { + await tool.setSchema(this.helpers) } } } - } - return addedMessages + }) + return result.addedMessages } catch (err) { console.log('chatRequest error', err) console.error('chatRequest error', err) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/app/appChat.eval.test.ts b/frontend/src/lib/components/copilot/chat/__tests__/app/appChat.eval.test.ts index 5183377caf..a42ee1f099 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/app/appChat.eval.test.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/app/appChat.eval.test.ts @@ -6,44 +6,77 @@ import { loadAppFixtureForEval } from './appFixtureLoader' import { dirname, join } from 'path' // @ts-ignore - Node.js url import { fileURLToPath } from 'url' +import type { AIProvider } from '$lib/gen/types.gen' -// Get API key from environment - tests will be skipped if not set +// Get API keys from environment - tests will be skipped if none are set // @ts-ignore -const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY +const OPENAI_API_KEY = process.env.OPENAI_API_KEY +// @ts-ignore +const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY -// Skip all tests if no API key is provided -const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip +const hasAnyKey = OPENAI_API_KEY || ANTHROPIC_API_KEY +const describeWithApiKey = hasAnyKey ? describe : describe.skip // Get __dirname equivalent for ES modules const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) -const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o'] +// Build model variants based on available keys +interface ModelVariant { + model: string + provider: AIProvider + apiKey: string +} + +const MODEL_VARIANTS: ModelVariant[] = [ + ...(OPENAI_API_KEY + ? [{ model: 'gpt-4o', provider: 'openai' as AIProvider, apiKey: OPENAI_API_KEY }] + : []), + ...(ANTHROPIC_API_KEY + ? [ + { + model: 'claude-haiku-4-5-20241022', + provider: 'anthropic' as AIProvider, + apiKey: ANTHROPIC_API_KEY + } + ] + : []) +] + const VARIANTS = [ - ...MODELS.map((model) => ({ + ...MODEL_VARIANTS.map((mv) => ({ ...BASELINE_VARIANT, - model, - name: `baseline-${model.replace('/', '-')}` + model: mv.model, + name: `baseline-${mv.provider}-${mv.model}`, + _provider: mv.provider, + _apiKey: mv.apiKey })), - ...MODELS.map((model) => ({ + ...MODEL_VARIANTS.map((mv) => ({ ...STREAMLINED_VARIANT, - model, - name: `streamlined-${model.replace('/', '-')}` + model: mv.model, + name: `streamlined-${mv.provider}-${mv.model}`, + _provider: mv.provider, + _apiKey: mv.apiKey })) ] describeWithApiKey('App Chat LLM Evaluation', () => { const TEST_TIMEOUT = 120_000 - if (!OPENROUTER_API_KEY) { - console.warn('OPENROUTER_API_KEY is not set, skipping tests') + if (!hasAnyKey) { + console.warn('No API keys set (OPENAI_API_KEY or ANTHROPIC_API_KEY), skipping tests') } it( 'test1: creates a simple counter app', async () => { const USER_PROMPT = `Create a counter app with increment/decrement buttons` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!) - // Write results to files + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + undefined, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) console.log(`App files: ${appPaths.join(', ')}`) @@ -56,17 +89,21 @@ describeWithApiKey('App Chat LLM Evaluation', () => { it( 'test2: modifies existing counter app to add reset button', async () => { - // Load initial app from fixture folder const { initialFrontend, initialBackend } = await loadAppFixtureForEval( join(__dirname, 'initial', 'test1_counter_app') ) const USER_PROMPT = `Add a reset button that sets the counter back to 0` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) - // Write results to files + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) console.log(`App files: ${appPaths.join(', ')}`) @@ -86,10 +123,16 @@ describeWithApiKey('App Chat LLM Evaluation', () => { ) const USER_PROMPT = `Add a quantity selector (+ and - buttons) to each cart item so users can adjust quantities without removing and re-adding items` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -108,10 +151,16 @@ describeWithApiKey('App Chat LLM Evaluation', () => { ) const USER_PROMPT = `Add a discount code input field in the cart. When the code "SAVE10" is entered, apply a 10% discount to the total` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -132,10 +181,16 @@ describeWithApiKey('App Chat LLM Evaluation', () => { ) const USER_PROMPT = `Add a search bar in the toolbar that filters files and folders by name as the user types` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -154,10 +209,16 @@ describeWithApiKey('App Chat LLM Evaluation', () => { ) const USER_PROMPT = `Show file size (formatted as KB/MB) and modified date in the file list for each item` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -176,10 +237,16 @@ describeWithApiKey('App Chat LLM Evaluation', () => { ) const USER_PROMPT = `Add a "Select All" checkbox in the file list header and individual checkboxes for each file. Add a "Delete Selected" button that appears when items are selected` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialFrontend, - initialBackend - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialFrontend, + initialBackend + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -196,7 +263,13 @@ describeWithApiKey('App Chat LLM Evaluation', () => { 'test8: create quiz app from scratch', async () => { const USER_PROMPT = `Create a multiple choice quiz app with 5 questions about general knowledge. Show one question at a time with 4 answer options. Track the score and show results at the end with percentage correct.` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + undefined, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -211,7 +284,13 @@ describeWithApiKey('App Chat LLM Evaluation', () => { 'test9: create recipe book from scratch', async () => { const USER_PROMPT = `Create a recipe book app where users can add recipes with a name, ingredients list, and instructions. Include a search bar to filter recipes by name and the ability to delete recipes.` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + undefined, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, appPaths } = await writeAppComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts index 456299c142..e6c795d445 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalComparison.ts @@ -1,4 +1,4 @@ -import OpenAI from 'openai' +import Anthropic from '@anthropic-ai/sdk' import type { AppFiles, BackendRunnable } from '../../app/core' import { BASE_EVALUATOR_RESPONSE_FORMAT } from '../shared' import type { EvaluationResult } from '../shared' @@ -71,12 +71,7 @@ ${BASE_EVALUATOR_RESPONSE_FORMAT}` /** * Evaluates how well a generated app fulfills the user's request, considering any initial app state. - * This evaluator does not require an expected reference app - it evaluates based on the request alone. - * - * @param userPrompt The original user request - * @param generatedApp The app generated by the AI - * @param initialApp Optional initial app state (what the app looked like before AI changes) - * @returns Evaluation result with score, statement, and missing requirements + * Uses Anthropic API directly. */ export async function evaluateAppGeneration( userPrompt: string, @@ -84,9 +79,17 @@ export async function evaluateAppGeneration( initialApp?: InitialApp ): Promise { // @ts-ignore - const apiKey = process.env.OPENROUTER_API_KEY + const apiKey = process.env.ANTHROPIC_API_KEY + if (!apiKey) { + return { + success: false, + resemblanceScore: 0, + statement: 'No API key available for evaluation', + error: 'ANTHROPIC_API_KEY not set' + } + } - const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey }) + const client = new Anthropic({ apiKey }) let userMessage = `## User's Original Request ${userPrompt} @@ -117,16 +120,18 @@ Please evaluate how well the generated app: 2. ${initialApp ? 'Makes appropriate modifications to the initial app state' : 'Implements a complete and correct new app'}` try { - const response = await client.chat.completions.create({ - model: 'anthropic/claude-sonnet-4.5', + const response = await client.messages.create({ + model: 'claude-sonnet-4-5-20250514', + max_tokens: 2048, + system: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT, messages: [ - { role: 'system', content: APP_GENERATION_EVALUATOR_SYSTEM_PROMPT }, { role: 'user', content: userMessage } ], temperature: 0 }) - const content = response.choices[0]?.message?.content + const textBlock = response.content.find((block) => block.type === 'text') + const content = textBlock?.text if (!content) { return { success: false, diff --git a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts index 3f0da73c92..2e6a491bce 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/app/appEvalRunner.ts @@ -14,6 +14,7 @@ import { type VariantDefaults } from '../shared' import { writeAppComparisonResultsToFolders } from './appResultsWriter' +import type { AIProvider } from '$lib/gen/types.gen' // Re-export for convenience export type { InitialApp } from './appEvalComparison' @@ -38,6 +39,8 @@ export interface AppEvalOptions { variant?: VariantConfig /** Whether to evaluate the generated app with LLM. Default: true. Set to false to skip evaluation. */ evaluateWithLLM?: boolean + /** AI provider (inferred from model name if omitted) */ + provider?: AIProvider } /** @@ -49,12 +52,11 @@ const appDefaults: VariantDefaults = { } /** - * Runs an app chat evaluation with real OpenAI API calls. - * Executes tool calls using the actual app tools from core.ts or variant-configured tools. + * Runs an app chat evaluation using the shared chat loop (same code path as production). */ export async function runAppEval( userPrompt: string, - openaiApiKey: string, + apiKey: string, options?: AppEvalOptions ): Promise { const { helpers, getFiles } = createAppEvalHelpers( @@ -69,7 +71,7 @@ export async function runAppEval( appDefaults, options?.customSystemPrompt ) - const { toolDefs, tools } = resolveTools(options?.variant, appDefaults) + const { tools } = resolveTools(options?.variant, appDefaults) const model = resolveModel(options?.variant, options?.model) // Build user message @@ -80,15 +82,15 @@ export async function runAppEval( userPrompt, systemMessage, userMessage, - toolDefs, tools, helpers, - apiKey: openaiApiKey, + apiKey, getOutput: getFiles, options: { maxIterations: options?.maxIterations, model, - workspace: 'test-workspace' + workspace: 'test-workspace', + provider: options?.provider } }) @@ -114,21 +116,32 @@ export async function runAppEval( } } +/** + * Per-variant provider override. + */ +export interface VariantProviderOverride { + provider: AIProvider + apiKey: string +} + /** * Runs the same prompt against multiple variants sequentially for comparison. - * Returns results in the same order as the input variants. + * Accepts optional per-variant provider/apiKey overrides. */ export async function runVariantComparison( userPrompt: string, variants: VariantConfig[], - openaiApiKey: string, - baseOptions?: Omit + defaultApiKey: string, + baseOptions?: Omit, + providerOverrides?: VariantProviderOverride[] ): Promise { const results: AppEvalResult[] = await Promise.all( - variants.map(async (variant) => { - return await runAppEval(userPrompt, openaiApiKey, { + variants.map(async (variant, i) => { + const override = providerOverrides?.[i] + return await runAppEval(userPrompt, override?.apiKey ?? defaultApiKey, { ...baseOptions, - variant + variant, + provider: override?.provider ?? baseOptions?.provider }) }) ) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowChat.eval.test.ts b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowChat.eval.test.ts index 8210ea50fb..de9b8e5f43 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowChat.eval.test.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowChat.eval.test.ts @@ -22,35 +22,60 @@ import initialTest6 from './initial/test6_initial.json' // @ts-ignore - JSON import import initialTest7 from './initial/test7_initial.json' import type { FlowModule } from '$lib/gen' +import type { AIProvider } from '$lib/gen/types.gen' -// Get API key from environment - tests will be skipped if not set +// Get API keys from environment - tests will be skipped if none are set // @ts-ignore -// const OPENAI_API_KEY = process.env.OPENAI_API_KEY -const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY +const OPENAI_API_KEY = process.env.OPENAI_API_KEY +// @ts-ignore +const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY -// Skip all tests if no API key is provided -// const describeWithApiKey = OPENAI_API_KEY ? describe : describe.skip -const describeWithApiKey = OPENROUTER_API_KEY ? describe : describe.skip +const hasAnyKey = OPENAI_API_KEY || ANTHROPIC_API_KEY +const describeWithApiKey = hasAnyKey ? describe : describe.skip -const MODELS = ['google/gemini-2.5-flash', 'anthropic/claude-haiku-4.5', 'openai/gpt-4o'] +// Build model variants based on available keys +interface ModelVariant { + model: string + provider: AIProvider + apiKey: string +} + +const MODEL_VARIANTS: ModelVariant[] = [ + ...(OPENAI_API_KEY + ? [{ model: 'gpt-4o', provider: 'openai' as AIProvider, apiKey: OPENAI_API_KEY }] + : []), + ...(ANTHROPIC_API_KEY + ? [ + { + model: 'claude-haiku-4-5-20241022', + provider: 'anthropic' as AIProvider, + apiKey: ANTHROPIC_API_KEY + } + ] + : []) +] const VARIANTS = [ - ...MODELS.map((model) => ({ + ...MODEL_VARIANTS.map((mv) => ({ ...BASELINE_VARIANT, - model, - name: `baseline-${model.replace('/', '-')}` + model: mv.model, + name: `baseline-${mv.provider}-${mv.model}`, + _provider: mv.provider, + _apiKey: mv.apiKey })), - ...MODELS.map((model) => ({ + ...MODEL_VARIANTS.map((mv) => ({ ...MINIMAL_SINGLE_TOOL_VARIANT, - model, - name: `minimal-single-tool-${model.replace('/', '-')}` + model: mv.model, + name: `minimal-single-tool-${mv.provider}-${mv.model}`, + _provider: mv.provider, + _apiKey: mv.apiKey })) ] describeWithApiKey('Flow Chat LLM Evaluation', () => { const TEST_TIMEOUT = 120_000 - if (!OPENROUTER_API_KEY) { - console.warn('OPENROUTER_API_KEY is not set, skipping tests') + if (!hasAnyKey) { + console.warn('No API keys set (OPENAI_API_KEY or ANTHROPIC_API_KEY), skipping tests') } it( @@ -65,9 +90,15 @@ STEP 3: Loop on all users STEP 4: Do branches based on user's role, do different action based on that. Roles are admin, user, moderator STEP 5: Return action taken for each user ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - expectedFlow: expectedTest1 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + expectedFlow: expectedTest1 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) // Write results to files const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) @@ -112,9 +143,15 @@ STEP 5: Branch based on inventory - if all items available, create shipment reco STEP 6: Send confirmation (mock email to customer_email) STEP 7: Return final order summary with status ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - expectedFlow: expectedTest2 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + expectedFlow: expectedTest2 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -161,9 +198,15 @@ STEP 5: Branch based on quality score: - If score < 70: Store in quarantine and send alert STEP 6: Return processing report with statistics (total records, quality score, destination) ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - expectedFlow: expectedTest3 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + expectedFlow: expectedTest3 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -210,9 +253,15 @@ STEP 3: Use an AI agent to handle the customer query. The agent should have acce STEP 4: Log the interaction to audit trail (customer_id, query, response summary) STEP 5: Return the agent's response and any actions taken ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - expectedFlow: expectedTest4 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + expectedFlow: expectedTest4 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -256,11 +305,17 @@ Modify this existing flow to add error handling: - If validation passes, return the data for the next step - Update save_results to handle the validation result appropriately ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialModules: initialTest5.value.modules as FlowModule[], - initialSchema: initialTest5.schema, - expectedFlow: expectedTest5 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialModules: initialTest5.value.modules as FlowModule[], + initialSchema: initialTest5.schema, + expectedFlow: expectedTest5 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -302,11 +357,17 @@ Modify the order processing loop to handle different order types: - Move the original process_order step to the default branch for unknown order types - Each branch step should return the orderId, shipping cost, and shipping type ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialModules: initialTest6.value.modules as FlowModule[], - initialSchema: initialTest6.schema, - expectedFlow: expectedTest6 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialModules: initialTest6.value.modules as FlowModule[], + initialSchema: initialTest6.schema, + expectedFlow: expectedTest6 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) @@ -348,11 +409,17 @@ Refactor this flow for better performance by parallelizing the enrichment steps: - The combine_data step should check if any enrichment used a fallback value and set a hasFallbacks flag - Keep get_item as the first step and return_result as the last step unchanged ` - const results = await runVariantComparison(USER_PROMPT, VARIANTS, OPENROUTER_API_KEY!, { - initialModules: initialTest7.value.modules as FlowModule[], - initialSchema: initialTest7.schema, - expectedFlow: expectedTest7 as ExpectedFlow - }) + const results = await runVariantComparison( + USER_PROMPT, + VARIANTS, + VARIANTS[0]._apiKey, + { + initialModules: initialTest7.value.modules as FlowModule[], + initialSchema: initialTest7.schema, + expectedFlow: expectedTest7 as ExpectedFlow + }, + VARIANTS.map((v) => ({ provider: v._provider, apiKey: v._apiKey })) + ) const { summaryPath, flowPaths } = await writeFlowComparisonResults(USER_PROMPT, results) console.log(`\nResults written to: ${summaryPath}`) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalComparison.ts b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalComparison.ts index f55979bb40..4c2b41d577 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalComparison.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalComparison.ts @@ -59,14 +59,10 @@ export async function evaluateFlowComparison( expectedFlow: ExpectedFlow, userPrompt: string ): Promise { - // @ts-ignore - const apiKey = process.env.OPENROUTER_API_KEY - return evaluateWithLLM({ userPrompt, generatedOutput: generatedFlow, expectedOutput: expectedFlow, - evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT, - apiKey + evaluatorSystemPrompt: FLOW_EVALUATOR_SYSTEM_PROMPT }) } diff --git a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalRunner.ts b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalRunner.ts index 3f27143c69..f3c976950d 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalRunner.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/flow/flowEvalRunner.ts @@ -1,4 +1,5 @@ import type { FlowModule } from '$lib/gen' +import type { AIProvider } from '$lib/gen/types.gen' import type { ExtendedOpenFlow } from '$lib/components/flows/types' import { flowTools, prepareFlowSystemMessage, prepareFlowUserMessage, type FlowAIChatHelpers } from '../../flow/core' import { createFlowEvalHelpers } from './flowEvalHelpers' @@ -38,6 +39,8 @@ export interface FlowEvalOptions { maxIterations?: number variant?: VariantConfig expectedFlow?: ExpectedFlow + /** AI provider (inferred from model name if omitted) */ + provider?: AIProvider } /** @@ -49,12 +52,11 @@ const flowDefaults: VariantDefaults = { } /** - * Runs a flow chat evaluation with real OpenAI API calls. - * Executes tool calls using the actual flowTools from core.ts or variant-configured tools. + * Runs a flow chat evaluation using the shared chat loop (same code path as production). */ export async function runFlowEval( userPrompt: string, - openaiApiKey: string, + apiKey: string, options?: FlowEvalOptions ): Promise { const { helpers, getFlow } = createFlowEvalHelpers( @@ -65,7 +67,7 @@ export async function runFlowEval( // Resolve variant configuration const variantName = options?.variant?.name ?? 'baseline' const systemMessage = resolveSystemPrompt(options?.variant, flowDefaults, options?.customSystemPrompt) - const { toolDefs, tools } = resolveTools(options?.variant, flowDefaults) + const { tools } = resolveTools(options?.variant, flowDefaults) const model = resolveModel(options?.variant, options?.model) // Build user message @@ -76,15 +78,15 @@ export async function runFlowEval( userPrompt, systemMessage, userMessage, - toolDefs, tools, helpers, - apiKey: openaiApiKey, + apiKey, getOutput: getFlow, options: { maxIterations: options?.maxIterations, model, - workspace: 'test-workspace' + workspace: 'test-workspace', + provider: options?.provider } }) @@ -111,21 +113,32 @@ export async function runFlowEval( } } +/** + * Per-variant provider override. + */ +export interface VariantProviderOverride { + provider: AIProvider + apiKey: string +} + /** * Runs the same prompt against multiple variants sequentially for comparison. - * Returns results in the same order as the input variants. + * Accepts optional per-variant provider/apiKey overrides. */ export async function runVariantComparison( userPrompt: string, variants: VariantConfig[], - openaiApiKey: string, - baseOptions?: Omit + defaultApiKey: string, + baseOptions?: Omit, + providerOverrides?: VariantProviderOverride[] ): Promise { const results: FlowEvalResult[] = await Promise.all( - variants.map(async (variant) => { - return await runFlowEval(userPrompt, openaiApiKey, { + variants.map(async (variant, i) => { + const override = providerOverrides?.[i] + return await runFlowEval(userPrompt, override?.apiKey ?? defaultApiKey, { ...baseOptions, - variant + variant, + provider: override?.provider ?? baseOptions?.provider }) }) ) diff --git a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseEvalRunner.ts b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseEvalRunner.ts index b9b7820568..f46acb9108 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseEvalRunner.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseEvalRunner.ts @@ -1,8 +1,14 @@ -import OpenAI, { APIError } from 'openai' -import type { ChatCompletionMessageParam, ChatCompletionSystemMessageParam } from 'openai/resources/chat/completions.mjs' -import type { ChatCompletionTool } from 'openai/resources/chat/completions.mjs' +import OpenAI from 'openai' +import Anthropic from '@anthropic-ai/sdk' +import type { + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam +} from 'openai/resources/chat/completions.mjs' +import type { AIProvider, AIProviderModel } from '$lib/gen/types.gen' import type { TokenUsage, ToolCallDetail, EvalRunnerOptions } from './types' import type { Tool } from './baseVariants' +import { runChatLoop, type ChatClients } from '../../chatLoop' +import type { Tool as ProductionTool, ToolCallbacks } from '../../shared' /** * Result from a single eval run (before domain-specific evaluation). @@ -29,13 +35,13 @@ export interface RunEvalParams { systemMessage: ChatCompletionSystemMessageParam /** User message for the LLM */ userMessage: ChatCompletionMessageParam - /** Tool definitions for the LLM API */ - toolDefs: ChatCompletionTool[] + /** Tool definitions for the LLM API (unused — derived from tools) */ + toolDefs?: unknown /** Full tool implementations for execution */ tools: Tool[] /** Domain-specific helpers for tool execution */ helpers: THelpers - /** API key for OpenRouter */ + /** API key for the provider */ apiKey: string /** Function to get the current output state */ getOutput: () => TOutput @@ -44,10 +50,37 @@ export interface RunEvalParams { } /** - * Runs a generic evaluation with real LLM API calls. - * Executes tool calls in a loop until the LLM stops calling tools. - * - * This is the core execution loop shared across all chat eval tests. + * Creates SDK clients for the given provider. + */ +function createEvalClients(provider: AIProvider, apiKey: string): ChatClients { + if (provider === 'anthropic') { + return { + openai: new OpenAI({ apiKey: 'unused' }), + anthropic: new Anthropic({ apiKey }) + } + } + return { + openai: new OpenAI({ apiKey }), + anthropic: new Anthropic({ apiKey: 'unused' }) + } +} + +/** + * Resolves model string to AIProviderModel. + */ +function resolveModelProvider( + model: string, + provider?: AIProvider +): AIProviderModel { + if (provider) return { provider, model } + if (model.startsWith('claude')) return { provider: 'anthropic', model } + if (model.startsWith('gpt') || model.startsWith('o')) return { provider: 'openai', model } + return { provider: 'openai', model } +} + +/** + * Runs a generic evaluation using the shared chat loop (same code path as production). + * Uses streaming via real provider SDKs instead of OpenRouter non-streaming. */ export async function runEval( params: RunEvalParams @@ -55,7 +88,6 @@ export async function runEval( const { systemMessage, userMessage, - toolDefs, tools, helpers, apiKey, @@ -63,134 +95,82 @@ export async function runEval( options } = params - const client = new OpenAI({ baseURL: 'https://openrouter.ai/api/v1', apiKey }) const model = options?.model ?? 'gpt-4o' const maxIterations = options?.maxIterations ?? 20 const workspace = options?.workspace ?? 'test-workspace' + const provider = options?.provider - const messages: ChatCompletionMessageParam[] = [systemMessage, userMessage] - const totalTokens: TokenUsage = { prompt: 0, completion: 0, total: 0 } + const modelProvider = resolveModelProvider(model, provider) + const clients = createEvalClients(modelProvider.provider, apiKey) + + const messages: ChatCompletionMessageParam[] = [userMessage] let toolCallsCount = 0 const toolsCalled: string[] = [] const toolCallDetails: ToolCallDetail[] = [] - let iterations = 0 - // No-op tool callbacks for eval - const toolCallbacks = { + // Wrap tools to intercept fn calls for tracking. + // Cast to ProductionTool since the eval Tool has a narrower toolCallbacks type + // but the actual callbacks passed at runtime will satisfy both interfaces. + const wrappedTools = tools.map((tool) => ({ + ...tool, + fn: async (p: any) => { + toolCallsCount++ + toolsCalled.push(tool.def.function.name) + try { + const args = + typeof p.args === 'string' ? JSON.parse(p.args) : p.args + toolCallDetails.push({ name: tool.def.function.name, arguments: args }) + } catch { + toolCallDetails.push({ + name: tool.def.function.name, + arguments: p.args + }) + } + return tool.fn(p) + } + })) as ProductionTool[] + + // No-op callbacks for eval + const callbacks: ToolCallbacks & { + onNewToken: (token: string) => void + onMessageEnd: () => void + } = { setToolStatus: () => {}, - removeToolStatus: () => {} + removeToolStatus: () => {}, + onNewToken: () => {}, + onMessageEnd: () => {} } + const abortController = new AbortController() + try { - // Tool resolution loop - while (iterations < maxIterations) { - iterations++ - - const response = await client.chat.completions.create({ - model, - messages, - tools: toolDefs, - temperature: 0 - }) - - // Track token usage - if (response.usage) { - totalTokens.prompt += response.usage.prompt_tokens - totalTokens.completion += response.usage.completion_tokens - totalTokens.total += response.usage.total_tokens - } - - if (!response.choices.length) { - throw new Error('No response from API') - } - - const choice = response.choices[0] - const assistantMessage = choice.message - - // Add assistant message to history - messages.push(assistantMessage) - - // If no tool calls, we're done - if (!assistantMessage.tool_calls?.length) { - break - } - - // Execute each tool call - for (const toolCall of assistantMessage.tool_calls) { - toolCallsCount++ - - // Type guard: only handle function tool calls - if (toolCall.type !== 'function') { - messages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: `Unsupported tool type: ${toolCall.type}` - }) - continue - } - - toolsCalled.push(toolCall.function.name) - - const tool = tools.find((t) => t.def.function.name === toolCall.function.name) - if (!tool) { - messages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: `Unknown tool: ${toolCall.function.name}` - }) - continue - } - - try { - const args = JSON.parse(toolCall.function.arguments) - toolCallDetails.push({ name: toolCall.function.name, arguments: args }) - const result = await tool.fn({ - args, - workspace, - helpers, - toolCallbacks, - toolId: toolCall.id - }) - messages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: result - }) - } catch (err) { - const errorMessage = err instanceof Error ? err.message : String(err) - messages.push({ - role: 'tool', - tool_call_id: toolCall.id, - content: `Error: ${errorMessage}` - }) - } - } - } + const result = await runChatLoop({ + messages, + systemMessage, + tools: wrappedTools, + helpers, + abortController, + callbacks, + modelProvider, + clients, + workspace, + maxIterations, + skipResponsesApi: modelProvider.provider !== 'openai' && modelProvider.provider !== 'azure_openai' + }) return { success: true, output: getOutput(), - tokenUsage: totalTokens, + tokenUsage: { prompt: 0, completion: 0, total: 0 }, toolCallsCount, toolsCalled, toolCallDetails, - iterations, + iterations: Math.max(1, result.addedMessages.filter((m) => m.role === 'assistant').length), messages } } catch (err) { - // Build detailed error message let errorMessage: string - if (err instanceof APIError) { - const details: string[] = [`${err.status} ${err.message}`] - if (err.code) details.push(`Code: ${err.code}`) - if (err.type) details.push(`Type: ${err.type}`) - if (err.param) details.push(`Param: ${err.param}`) - if (err.requestID) details.push(`Request ID: ${err.requestID}`) - if (err.error && typeof err.error === 'object') { - details.push(`Response: ${JSON.stringify(err.error, null, 2)}`) - } - errorMessage = details.join('\n') - } else if (err instanceof Error) { + if (err instanceof Error) { errorMessage = err.stack ?? err.message } else { errorMessage = String(err) @@ -200,11 +180,11 @@ export async function runEval( success: false, output: getOutput(), error: errorMessage, - tokenUsage: totalTokens, + tokenUsage: { prompt: 0, completion: 0, total: 0 }, toolCallsCount, toolsCalled, toolCallDetails, - iterations, + iterations: 0, messages } } diff --git a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts index 63c17828f4..bd7bd06d44 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/shared/baseLLMEvaluator.ts @@ -1,4 +1,4 @@ -import OpenAI from 'openai' +import Anthropic from '@anthropic-ai/sdk' import type { EvaluationResult } from './types' /** @@ -13,9 +13,9 @@ export interface EvaluateParams { expectedOutput: unknown /** Domain-specific system prompt for the evaluator */ evaluatorSystemPrompt: string - /** API key for OpenRouter */ - apiKey: string - /** Model to use for evaluation (default: 'anthropic/claude-sonnet-4.5') */ + /** Anthropic API key for evaluation */ + apiKey?: string + /** Model to use for evaluation (default: 'claude-sonnet-4-5-20250514') */ model?: string } @@ -41,10 +41,7 @@ Score guidelines: /** * Evaluates how well a generated output matches an expected output using an LLM. - * Returns a resemblance score (0-100), a qualitative statement, and any missing requirements. - * - * @param params Evaluation parameters including prompts, outputs, and API configuration - * @returns Evaluation result with score, statement, and missing requirements + * Uses Anthropic API directly instead of OpenRouter. */ export async function evaluateWithLLM(params: EvaluateParams): Promise { const { @@ -53,10 +50,21 @@ export async function evaluateWithLLM(params: EvaluateParams): Promise block.type === 'text') + const content = textBlock?.text if (!content) { return { success: false, @@ -98,7 +108,6 @@ Please evaluate how well the generated output: // Parse JSON response - handle potential markdown code blocks let jsonContent = content.trim() if (jsonContent.startsWith('```')) { - // Remove markdown code block wrapper jsonContent = jsonContent.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '') } diff --git a/frontend/src/lib/components/copilot/chat/__tests__/shared/types.ts b/frontend/src/lib/components/copilot/chat/__tests__/shared/types.ts index 021e776440..61f7f1fd1f 100644 --- a/frontend/src/lib/components/copilot/chat/__tests__/shared/types.ts +++ b/frontend/src/lib/components/copilot/chat/__tests__/shared/types.ts @@ -1,4 +1,5 @@ import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import type { AIProvider } from '$lib/gen/types.gen' /** * Token usage tracking for LLM calls. @@ -83,6 +84,8 @@ export interface EvalRunnerOptions { model?: string /** Workspace ID for tool calls */ workspace?: string + /** AI provider (inferred from model name if omitted) */ + provider?: AIProvider } /** diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 03d0f363a0..ac45c175a6 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -1,4 +1,5 @@ import { OpenAI } from 'openai' +import Anthropic from '@anthropic-ai/sdk' import type { ChatCompletionMessageParam, ChatCompletionMessageFunctionToolCall @@ -13,19 +14,28 @@ import type { RawMessageStreamEvent } from '@anthropic-ai/sdk/resources' import type { MessageStream } from '@anthropic-ai/sdk/lib/MessageStream' +import type { AIProviderModel } from '$lib/gen' import { getProviderAndCompletionConfig, workspaceAIClients } from '../lib' import { processToolCall, type Tool, type ToolCallbacks } from './shared' export async function getAnthropicCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - tools?: OpenAI.Chat.Completions.ChatCompletionFunctionTool[] + tools?: OpenAI.Chat.Completions.ChatCompletionFunctionTool[], + options?: { + forceModelProvider?: AIProviderModel + anthropicClient?: Anthropic + } ): Promise { - const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true }) + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: true, + forceModelProvider: options?.forceModelProvider + }) const { system, messages: anthropicMessages } = convertOpenAIToAnthropicMessages(messages) const anthropicTools = convertOpenAIToolsToAnthropic(tools) - const anthropicClient = workspaceAIClients.getAnthropicClient() + const client = options?.anthropicClient ?? workspaceAIClients.getAnthropicClient() const anthropicParams = { model: config.model, @@ -36,7 +46,7 @@ export async function getAnthropicCompletion( ...(typeof config.temperature === 'number' && { temperature: config.temperature }) } - const stream = anthropicClient.messages.stream(anthropicParams, { + const stream = client.messages.stream(anthropicParams, { signal: abortController.signal, headers: { 'X-Provider': provider, @@ -58,7 +68,8 @@ export async function parseAnthropicCompletion( addedMessages: ChatCompletionMessageParam[], tools: Tool[], helpers: any, - abortController?: AbortController + abortController?: AbortController, + options?: { workspace?: string } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error = null @@ -209,7 +220,8 @@ export async function parseAnthropicCompletion( tools, toolCall, helpers, - toolCallbacks: callbacks + toolCallbacks: callbacks, + workspace: options?.workspace }) messages.push(messageToAdd) addedMessages.push(messageToAdd) diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts new file mode 100644 index 0000000000..4b239e4a05 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -0,0 +1,211 @@ +import OpenAI from 'openai' +import Anthropic from '@anthropic-ai/sdk' +import type { + ChatCompletionMessageParam, + ChatCompletionSystemMessageParam, + ChatCompletionUserMessageParam +} from 'openai/resources/chat/completions.mjs' +import type { AIProviderModel } from '$lib/gen' +import { getCompletion, parseOpenAICompletion } from '../lib' +import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' +import { + getOpenAIResponsesCompletion, + parseOpenAIResponsesCompletion +} from './openai-responses' +import type { Tool, ToolCallbacks } from './shared' + +export interface ChatClients { + openai: OpenAI + anthropic: Anthropic +} + +export interface ChatLoopConfig { + messages: ChatCompletionMessageParam[] + /** + * System message, tools, helpers, and modelProvider are re-read from this config + * on every iteration. Callers can use JS getters to provide dynamic values + * (e.g. AIChatManager uses getters so mode changes mid-loop take effect). + */ + systemMessage: ChatCompletionSystemMessageParam + tools: Tool[] + helpers: any + abortController: AbortController + callbacks: ToolCallbacks & { + onNewToken: (token: string) => void + onMessageEnd: () => void + } + modelProvider: AIProviderModel + clients: ChatClients + workspace: string + /** Maximum iterations for the loop. undefined = unlimited (production). */ + maxIterations?: number + skipResponsesApi?: boolean + onSkipResponsesApi?: () => void + /** Return a pending user message to inject between iterations, or undefined. */ + getPendingUserMessage?: () => ChatCompletionUserMessageParam | undefined + /** Called before each iteration (e.g. to refresh tool schemas). */ + onBeforeIteration?: (tools: Tool[], helpers: any) => Promise +} + +export interface ChatLoopResult { + addedMessages: ChatCompletionMessageParam[] +} + +export async function runChatLoop(config: ChatLoopConfig): Promise { + const { + messages, + abortController, + callbacks, + clients, + workspace, + maxIterations, + onSkipResponsesApi, + getPendingUserMessage, + onBeforeIteration + } = config + let skipResponsesApi = config.skipResponsesApi ?? false + + const addedMessages: ChatCompletionMessageParam[] = [] + let iterations = 0 + + while (true) { + if (maxIterations !== undefined && iterations >= maxIterations) { + break + } + iterations++ + + // Re-read these from config each iteration so that mode changes + // (e.g. changeModeTool in Navigator) take effect immediately. + // Callers can use JS getter properties to provide dynamic values. + const tools = config.tools + const helpers = config.helpers + const systemMessage = config.systemMessage + const modelProvider = config.modelProvider + + if (onBeforeIteration) { + await onBeforeIteration(tools, helpers) + } + + const pendingUserMessage = getPendingUserMessage?.() + + const isOpenAI = + modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai' + const isAnthropic = modelProvider.provider === 'anthropic' + + const messageParams = [ + systemMessage, + ...messages, + ...(pendingUserMessage ? [pendingUserMessage] : []) + ] + const toolDefs = tools.map((t) => t.def) + const parseOptions = { workspace } + + if (isOpenAI) { + let useCompletionsApi = skipResponsesApi + if (!skipResponsesApi) { + try { + const completion = await getOpenAIResponsesCompletion( + messageParams, + abortController, + toolDefs, + { + forceModelProvider: modelProvider, + openaiClient: clients.openai + } + ) + const continueCompletion = await parseOpenAIResponsesCompletion( + completion, + callbacks, + messages, + addedMessages, + tools, + helpers, + parseOptions + ) + if (!continueCompletion) { + break + } + } catch (err) { + console.warn( + 'OpenAI Responses API failed, falling back to Completions API:', + err + ) + const errorMessage = err instanceof Error ? err.message : String(err) + if (errorMessage.includes('Responses API is not enabled')) { + skipResponsesApi = true + onSkipResponsesApi?.() + } + useCompletionsApi = true + } + } + + if (useCompletionsApi) { + const completion = await getCompletion(messageParams, abortController, toolDefs, { + forceCompletions: true, + forceModelProvider: modelProvider, + openaiClient: clients.openai + }) + const continueCompletion = await parseOpenAICompletion( + completion, + callbacks, + messages, + addedMessages, + tools, + helpers, + undefined, + parseOptions + ) + if (!continueCompletion) { + break + } + } + } else if (isAnthropic) { + const completion = await getAnthropicCompletion( + messageParams, + abortController, + toolDefs, + { + forceModelProvider: modelProvider, + anthropicClient: clients.anthropic + } + ) + if (completion) { + const continueCompletion = await parseAnthropicCompletion( + completion, + callbacks, + messages, + addedMessages, + tools, + helpers, + abortController, + parseOptions + ) + if (!continueCompletion) { + break + } + } + } else { + const completion = await getCompletion(messageParams, abortController, toolDefs, { + forceModelProvider: modelProvider, + openaiClient: clients.openai + }) + if (completion) { + const continueCompletion = await parseOpenAICompletion( + completion, + callbacks, + messages, + addedMessages, + tools, + helpers, + undefined, + parseOptions + ) + if (!continueCompletion) { + break + } + } + } + } + + return { addedMessages } +} diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 5003f48099..56364e1401 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -125,15 +125,24 @@ function convertCompletionConfigToResponsesConfig( export async function getOpenAIResponsesCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] + tools?: OpenAI.Chat.Completions.ChatCompletionTool[], + options?: { + forceModelProvider?: AIProviderModel + openaiClient?: OpenAI + } ) { - const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: true, + tools, + forceModelProvider: options?.forceModelProvider + }) const { instructions, input } = convertMessagesToResponsesInput(messages) const responsesConfig = convertCompletionConfigToResponsesConfig(config) - const openaiClient = workspaceAIClients.getOpenaiClient() + const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() - const runner = openaiClient.responses.stream( + const runner = client.responses.stream( { ...responsesConfig, input, @@ -208,7 +217,8 @@ export async function parseOpenAIResponsesCompletion( messages: ChatCompletionMessageParam[], addedMessages: ChatCompletionMessageParam[], tools: Tool[], - helpers: any + helpers: any, + options?: { workspace?: string } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error: OpenAIError | ResponseErrorEvent | null = null @@ -342,7 +352,8 @@ export async function parseOpenAIResponsesCompletion( tools, toolCall, helpers, - toolCallbacks: callbacks + toolCallbacks: callbacks, + workspace: options?.workspace }) messages.push(messageToAdd) addedMessages.push(messageToAdd) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 4a95912b47..20e488d923 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -417,12 +417,14 @@ export async function processToolCall({ tools, toolCall, helpers, - toolCallbacks + toolCallbacks, + workspace }: { tools: Tool[] toolCall: ChatCompletionMessageFunctionToolCall helpers: T toolCallbacks: ToolCallbacks + workspace?: string }): Promise { try { const args = JSON.parse(toolCall.function.arguments || '{}') @@ -472,7 +474,7 @@ export async function processToolCall({ tools, functionName: toolCall.function.name, args, - workspace: get(workspaceStore) ?? '', + workspace: workspace ?? get(workspaceStore) ?? '', helpers, toolCallbacks, toolId: toolCall.id diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index dc05e3a247..d8149086d4 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -296,7 +296,12 @@ function getModelSpecificConfig( ) { const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) const modelKey = `${modelProvider.provider}:${modelProvider.model}` - const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel + let customMaxTokensStore: Record | undefined + try { + customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel + } catch { + // copilotInfo store may not be initialized in vitest + } const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && @@ -876,9 +881,16 @@ export async function getCompletion( tools?: OpenAI.Chat.Completions.ChatCompletionTool[], options?: { forceCompletions?: boolean + forceModelProvider?: AIProviderModel + openaiClient?: OpenAI } ): Promise> { - const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: true, + tools, + forceModelProvider: options?.forceModelProvider + }) // Use Responses API for OpenAI and Azure OpenAI if ((provider === 'openai' || provider === 'azure_openai') && !options?.forceCompletions) { @@ -891,8 +903,8 @@ export async function getCompletion( } // Use Completions API for other providers - const openaiClient = workspaceAIClients.getOpenaiClient() - const completion = openaiClient.chat.completions.create(config, { + const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() + const completion = client.chat.completions.create(config, { signal: abortController.signal, headers: { 'X-Provider': provider @@ -921,7 +933,8 @@ export async function parseOpenAICompletion( addedMessages: ChatCompletionMessageParam[], tools: Tool[], helpers: any, - _abortController?: AbortController // unused, for signature compatibility with parseAnthropicCompletion + _abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion + options?: { workspace?: string } ): Promise { const finalToolCalls: Record = {} let malformedFunctionCallError = false @@ -1060,7 +1073,8 @@ export async function parseOpenAICompletion( tools, toolCall, helpers, - toolCallbacks: callbacks + toolCallbacks: callbacks, + workspace: options?.workspace }) messages.push(messageToAdd) addedMessages.push(messageToAdd) From 34e3115bcbd19a8e0b6f483435586a2ab43d0a8e Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Wed, 25 Mar 2026 11:59:48 +0100 Subject: [PATCH 37/48] fix: raw apps bundle not found during deployment error (#8515) --- backend/Cargo.lock | 1 + backend/windmill-api-workspaces/Cargo.toml | 2 + .../windmill-api-workspaces/src/workspaces.rs | 65 +++++++++++++++++++ backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/src/apps.rs | 26 ++++++-- 5 files changed, 88 insertions(+), 8 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a5e2e8fc98..34532bc173 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16433,6 +16433,7 @@ dependencies = [ "windmill-common", "windmill-dep-map", "windmill-git-sync", + "windmill-object-store", "windmill-queue", "windmill-types", ] diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index a03bb3a490..86f0649c73 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -14,9 +14,11 @@ enterprise = ["windmill-common/enterprise"] private = ["windmill-common/private"] cloud = ["windmill-common/cloud"] no_auth = ["windmill-api-auth/no_auth"] +parquet = ["windmill-object-store/parquet"] [dependencies] windmill-common = { workspace = true, default-features = false } +windmill-object-store = { workspace = true, optional = true } windmill-types.workspace = true windmill-api-auth.workspace = true windmill-api-users.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index fb234a13a3..4e832b3a93 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -3474,6 +3474,11 @@ async fn clone_apps( .fetch_all(&mut **tx) .await?; + let mut cloned_from_db: std::collections::HashSet<(i64, String)> = HashSet::new(); + for bundle in &bundles { + cloned_from_db.insert((bundle.app_version_id, bundle.file_type.clone())); + } + for bundle in bundles { if let Some(&new_version_id) = version_id_mapping.get(&bundle.app_version_id) { sqlx::query!( @@ -3488,6 +3493,66 @@ async fn clone_apps( .await?; } } + + // Clone bundles from S3 for versions not found in DB + #[cfg(all(feature = "enterprise", feature = "parquet"))] + { + let object_store = windmill_object_store::get_object_store().await; + if let Some(os) = object_store { + for (&old_version_id, &new_version_id) in &version_id_mapping { + for file_type in &["js", "css"] { + if cloned_from_db.contains(&(old_version_id, file_type.to_string())) { + continue; + } + let src_path = format!( + "/app_bundles/{}/{}.{}", + source_workspace_id, old_version_id, file_type + ); + let get_result = os + .get(&windmill_object_store::object_store_reexports::Path::from( + src_path, + )) + .await; + match get_result { + Ok(result) => { + let data = result.bytes().await.map_err( + windmill_object_store::object_store_error_to_error, + )?; + let dst_path = format!( + "/app_bundles/{}/{}.{}", + target_workspace_id, new_version_id, file_type + ); + os.put( + &windmill_object_store::object_store_reexports::Path::from( + dst_path.clone(), + ), + data.into(), + ) + .await + .map_err( + windmill_object_store::object_store_error_to_error, + )?; + tracing::info!( + "Cloned app bundle from S3: {}.{} -> {}.{}", + old_version_id, + file_type, + new_version_id, + file_type + ); + } + Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) => { + // No bundle in S3 for this version/type, skip + } + Err(e) => { + return Err( + windmill_object_store::object_store_error_to_error(e), + ); + } + } + } + } + } + } } // Update app versions arrays diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 1b09f37861..3f015513e2 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -18,7 +18,7 @@ agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["windmill-api-embeddings/embedding"] -parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] +parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "dep:aws-sigv4", "dep:aws-sdk-config"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus"] openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] tantivy = ["dep:windmill-indexer"] diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 33a632ecbd..eaead14558 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -440,17 +440,29 @@ async fn get_raw_app_data( #[cfg(all(feature = "enterprise", feature = "parquet"))] if let Some(os) = object_store { let path = format!("/app_bundles/{}/{}.{}", w_id, id, file_type); - let stream = os + match os .get(&windmill_object_store::object_store_reexports::Path::from( path, )) .await - .map_err(windmill_object_store::object_store_error_to_error)? - .bytes() - .await - .map_err(windmill_object_store::object_store_error_to_error)?; - tracing::info!("stream: {}", stream.len()); - body = Some(Body::from(stream)); + { + Ok(result) => { + let stream = result + .bytes() + .await + .map_err(windmill_object_store::object_store_error_to_error)?; + tracing::info!("stream: {}", stream.len()); + body = Some(Body::from(stream)); + } + Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }) => { + // S3 key not found, fall through to DB lookup below + } + Err(e) => { + return Err(windmill_object_store::object_store_error_to_error(e)); + } + } } if body.is_none() { From b7d14c8614f4da0da262bb20c0eb01854975cf65 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 11:20:39 +0000 Subject: [PATCH 38/48] regenerate sqlx offline query cache for integration tests (#8518) Co-authored-by: Claude Opus 4.5 --- ...aacf6af2c284ae446860113c82bc4e1da08ab.json | 12 ++++++++++ ...bc47caebc25215a430d6b301b35e265888159.json | 12 ++++++++++ ...fb7cf5f2b76f013c274245af13d7d727ebf1f.json | 12 ++++++++++ ...e2e60e3183fa81a411622891caea6dc03fa90.json | 15 +++++++++++++ ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...960ffc33da5f31bf780e8fd6a66d5150b8027.json | 12 ++++++++++ ...69c87a9d29370ec985d2c8c28633cd078ffaf.json | 12 ++++++++++ ...74da8c73120b3e16194904575f79a4e055002.json | 12 ++++++++++ ...437ab3e02d8c3c10c53decc664533b8d04bc0.json | 22 +++++++++++++++++++ 9 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json create mode 100644 backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json create mode 100644 backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json create mode 100644 backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json create mode 100644 backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json create mode 100644 backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json create mode 100644 backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json create mode 100644 backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json diff --git a/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json b/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json new file mode 100644 index 0000000000..0ad1fe4367 --- /dev/null +++ b/backend/.sqlx/query-01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version (id, app_id, value, created_by, created_at)\n VALUES (3001, 3001, '{\"grid\": []}', 'admin', NOW())", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "01c040b04b487e86b7f4ff38b0faacf6af2c284ae446860113c82bc4e1da08ab" +} diff --git a/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json b/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json new file mode 100644 index 0000000000..24d3c8929a --- /dev/null +++ b/backend/.sqlx/query-1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app SET versions = ARRAY[3001::bigint] WHERE id = 3001", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1ad384eeb3946ef7c492124c69fbc47caebc25215a430d6b301b35e265888159" +} diff --git a/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json b/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json new file mode 100644 index 0000000000..10cab9117a --- /dev/null +++ b/backend/.sqlx/query-1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema, extra_perms)\n VALUES ('test-workspace', 'u/operator/existing_flow', 'Existing flow', '', '{\"modules\": []}', 'admin', NOW(), '{}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "1fa27bd47ab66b3f301ca43ef93fb7cf5f2b76f013c274245af13d7d727ebf1f" +} diff --git a/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json b/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json new file mode 100644 index 0000000000..27d46b27ed --- /dev/null +++ b/backend/.sqlx/query-4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token (token_hash, token_prefix, email, label, super_admin, owner, workspace_id)\n VALUES ($1, $2, 'charlie@windmill.dev', 'Charlie new token', false, 'u/charlie', 'test-workspace')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "4e88aec662ebc70e0425a48a1b4e2e60e3183fa81a411622891caea6dc03fa90" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json b/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json new file mode 100644 index 0000000000..8e558fe67b --- /dev/null +++ b/backend/.sqlx/query-5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app (id, workspace_id, path, summary, versions, policy, extra_perms)\n VALUES (3001, 'test-workspace', 'u/operator/existing_app', 'Existing app', '{}',\n '{\"on_behalf_of\": \"u/admin\", \"on_behalf_of_email\": \"admin@windmill.dev\", \"execution_mode\": \"viewer\"}', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "5b140cb478aa32ac38fb831d6af960ffc33da5f31bf780e8fd6a66d5150b8027" +} diff --git a/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json b/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json new file mode 100644 index 0000000000..6da123cbc4 --- /dev/null +++ b/backend/.sqlx/query-6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr_to_group (workspace_id, group_, usr) VALUES ('test-workspace', 'editors', 'charlie')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "6c24b5c08341979e02b7968242369c87a9d29370ec985d2c8c28633cd078ffaf" +} diff --git a/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json b/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json new file mode 100644 index 0000000000..d7cc49fe3f --- /dev/null +++ b/backend/.sqlx/query-be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, extra_perms)\n VALUES ('test-workspace', 3001, 'u/operator/existing_script', 'export function main() { return \"original\"; }', 'deno', 'script', 'admin', '{}', 'Existing script', '', '', '{}')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "be35e85e94641c71620e5402dcf74da8c73120b3e16194904575f79a4e055002" +} diff --git a/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json b/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json new file mode 100644 index 0000000000..d9b7688eba --- /dev/null +++ b/backend/.sqlx/query-ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id,\n script_path, is_flow, workspace_id, edited_by, permissioned_as\n )\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ccd76be88fa9c11b3dc2e6d7711437ab3e02d8c3c10c53decc664533b8d04bc0" +} From 520706b640a1f9c8470d41f169b76d90db70a1e4 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 25 Mar 2026 13:01:03 +0100 Subject: [PATCH 39/48] chore: use workingdir in webmux panes (#8516) Co-authored-by: Claude Opus 4.5 --- .webmux.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.webmux.yaml b/.webmux.yaml index 6a765fbb12..c41d0aa699 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -55,11 +55,13 @@ profiles: - id: backend kind: command split: right - command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/backend" && cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}" + workingDir: backend + command: PORT=${BACKEND_PORT:-8000} cargo watch -x "run ${CARGO_FEATURES:+--features $CARGO_FEATURES}" - id: frontend kind: command split: bottom - command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0 + workingDir: frontend + command: npm run generate-backend-client && REMOTE=${REMOTE:-http://localhost:${BACKEND_PORT:-8000}} npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 frontendOnly: runtime: host @@ -82,7 +84,8 @@ profiles: - id: frontend kind: command split: right - command: ROOT="$(git rev-parse --show-toplevel)"; cd "$ROOT/frontend" && npm run generate-backend-client && npm run dev -- --host 0.0.0.0 + workingDir: frontend + command: npm run generate-backend-client && npm run dev -- --port ${FRONTEND_PORT:-3000} --host 0.0.0.0 agentOnly: runtime: host From 0904d7fffeb0cc4ad1627b60ebc0340cfad74bcf Mon Sep 17 00:00:00 2001 From: Samuel Wilk <34423885+da-wilky@users.noreply.github.com> Date: Wed, 25 Mar 2026 14:51:18 +0100 Subject: [PATCH 40/48] Add 'fast' query parameter to API definition (#8521) --- backend/windmill-api/openapi.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a8f83c368a..825842f057 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10644,6 +10644,10 @@ paths: in: query schema: type: boolean + - name: fast + in: query + schema: + type: boolean responses: "200": From 34cf0a0324627d4da6d4324ab662155045ddf327 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 13:51:34 +0000 Subject: [PATCH 41/48] show sync resource types button when resource type is missing (#8514) * feat: show sync resource types button when resource type is missing Co-Authored-By: Claude Opus 4.6 (1M context) * fix: show prominent error message when resource type is not found Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use sync_cached_resource_types endpoint instead of hub_sync script Co-Authored-By: Claude Opus 4.6 (1M context) * fix: fallback to fetching resource types from hub when cache file missing Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- backend/windmill-api-settings/src/lib.rs | 69 ++++++++++++--- .../src/lib/components/ApiConnectForm.svelte | 6 +- .../src/lib/components/AppConnectInner.svelte | 87 +++++++++---------- .../src/lib/components/ResourceEditor.svelte | 12 ++- .../lib/components/SyncResourceTypes.svelte | 41 +++++++++ 5 files changed, 153 insertions(+), 62 deletions(-) create mode 100644 frontend/src/lib/components/SyncResourceTypes.svelte diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index dbe80aca94..d1b52a8cc3 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -1124,6 +1124,59 @@ struct CachedResourceType { description: Option, } +#[derive(serde::Deserialize)] +struct HubResourceTypeRaw { + id: i64, + name: String, + schema: Option, + app: String, + description: Option, +} + +async fn fetch_resource_types_from_hub() -> error::Result> { + let response = HTTP_CLIENT + .get(format!( + "{}/resource_types/list", + windmill_common::DEFAULT_HUB_BASE_URL + )) + .header("Accept", "application/json") + .send() + .await + .map_err(|e| error::Error::InternalErr(format!("Failed to fetch from hub: {}", e)))?; + + if !response.status().is_success() { + return Err(error::Error::InternalErr(format!( + "Hub returned status {}", + response.status() + ))); + } + + let raw_types: Vec = response + .json() + .await + .map_err(|e| error::Error::InternalErr(format!("Failed to parse hub response: {}", e)))?; + + Ok(raw_types + .into_iter() + .filter_map(|rt| { + let schema = match rt.schema { + Some(s) => match serde_json::from_str(&s) { + Ok(v) => Some(v), + Err(_) => return None, + }, + None => None, + }; + Some(CachedResourceType { + id: rt.id, + name: rt.name, + schema, + app: rt.app, + description: rt.description, + }) + }) + .collect()) +} + async fn sync_cached_resource_types( Extension(db): Extension, authed: ApiAuthed, @@ -1133,16 +1186,12 @@ async fn sync_cached_resource_types( use windmill_common::worker::HUB_RT_CACHE_DIR; let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); - let content = tokio::fs::read_to_string(&cache_path).await.map_err(|e| { - error::Error::NotFound(format!( - "No cached resource types found at {}: {}", - cache_path, e - )) - })?; - - let cached_types: Vec = serde_json::from_str(&content).map_err(|e| { - error::Error::InternalErr(format!("Failed to parse cached resource types: {}", e)) - })?; + let cached_types = match tokio::fs::read_to_string(&cache_path).await { + Ok(content) => serde_json::from_str::>(&content).map_err(|e| { + error::Error::InternalErr(format!("Failed to parse cached resource types: {}", e)) + })?, + Err(_) => fetch_resource_types_from_hub().await?, + }; let mut synced_count = 0; diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index 636a20f290..7d10db1d48 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -16,6 +16,7 @@ import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' import ResourceGen from './copilot/ResourceGen.svelte' + import SyncResourceTypes from './SyncResourceTypes.svelte' interface Props { resourceType: string @@ -25,6 +26,7 @@ isValid?: boolean linkedSecretCandidates?: string[] | undefined description?: string | undefined + onSynced?: () => void } let { @@ -34,7 +36,8 @@ linkedSecrets = $bindable([]), isValid = $bindable(true), linkedSecretCandidates = undefined, - description = $bindable(undefined) + description = $bindable(undefined), + onSynced = undefined }: Props = $props() let schema = $state(emptySchema()) @@ -227,6 +230,7 @@ >No corresponding resource type found in your workspace for {resourceType}. Define the value in JSON directly

    + {/if} {#if notFound || viewJsonSchema} {#if !emptyString(error)} import { run } from 'svelte/legacy' - import { superadmin, userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore } from '$lib/stores' import IconedResourceType from './IconedResourceType.svelte' import { OauthService, ResourceService, VariableService, type TokenResponse, - type ResourceType, - JobService + type ResourceType } from '$lib/gen' import { emptyString, truncateRev, urlize } from '$lib/utils' import { createEventDispatcher, onDestroy } from 'svelte' @@ -31,9 +30,8 @@ import type { SchemaProperty } from '$lib/common' import Tooltip from './Tooltip.svelte' import TextInput from './text_input/TextInput.svelte' - import { usePromise } from '$lib/svelte5Utils.svelte' - import { pollJobResult } from './jobs/utils' import { sameTopDomainOrigin } from '$lib/cookies' + import SyncResourceTypes from './SyncResourceTypes.svelte' interface Props { step?: number @@ -135,6 +133,7 @@ let tokenUrl = $state('') let resourceTypeInfo: ResourceType | undefined = $state(undefined) + let resourceTypeNotFound = $state(false) let pathError = $state('') @@ -319,18 +318,24 @@ } async function getResourceTypeInfo() { - resourceTypeInfo = await ResourceService.getResourceType({ - workspace: effectiveWorkspace, - path: resourceType - }) - const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} - const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] + try { + resourceTypeNotFound = false + resourceTypeInfo = await ResourceService.getResourceType({ + workspace: effectiveWorkspace, + path: resourceType + }) + const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} + const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] - const passwords = newArgsKeys.filter((x) => { - return props?.[x]?.password - }) - if (linkedSecrets.length === 0) { - linkedSecrets = computeDefaultLinkedSecrets(resourceType, newArgsKeys, passwords) + const passwords = newArgsKeys.filter((x) => { + return props?.[x]?.password + }) + if (linkedSecrets.length === 0) { + linkedSecrets = computeDefaultLinkedSecrets(resourceType, newArgsKeys, passwords) + } + } catch (err) { + resourceTypeInfo = undefined + resourceTypeNotFound = true } } export async function next() { @@ -589,23 +594,6 @@ let filteredConnectsManual: { key: string; img?: string; instructions: string[] }[] = $state([]) let editScopes = $state(false) - - let hubRtSync = usePromise( - async () => { - let jobUuid = await JobService.runScriptByPath({ - workspace: 'admins', - path: 'u/admin/hub_sync', - requestBody: {} - }) - await pollJobResult(jobUuid, 'admins') - connectsManual = undefined - await loadResourceTypes() - connects = undefined - await loadConnects() - sendUserToast('Hub resource types sync completed') - }, - { loadInit: false } - ) {#if !express} @@ -722,20 +710,16 @@ {/each} {/if}
    - {#if $superadmin} - - {#if hubRtSync.status === 'error'} - - Error syncing resource types : {JSON.stringify(hubRtSync.error)} - - {/if} - {/if} +
    + { + connectsManual = undefined + await loadResourceTypes() + connects = undefined + await loadConnects() + }} + /> +
    {:else if step == 2 && manual}
    + {#if resourceTypeNotFound} +
    +

    + Resource type '{resourceType}' not found in your workspace +

    + +
    + {/if} {#key resourceTypeInfo} {/key}
    diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 3956cea7eb..416068bf8b 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -25,6 +25,7 @@ import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' import ResourceGen from './copilot/ResourceGen.svelte' + import SyncResourceTypes from './SyncResourceTypes.svelte' interface Props { canSave?: boolean @@ -347,10 +348,13 @@ {:else} {#if !viewJsonSchema} -

    - No corresponding resource type found in your workspace for {resource_type}. Define the - value in JSON directly -

    +
    +

    + Resource type '{resource_type}' not found in your workspace +

    + +

    Define the value in JSON directly

    +
    {/if} {#if !emptyString(jsonError)} + import { superadmin } from '$lib/stores' + import { usePromise } from '$lib/svelte5Utils.svelte' + import { sendUserToast } from '$lib/toast' + import Button from './common/button/Button.svelte' + + interface Props { + onSynced?: () => void + } + + let { onSynced = undefined }: Props = $props() + + let hubRtSync = usePromise( + async () => { + const res = await fetch('/api/settings/sync_cached_resource_types', { method: 'POST' }) + if (!res.ok) { + const body = await res.text() + throw new Error(body || res.statusText) + } + sendUserToast('Hub resource types sync completed') + onSynced?.() + }, + { loadInit: false } + ) + + +{#if $superadmin} + + {#if hubRtSync.status === 'error'} + + Error syncing resource types: {hubRtSync.error?.message ?? JSON.stringify(hubRtSync.error)} + + {/if} +{/if} From 031766808945aefc926f0836d011c0b2a5d2243d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 14:33:20 +0000 Subject: [PATCH 42/48] fix: require admin for workspace encryption key export (#8523) Move the require_admin check from blocking the entire tarball export to only guarding the include_key=true path. Non-admins can still export tarballs for workspace sync/git, but only admins can export the raw workspace encryption key. Co-authored-by: Claude Opus 4.6 (1M context) --- backend/windmill-api/src/workspaces_export.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index d685d4c4cc..6c6c783ba7 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -388,8 +388,6 @@ pub(crate) async fn tarball_workspace( settings_version, }): Query, ) -> Result<([(HeaderName, String); 2], impl IntoResponse)> { - // require_admin(authed.is_admin, &authed.username)?; - tracing::info!( "tarball_workspace called for workspace {}: include_workspace_dependencies={:?}, skip_variables={:?}, skip_resources={:?}", w_id, @@ -1078,6 +1076,8 @@ pub(crate) async fn tarball_workspace( } if include_key.unwrap_or(false) { + require_admin(authed.is_admin, &authed.username)?; + let key = sqlx::query_scalar!( "SELECT key FROM workspace_key WHERE workspace_id = $1", &w_id From 8a32322c187ccc60ec7eafb61a9678f267a82282 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 25 Mar 2026 15:38:29 +0100 Subject: [PATCH 43/48] fix: auto-generate datatable SDK reference for app mode system prompt (#8522) The app mode AI chat system prompt had hand-written datatable API docs that were missing methods (fetchOneScalar, execute, query). This adds datatable-specific extraction to generate.py so the prompt stays in sync with the actual TypeScript and Python client APIs. Co-authored-by: Claude Opus 4.6 (1M context) --- .../lib/components/copilot/chat/app/core.ts | 17 +- system_prompts/auto-generated/index.d.ts | 1 + system_prompts/auto-generated/index.ts | 8 + system_prompts/auto-generated/prompts.ts | 148 ++++++++++++ .../auto-generated/sdks/datatable-python.md | 68 ++++++ .../sdks/datatable-typescript.md | 76 +++++++ system_prompts/generate.py | 215 ++++++++++++++++++ 7 files changed, 521 insertions(+), 12 deletions(-) create mode 100644 system_prompts/auto-generated/sdks/datatable-python.md create mode 100644 system_prompts/auto-generated/sdks/datatable-typescript.md diff --git a/frontend/src/lib/components/copilot/chat/app/core.ts b/frontend/src/lib/components/copilot/chat/app/core.ts index 8fc88d4d16..c6a64a3e5a 100644 --- a/frontend/src/lib/components/copilot/chat/app/core.ts +++ b/frontend/src/lib/components/copilot/chat/app/core.ts @@ -10,6 +10,7 @@ import { createGetRunnableDetailsTool, type Tool } from '../shared' +import { getDatatableSdkReference } from '$system_prompts' import { aiChatManager } from '../AIChatManager.svelte' import type { ContextElement, @@ -842,38 +843,30 @@ For inline scripts, the code must have a \`main\` function as its entrypoint. Backend runnables should only perform **data operations** (SELECT, INSERT, UPDATE, DELETE) on **existing tables**. Never use CREATE TABLE, DROP TABLE, or ALTER TABLE inside runnables. -**TypeScript (Bun)**: +**TypeScript (Bun) example**: \`\`\`typescript import * as wmill from 'windmill-client'; export async function main(user_id: string) { const sql = ${datatableCall}; - - // Safe string interpolation (parameterized query) const user = await sql\`SELECT * FROM ${schemaPrefix}users WHERE id = \${user_id}\`.fetchOne(); return user; } \`\`\` -**Python**: +**Python example**: \`\`\`python import wmill def main(user_id: str): db = ${datatableCall} - - # Use positional arguments ($1, $2, etc.) user = db.query('SELECT * FROM ${schemaPrefix}users WHERE id = $1', user_id).fetch_one() return user \`\`\` -### Common Operations (for use in backend runnables) +### Datatable Client API Reference -- **Fetch all**: \`sql\`SELECT * FROM ${schemaPrefix}table\`.fetch()\` or \`db.query('SELECT * FROM ${schemaPrefix}table').fetch()\` -- **Fetch one**: \`.fetchOne()\` or \`.fetch_one()\` -- **Insert**: \`sql\`INSERT INTO ${schemaPrefix}table (col) VALUES (\${value})\`\` -- **Update**: \`sql\`UPDATE ${schemaPrefix}table SET col = \${value} WHERE id = \${id}\`\` -- **Delete**: \`sql\`DELETE FROM ${schemaPrefix}table WHERE id = \${id}\`\` +${getDatatableSdkReference()} ### Schema Modifications (DDL) - Use exec_datatable_sql tool ONLY diff --git a/system_prompts/auto-generated/index.d.ts b/system_prompts/auto-generated/index.d.ts index ab649a2090..4f2dd469ac 100644 --- a/system_prompts/auto-generated/index.d.ts +++ b/system_prompts/auto-generated/index.d.ts @@ -1,3 +1,4 @@ export * from './prompts'; export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; +export declare function getDatatableSdkReference(): string; diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index b283892470..b2c4bf0ec8 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -37,3 +37,11 @@ export function getFlowPrompt(): string { prompts.OPENFLOW_SCHEMA ].filter(Boolean).join('\n\n'); } + +// Helper to get datatable SDK reference for app mode +export function getDatatableSdkReference(): string { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\n\n'); +} diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index d30a5be3eb..6eb870e62b 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1371,6 +1371,154 @@ async def parallel(items, fn, concurrency: Optional[int] = None) # offset: Message offset to commit (from event['offset']) def commit_kafka_offsets(trigger_path: str, topic: str, partition: int, offset: int) -> None +`; + +export const DATATABLE_SDK_TYPESCRIPT = `## TypeScript Datatable API (windmill-client) + +Import: \`import * as wmill from 'windmill-client'\` + +SQL statement object with query content, arguments, and execution methods +\`\`\`typescript +type SqlStatement = { + /** Raw SQL content with formatted arguments */ + content: string; + + /** Argument values keyed by parameter name */ + args: Record; + + /** + * Execute the SQL query and return results + * @param params - Optional parameters including result collection mode + * @returns Query results based on the result collection mode + */ + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + /** + * Execute the SQL query and return only the first row + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit, "resultCollection"> + ): Promise; +}; +\`\`\` + +\`\`\`typescript +// Template tag function: sql\`SELECT * FROM table WHERE id = \${id}\`.fetch() +interface DatatableSqlTemplateFunction { + // Tagged template usage: + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + query(sql: string, ...params: any[]): SqlStatement; +}; +\`\`\` + +Create a SQL template function for PostgreSQL/datatable queries +@param name - Database/datatable name (default: "main") +@returns SQL template function for building parameterized queries +@example +let sql = wmill.datatable() +let name = 'Robin' +let age = 21 +await sql\` + SELECT * FROM friends + WHERE name = \${name} AND age = \${age}::int +\`.fetch() +\`\`\`typescript +function datatable(name: string = "main"): DatatableSqlTemplateFunction +\`\`\` +`; + +export const DATATABLE_SDK_PYTHON = `## Python Datatable API (wmill) + +Import: \`import wmill\` + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') -> DataTableClient + +# Client for executing SQL queries against Windmill DataTables. +class DataTableClient: + # Initialize DataTableClient. + # + # Args: + # client: Windmill client instance + # name: DataTable name + def __init__(client: Windmill, name: str) + + # Execute a SQL query against the DataTable. + # + # Args: + # sql: SQL query string with $1, $2, etc. placeholders + # *args: Positional arguments to bind to query placeholders + # + # Returns: + # SqlQuery instance for fetching results + def query(sql: str, *args) -> SqlQuery + + +# Query result handler for DataTable and DuckLake queries. +class SqlQuery: + # Initialize SqlQuery. + # + # Args: + # sql: SQL query string + # fetch_fn: Function to execute the query + def __init__(sql: str, fetch_fn) + + # Execute query and fetch results. + # + # Args: + # result_collection: Optional result collection mode + # + # Returns: + # Query results + def fetch(result_collection: str | None = None) + + # Execute query and fetch first row of results. + # + # Returns: + # First row of query results + def fetch_one() + + # Execute query and fetch first row of results. Return result as a scalar value. + # + # Returns: + # First row of query result as a scalar value + def fetch_one_scalar() + + # Execute query and don't return any results. + # + def execute() + + `; export const OPENFLOW_SCHEMA = `## OpenFlow Schema diff --git a/system_prompts/auto-generated/sdks/datatable-python.md b/system_prompts/auto-generated/sdks/datatable-python.md new file mode 100644 index 0000000000..752019a9a3 --- /dev/null +++ b/system_prompts/auto-generated/sdks/datatable-python.md @@ -0,0 +1,68 @@ +## Python Datatable API (wmill) + +Import: `import wmill` + +# Get a DataTable client for SQL queries. +# +# Args: +# name: Database name (default: "main") +# +# Returns: +# DataTableClient instance +def datatable(name: str = 'main') -> DataTableClient + +# Client for executing SQL queries against Windmill DataTables. +class DataTableClient: + # Initialize DataTableClient. + # + # Args: + # client: Windmill client instance + # name: DataTable name + def __init__(client: Windmill, name: str) + + # Execute a SQL query against the DataTable. + # + # Args: + # sql: SQL query string with $1, $2, etc. placeholders + # *args: Positional arguments to bind to query placeholders + # + # Returns: + # SqlQuery instance for fetching results + def query(sql: str, *args) -> SqlQuery + + +# Query result handler for DataTable and DuckLake queries. +class SqlQuery: + # Initialize SqlQuery. + # + # Args: + # sql: SQL query string + # fetch_fn: Function to execute the query + def __init__(sql: str, fetch_fn) + + # Execute query and fetch results. + # + # Args: + # result_collection: Optional result collection mode + # + # Returns: + # Query results + def fetch(result_collection: str | None = None) + + # Execute query and fetch first row of results. + # + # Returns: + # First row of query results + def fetch_one() + + # Execute query and fetch first row of results. Return result as a scalar value. + # + # Returns: + # First row of query result as a scalar value + def fetch_one_scalar() + + # Execute query and don't return any results. + # + def execute() + + diff --git a/system_prompts/auto-generated/sdks/datatable-typescript.md b/system_prompts/auto-generated/sdks/datatable-typescript.md new file mode 100644 index 0000000000..0256515911 --- /dev/null +++ b/system_prompts/auto-generated/sdks/datatable-typescript.md @@ -0,0 +1,76 @@ +## TypeScript Datatable API (windmill-client) + +Import: `import * as wmill from 'windmill-client'` + +SQL statement object with query content, arguments, and execution methods +```typescript +type SqlStatement = { + /** Raw SQL content with formatted arguments */ + content: string; + + /** Argument values keyed by parameter name */ + args: Record; + + /** + * Execute the SQL query and return results + * @param params - Optional parameters including result collection mode + * @returns Query results based on the result collection mode + */ + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + /** + * Execute the SQL query and return only the first row + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; + + /** + * Execute the SQL query and return only the first row as a scalar value + * @param params - Optional parameters + * @returns First row of the query result + */ + fetchOneScalar( + params?: Omit< + FetchParams<"last_statement_first_row_scalar">, + "resultCollection" + > + ): Promise>; + + /** + * Execute the SQL query without fetching rows + * @param params - Optional parameters + */ + execute( + params?: Omit, "resultCollection"> + ): Promise; +}; +``` + +```typescript +// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch() +interface DatatableSqlTemplateFunction { + // Tagged template usage: + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; + query(sql: string, ...params: any[]): SqlStatement; +}; +``` + +Create a SQL template function for PostgreSQL/datatable queries +@param name - Database/datatable name (default: "main") +@returns SQL template function for building parameterized queries +@example +let sql = wmill.datatable() +let name = 'Robin' +let age = 21 +await sql` + SELECT * FROM friends + WHERE name = ${name} AND age = ${age}::int +`.fetch() +```typescript +function datatable(name: string = "main"): DatatableSqlTemplateFunction +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index cf57a3cecb..034c94e20d 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -655,6 +655,202 @@ def generate_schema_files(cli_schemas: dict[str, dict]) -> dict[str, str]: return schema_yaml_content +# ============================================================================= +# Datatable SDK Extraction +# ============================================================================= + + +TS_SQL_UTILS_PATH = TS_SDK_DIR / "sqlUtils.ts" + + +def extract_datatable_ts_sdk() -> str: + """Extract datatable-specific type definitions from TypeScript SDK (sqlUtils.ts). + + Reads the source file and extracts the public API surface: + - SqlStatement type (fetch, fetchOne, fetchOneScalar, execute methods) + - DatatableSqlTemplateFunction interface (template tag + query method) + - datatable() function signature + """ + if not TS_SQL_UTILS_PATH.exists(): + print(f" Warning: sqlUtils.ts not found at {TS_SQL_UTILS_PATH}") + return '' + + content = TS_SQL_UTILS_PATH.read_text() + + md = "## TypeScript Datatable API (windmill-client)\n\n" + md += "Import: `import * as wmill from 'windmill-client'`\n\n" + + # Extract exported type/interface/function definitions from sqlUtils.ts + # We use extract_balanced to handle nested braces correctly + + # 1. Extract SqlStatement type + match = re.search(r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+type\s+SqlStatement\s*=\s*', content) + if match: + jsdoc_raw = match.group(1) + brace_start = content.index('{', match.end() - 1) + body, end = extract_balanced(content, brace_start, '{', '}') + if end != -1: + if jsdoc_raw: + md += clean_jsdoc(jsdoc_raw) + "\n" + md += "```typescript\n" + md += f"type SqlStatement = {{\n{_indent_body(body)}\n}};\n" + md += "```\n\n" + + # 2. Extract DatatableSqlTemplateFunction interface + match = re.search( + r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+interface\s+DatatableSqlTemplateFunction\s+extends\s+SqlTemplateFunction\s*', + content + ) + if match: + brace_start = content.index('{', match.end() - 1) + body, end = extract_balanced(content, brace_start, '{', '}') + if end != -1: + md += "```typescript\n" + md += "// Template tag function: sql`SELECT * FROM table WHERE id = ${id}`.fetch()\n" + md += f"interface DatatableSqlTemplateFunction {{\n" + md += f" // Tagged template usage:\n" + md += f" (strings: TemplateStringsArray, ...values: any[]): SqlStatement;\n" + md += f"{_indent_body(body)}\n" + md += "};\n" + md += "```\n\n" + + # 3. Extract datatable() function + match = re.search( + r'(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?export\s+function\s+datatable\s*\(([^)]*)\)\s*:\s*(\S+)', + content + ) + if match: + jsdoc_raw, params, return_type = match.groups() + if jsdoc_raw: + md += clean_jsdoc(jsdoc_raw) + "\n" + md += "```typescript\n" + md += f"function datatable({params.strip()}): {return_type}\n" + md += "```\n" + + return md + + +def extract_datatable_py_sdk(py_content: str) -> str: + """Extract datatable-specific class/function definitions from Python SDK. + + Uses Python AST to extract: + - datatable() function + - DataTableClient class with query() method + - SqlQuery class with fetch(), fetch_one(), fetch_one_scalar(), execute() methods + """ + if not py_content: + return '' + + try: + tree = ast.parse(py_content) + except SyntaxError as e: + print(f" Warning: Could not parse Python SDK for datatable extraction: {e}") + return '' + + md = "## Python Datatable API (wmill)\n\n" + md += "Import: `import wmill`\n\n" + + # Target classes and the top-level datatable function + target_classes = {'DataTableClient', 'SqlQuery'} + + # 1. Extract datatable() top-level function + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == 'datatable': + docstring = ast.get_docstring(node) or '' + params = _format_py_params(node) + return_ann = f" -> {ast.unparse(node.returns)}" if node.returns else '' + if docstring: + for line in docstring.split('\n'): + md += f"# {line}\n" + md += f"def datatable({params}){return_ann}\n\n" + break + + # 2. Extract target classes with their public methods + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name in target_classes: + class_doc = ast.get_docstring(node) or '' + if class_doc: + for line in class_doc.split('\n'): + md += f"# {line}\n" + md += f"class {node.name}:\n" + + for item in node.body: + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + if item.name.startswith('_') and item.name != '__init__': + continue + docstring = ast.get_docstring(item) or '' + params = _format_py_params(item, skip_self=True) + return_ann = f" -> {ast.unparse(item.returns)}" if item.returns else '' + async_prefix = 'async ' if isinstance(item, ast.AsyncFunctionDef) else '' + if docstring: + for line in docstring.split('\n'): + md += f" # {line}\n" + md += f" {async_prefix}def {item.name}({params}){return_ann}\n\n" + + md += "\n" + + return md + + +def _format_py_params(node: ast.FunctionDef, skip_self: bool = False) -> str: + """Format function parameters from AST node.""" + params = [] + args = node.args + num_defaults = len(args.defaults) + num_args = len(args.args) + + for i, arg in enumerate(args.args): + if skip_self and arg.arg == 'self': + continue + param_str = arg.arg + if arg.annotation: + param_str += f": {ast.unparse(arg.annotation)}" + default_idx = i - (num_args - num_defaults) + if default_idx >= 0: + default = args.defaults[default_idx] + param_str += f" = {ast.unparse(default)}" + params.append(param_str) + + if args.vararg: + vararg_str = f"*{args.vararg.arg}" + if args.vararg.annotation: + vararg_str += f": {ast.unparse(args.vararg.annotation)}" + params.append(vararg_str) + + for i, arg in enumerate(args.kwonlyargs): + param_str = arg.arg + if arg.annotation: + param_str += f": {ast.unparse(arg.annotation)}" + if args.kw_defaults[i]: + param_str += f" = {ast.unparse(args.kw_defaults[i])}" + params.append(param_str) + + if args.kwarg: + kwarg_str = f"**{args.kwarg.arg}" + if args.kwarg.annotation: + kwarg_str += f": {ast.unparse(args.kwarg.annotation)}" + params.append(kwarg_str) + + return ', '.join(params) + + +def _indent_body(body: str) -> str: + """Clean and re-indent a type body for readable output.""" + lines = body.strip().split('\n') + result = [] + for line in lines: + stripped = line.strip() + if stripped: + # Keep JSDoc comments and method signatures with consistent indentation + if not stripped.startswith('//') and not stripped.startswith('/*') and not stripped.startswith('*'): + result.append(f" {stripped}") + else: + result.append(f" {stripped}") + else: + result.append('') + return '\n'.join(result) + + # ============================================================================= # Skill Generation # ============================================================================= @@ -947,6 +1143,13 @@ def main(): (OUTPUT_SDKS_DIR / "python.md").write_text(py_sdk_md) print(f" Found {len(py_functions)} functions, {len(py_classes)} classes") + # Extract datatable-specific SDK docs (for app mode system prompt) + print("Extracting datatable SDK docs...") + datatable_ts_md = extract_datatable_ts_sdk() + datatable_py_md = extract_datatable_py_sdk(py_content) + (OUTPUT_SDKS_DIR / "datatable-typescript.md").write_text(datatable_ts_md) + (OUTPUT_SDKS_DIR / "datatable-python.md").write_text(datatable_py_md) + # Read base prompts print("Assembling complete prompts...") base_dir = SCRIPT_DIR / "base" @@ -1009,6 +1212,10 @@ def main(): 'SDK_TYPESCRIPT': ts_sdk_md, 'SDK_PYTHON': py_sdk_md, + # Datatable-specific SDK docs (for app mode) + 'DATATABLE_SDK_TYPESCRIPT': datatable_ts_md, + 'DATATABLE_SDK_PYTHON': datatable_py_md, + # Schema (raw YAML content) 'OPENFLOW_SCHEMA': openflow_content, @@ -1077,6 +1284,14 @@ export function getFlowPrompt(): string { prompts.OPENFLOW_SCHEMA ].filter(Boolean).join('\\n\\n'); } + +// Helper to get datatable SDK reference for app mode +export function getDatatableSdkReference(): string { + return [ + prompts.DATATABLE_SDK_TYPESCRIPT, + prompts.DATATABLE_SDK_PYTHON + ].filter(Boolean).join('\\n\\n'); +} """ (OUTPUT_GENERATED_DIR / "index.ts").write_text(index_content) From 4c8edd5e944d77ed2d41c2b87171c1115c0fdcdc Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 14:51:13 +0000 Subject: [PATCH 44/48] fix: restrict logout redirect to whitelisted domains (#8524) Co-authored-by: Claude Opus 4.5 --- backend/Cargo.lock | 1 + backend/windmill-api-users/Cargo.toml | 1 + backend/windmill-api-users/src/users.rs | 36 +++++++++++++++++-- frontend/src/lib/logoutRedirect.ts | 23 ++++++++++++ .../(logged)/user/(user)/logout/+page@.svelte | 7 +++- 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 frontend/src/lib/logoutRedirect.ts diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 34532bc173..c903490c8d 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16386,6 +16386,7 @@ dependencies = [ "tokio", "tower-cookies", "tracing", + "url", "windmill-api-auth", "windmill-audit", "windmill-common", diff --git a/backend/windmill-api-users/Cargo.toml b/backend/windmill-api-users/Cargo.toml index e720b37eb7..13ab8143d8 100644 --- a/backend/windmill-api-users/Cargo.toml +++ b/backend/windmill-api-users/Cargo.toml @@ -34,3 +34,4 @@ time.workspace = true tokio.workspace = true tower-cookies.workspace = true tracing.workspace = true +url.workspace = true diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 8f7bec1398..75db0e2d24 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -49,13 +49,13 @@ use windmill_common::users::truncate_token; use windmill_common::users::COOKIE_NAME; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::BASE_URL; use windmill_common::{ auth::{get_folders_for_user, get_groups_for_user}, db::UserDB, error::{self, Error, JsonResult, Result}, utils::{not_found_if_none, rd_string, require_admin, Pagination, StripPath}, }; +use windmill_common::{BASE_URL, HUB_BASE_URL}; use windmill_git_sync::handle_deployment_metadata; const COOKIE_PATH: &str = "/"; @@ -577,12 +577,44 @@ async fn logout( } tx.commit().await?; if let Some(rd) = rd { - Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) + if is_valid_logout_redirect(&rd).await { + Ok((StatusCode::TEMPORARY_REDIRECT, [(LOCATION, rd)]).into_response()) + } else { + tracing::warn!("Blocked logout redirect to non-whitelisted URL: {}", rd); + Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) + } } else { Ok((StatusCode::OK, "logged out successfully".to_string()).into_response()) } } +async fn is_valid_logout_redirect(rd: &str) -> bool { + // Allow relative paths (same-origin redirects) + if rd.starts_with('/') && !rd.starts_with("//") { + return true; + } + let parsed = match url::Url::parse(rd) { + Ok(u) => u, + Err(_) => return false, + }; + let host: &str = match parsed.host_str() { + Some(h) => h, + None => return false, + }; + if host == "windmill.dev" || host.ends_with(".windmill.dev") { + return true; + } + let hub_url = HUB_BASE_URL.read().await.clone(); + if let Ok(hub_parsed) = url::Url::parse(&hub_url) { + if let Some(hub_host) = hub_parsed.host_str() { + if host == hub_host { + return true; + } + } + } + false +} + async fn whoami( Extension(db): Extension, Path(w_id): Path, diff --git a/frontend/src/lib/logoutRedirect.ts b/frontend/src/lib/logoutRedirect.ts new file mode 100644 index 0000000000..9b09880fd2 --- /dev/null +++ b/frontend/src/lib/logoutRedirect.ts @@ -0,0 +1,23 @@ +import { get } from 'svelte/store' +import { hubBaseUrlStore } from './stores' + +export function isValidLogoutRedirect(url: string): boolean { + if (url.startsWith('/') && !url.startsWith('//')) { + return true + } + try { + const parsed = new URL(url) + const host = parsed.hostname + if (host === 'windmill.dev' || host.endsWith('.windmill.dev')) { + return true + } + const hubBaseUrl = get(hubBaseUrlStore) + try { + const hubHost = new URL(hubBaseUrl).hostname + if (host === hubHost) { + return true + } + } catch {} + } catch {} + return false +} diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/logout/+page@.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/logout/+page@.svelte index e328c37f5a..75e31a9a00 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/logout/+page@.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/logout/+page@.svelte @@ -2,6 +2,7 @@ import { page } from '$app/state' import CenteredModal from '$lib/components/CenteredModal.svelte' import { clearUser } from '$lib/logout' + import { isValidLogoutRedirect } from '$lib/logoutRedirect' import { userStore } from '$lib/stores' import { onMount } from 'svelte' @@ -29,7 +30,11 @@ return } - window.location.href = rd ?? '/user/login' + if (rd && isValidLogoutRedirect(rd)) { + window.location.href = rd + } else { + window.location.href = '/user/login' + } }) From c28314f424ea0e04b86565ce88e6c91e0df1a0cf Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 25 Mar 2026 16:13:04 +0100 Subject: [PATCH 45/48] feat: runner groups for shared-process multi-script dedicated workers (#8434) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add runner groups for shared-process multi-script dedicated workers Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: unify dedicated worker and runner group wrappers into single multi-script wrapper Replace per-language single-script wrappers with the unified load/exec/exec_preprocess/end protocol. Each start_worker() now writes scripts to scripts// and uses generate_multi_script_wrapper(). handle_dedicated_process() sends load: on start and exec: per job instead of raw JSON args. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: merge runner groups into dedicated workers with inline arg metadata Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to match EE branch Co-Authored-By: Claude Opus 4.6 (1M context) * fix: gate EE-only functions behind cfg(feature = "private") to fix OSS dead_code errors Co-Authored-By: Claude Opus 4.6 (1M context) * feat: auto-detect runner groups from workspace dependency annotations - New endpoint GET /scripts/list_dedicated_with_deps: returns dedicated scripts with parsed workspace dependency names from content annotations - Frontend: show dep badges in DedicatedWorkersSelector with links to workspace settings, warn when referenced dep doesn't exist, group scripts sharing deps into "Shared runner" sections - Remove manual "Runner groups" tab and RunnerGroupSelector component - Remove runner_groups from WorkerConfigOpt/WorkerConfig (auto-detected) - Fix Node.js single dedicated workers: transpile main.ts -> main.js via Bun.build so the multi-script wrapper's dynamic import() works under Node - Add package.json with type:module in scripts dir to silence Node warning Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: unify dedicated worker wrappers with baked-in codegen and routing Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * test: add e2e tests for multi-script dedicated worker routing (bun, deno, python) Co-Authored-By: Claude Opus 4.6 (1M context) * chore: remove dead generate_dedicated_worker_wrapper function Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add dependency installation to runner groups + make dep functions pub(crate) Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * fix: prevent bun loader from intercepting absolute paths within cwd When a plugin's onResolve returns an absolute path, Bun re-invokes the resolver with that path. The loader was then routing it through the remote URL resolver, breaking runner group script imports. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use _wm_ prefix for runner group scripts to avoid bun loader interception Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: extract DENO_UNSTABLE_ARGS constant to avoid repeating flags Co-Authored-By: Claude Opus 4.6 (1M context) * chore: regenerate system prompts Co-Authored-By: Claude Opus 4.6 (1M context) * fix: gate private-only exports behind cfg(feature = "private") for OSS build Co-Authored-By: Claude Opus 4.6 (1M context) * fix: move format strings before handle_dedicated_process to fix lifetime Co-Authored-By: Claude Opus 4.6 (1M context) * chore: regenerate sqlx offline cache Co-Authored-By: Claude Opus 4.6 (1M context) * fix sqlx * fix: skip empty lines in deno e2e tests (double newline from console.log + '\n') Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use dict() instead of {{}} in python wrapper to avoid set literal {{{{}}}} in format!() produces {{}} which Python interprets as an empty set, not a dict. Use dict() which is unambiguous. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: remove deno from runner groups and associated tests Deno resolves dependencies at runtime via URLs/import maps, so there's no shared node_modules/pip install to benefit from runner groups. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: revert deno wrapper to inline old-style with exec: protocol Since deno doesn't support runner groups, the unified multi-script wrapper is unnecessary. Reverted to the old inline wrapper from main but adapted to use the exec:: protocol. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: extract deno wrapper into reusable function and add e2e tests Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use codebase presence (not nodejs annotation) to determine wrapper import extension On main, codebase scripts import ./main.js (pre-bundled JS). The wrapper_ext was incorrectly based on annotation.nodejs. Co-Authored-By: Claude Opus 4.6 (1M context) * feat: improve dedicated workers UI - combine lists, better badges, tooltips - Merge shared runners section with selected tags into one unified list - Move language tag to right side of selector for alignment - Change dep badge color from dark-gray to indigo - Add tooltip on yellow warning badge explaining missing workspace dep Co-Authored-By: Claude Opus 4.6 (1M context) * feat: group shared runners visually in dedicated workers list - Runner groups shown with a header (Shared runner · language · dep badge) - Scripts in the same group nested under the header - Standalone scripts/flows shown after groups - Used Svelte snippet for reusable tag row rendering Co-Authored-By: Claude Opus 4.6 (1M context) * fix: improve visual separation between shared runner groups and standalone items Co-Authored-By: Claude Opus 4.6 (1M context) * feat: give standalone runners same header style as shared runners - Each standalone script/flow gets its own header row with bg-surface-secondary - Header shows "Dedicated runner" / "Flow runner" label, dep link, language badge - Shared runner header: swapped language and dep badge positions - Dep shown as inline link instead of badge in headers for cleaner look Co-Authored-By: Claude Opus 4.6 (1M context) * feat: inline standalone runner path in header, language badge on right edge, no max height - Standalone items: path shown directly in header row (no sub-row) - Language badge placed after flex-1 spacer (right-aligned) - Removed max-h-64 overflow constraint from the list Co-Authored-By: Claude Opus 4.6 (1M context) * feat: consistent badges across runner list - dep+language on right, depBadge snippet - Shared runner scripts: show (workspace) and language badge on right - Standalone items: dep badges and language badge on right (after flex-1) - Shared runner header: dep badge and language badge on right - Extract depBadge snippet to deduplicate dep badge rendering - Picker selector also uses depBadge snippet Co-Authored-By: Claude Opus 4.6 (1M context) * fix: show language badge on standalone items, hide from shared runner sub-items - Fetch script language from API when not available from workspace deps - Hide dep+language badges from tagRow when script is inside a runner group (already shown in the group header) - Standalone items now always show language badge Co-Authored-By: Claude Opus 4.6 (1M context) * fix: differentiate badge colors - gray for language, indigo for workspace deps Matches codebase convention: gray for metadata (like script hashes), indigo for linkable features/entities. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use transparent (bordered) badge for language - visible on all backgrounds Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use gray badge for language everywhere Co-Authored-By: Claude Opus 4.6 (1M context) * fix: revert skills.ts and AI files, add _wm_ exclusion to Windows loader - Revert cli/src/guidance/skills.ts to main (not our change) - Revert AI provider formatting changes (not our change) - Add _wm_ prefix exclusion to loader.bun.windows.js filterResolve Co-Authored-By: Claude Opus 4.6 (1M context) * fix: update ee-repo-ref and regenerate system prompts after merge Co-Authored-By: Claude Opus 4.6 (1M context) * perf: use DISTINCT ON in list_dedicated_with_deps to dedup at DB level Avoids fetching all script versions and deduplicating in Rust. Addresses PR review feedback. Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use sqlx query! macro for list_dedicated_with_deps and regenerate cache Co-Authored-By: Claude Opus 4.6 (1M context) * fix: dedicated worker review fixes and test coverage - Fix Python relative imports in dedicated workers (write loader.py, add import loader to wrapper when needed) - Move Python colon parsing inside try/except to prevent crashes on malformed stdin - Add indexOf guard in Bun/Deno wrappers for malformed protocol messages - Add stderr logging for unrecognized stdin commands in all wrappers - Remove asyncio handling from Python wrapper (consistent with normal path) - Add exec_preprocess protocol tests for Bun, Deno, and Python - Add argument transformation tests (dates, bytes, kwargs, sentinel) - Add relative import detection test for Python wrapper - Add PreprocessedArgs variant to DedicatedWorkerResult test helper Co-Authored-By: Claude Opus 4.6 (1M context) * fix: remove symlink from git and gate has_relative_imports behind private feature Co-Authored-By: Claude Opus 4.6 (1M context) * fix: update ee-repo-ref for dedicated_worker_ee.rs changes Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add mixed exec+preprocess test to use ProtocolCmd::Exec variant Co-Authored-By: Claude Opus 4.6 (1M context) * fix: remove hanging deno missing-preprocessor test The Deno wrapper only generates the exec_preprocess handler when the script has a preprocessor function. Without one, the message is unrecognized and the test hangs reading stdout. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to 182943e5ad9bf2a905ccdf07d4e346437fb329a9 This commit updates the EE repository reference after PR #466 was merged in windmill-ee-private. Previous ee-repo-ref: 995f701fe3754be6260fc6b679e5de8fc636e68a New ee-repo-ref: 182943e5ad9bf2a905ccdf07d4e346437fb329a9 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...6be1bceb79c82e7b5542f17b23b6d70cc02d6.json | 104 +++ ...7d5ee9490240a627b20a1037444845e39c5f.json} | 4 +- backend/ee-repo-ref.txt | 2 +- backend/tests/bun_jobs.rs | 769 +++++++++++++++++- backend/tests/python_jobs.rs | 461 +++++++++++ backend/windmill-api-scripts/src/scripts.rs | 69 +- backend/windmill-api/openapi.yaml | 54 ++ backend/windmill-common/src/ai_google.rs | 40 +- backend/windmill-common/src/worker.rs | 12 +- backend/windmill-test-utils/src/lib.rs | 10 + backend/windmill-worker/loader.bun.js | 2 +- backend/windmill-worker/loader.bun.windows.js | 2 +- backend/windmill-worker/src/bun_executor.rs | 360 +++++--- backend/windmill-worker/src/deno_executor.rs | 284 +++---- backend/windmill-worker/src/lib.rs | 19 +- .../windmill-worker/src/python_executor.rs | 499 +++++++++--- .../DedicatedWorkersSelector.svelte | 412 ++++++++-- 17 files changed, 2590 insertions(+), 513 deletions(-) create mode 100644 backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json rename backend/.sqlx/{query-d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90.json => query-fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f.json} (50%) diff --git a/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json new file mode 100644 index 0000000000..a0d7f223d0 --- /dev/null +++ b/backend/.sqlx/query-f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, language AS \"language: ScriptLang\", content FROM script\n WHERE workspace_id = $1\n AND archived = false\n AND dedicated_worker = true\n AND language = ANY($2::SCRIPT_LANG[])\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "language: ScriptLang", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "script_lang[]", + "kind": { + "Array": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java", + "duckdb", + "ruby" + ] + } + } + } + } + } + } + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "f2fa27ed5020aa9c085176b25466be1bceb79c82e7b5542f17b23b6d70cc02d6" +} diff --git a/backend/.sqlx/query-d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90.json b/backend/.sqlx/query-fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f.json similarity index 50% rename from backend/.sqlx/query-d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90.json rename to backend/.sqlx/query-fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f.json index e778c17bf6..50cf586c79 100644 --- a/backend/.sqlx/query-d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90.json +++ b/backend/.sqlx/query-fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO token\n (token_hash, token_prefix, token, label, super_admin, email)\n VALUES ($1, $2, $3, $4, $5, $6)", + "query": "INSERT INTO token\n (token_hash, token_prefix, token, label, super_admin, email)\n VALUES ($1, $2, $3, $4, $5, $6)", "describe": { "columns": [], "parameters": { @@ -15,5 +15,5 @@ }, "nullable": [] }, - "hash": "d05f20431cd08f737bfbf904efedfdf104e3d77b0725c5355305d19f67359e90" + "hash": "fd4c5391107af34a3bf9b83b0c3f7d5ee9490240a627b20a1037444845e39c5f" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7d86a6114e..263ec1de9a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -b5c8af4df9ba2c39fdd494d7a40f9a92fbff8abc +182943e5ad9bf2a905ccdf07d4e346437fb329a9 diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index 2edfef8989..7183cf6e10 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -891,25 +891,34 @@ mod dedicated_worker_protocol { use std::process::{Command, Stdio}; use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult}; use windmill_worker::{ - build_loader, generate_dedicated_worker_wrapper, LoaderMode, BUN_DEDICATED_WORKER_ARGS, - BUN_PATH, NODE_BIN_PATH, + build_loader, compute_ts_codegen, generate_multi_script_wrapper, LoaderMode, TsScriptEntry, + BUN_DEDICATED_WORKER_ARGS, BUN_PATH, NODE_BIN_PATH, }; + const TEST_SCRIPT_PATH: &str = "f/test/script"; + /// Creates test worker files and optionally bundles for Node.js (like production) /// Returns the path to the wrapper file to execute fn create_test_worker_files( dir: &std::path::Path, script: &str, - arg_names: &[&str], bundle_for_node: bool, ) -> std::path::PathBuf { let dir_str = dir.to_str().unwrap(); + // Write main.ts at root (like production single-script) std::fs::write(dir.join("main.ts"), script).unwrap(); + let codegen = compute_ts_codegen(script); + let ext = if bundle_for_node { "js" } else { "ts" }; + let scripts = [TsScriptEntry { + import_name: "main", + original_path: TEST_SCRIPT_PATH, + codegen: &codegen, + }]; + let wrapper = generate_multi_script_wrapper(&scripts, ext); + if bundle_for_node { - // For Node.js: bundle to JavaScript first (like production's build_loader with LoaderMode::Node) - let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.js", None, None); - std::fs::write(dir.join("wrapper.mjs"), wrapper).unwrap(); + std::fs::write(dir.join("wrapper.mjs"), &wrapper).unwrap(); // Use the exact same build_loader function as production tokio::runtime::Runtime::new() @@ -919,7 +928,7 @@ mod dedicated_worker_protocol { "http://localhost:8000", "test_token", "test-workspace", - "f/test/script", + TEST_SCRIPT_PATH, LoaderMode::Node, &None, )) @@ -945,10 +954,8 @@ mod dedicated_worker_protocol { std::fs::rename(&bundled_path, &output_path).unwrap(); output_path } else { - // For Bun: use TypeScript directly (like production) - let wrapper = generate_dedicated_worker_wrapper(arg_names, "./main.ts", None, None); let wrapper_path = dir.join("wrapper.mjs"); - std::fs::write(&wrapper_path, wrapper).unwrap(); + std::fs::write(&wrapper_path, &wrapper).unwrap(); wrapper_path } } @@ -957,14 +964,12 @@ mod dedicated_worker_protocol { fn run_worker_test( runtime: &str, script: &str, - arg_names: &[&str], jobs: Vec, ) -> Vec> { let temp_dir = tempfile::tempdir().unwrap(); // Create files and get the wrapper path (bundled for node, raw for bun) - let wrapper_path = - create_test_worker_files(temp_dir.path(), script, arg_names, runtime == "node"); + let wrapper_path = create_test_worker_files(temp_dir.path(), script, runtime == "node"); let wrapper_str = wrapper_path.to_str().unwrap(); // Build args matching production behavior @@ -1008,7 +1013,8 @@ mod dedicated_worker_protocol { let mut results = Vec::new(); for job_args in jobs { - writeln!(stdin, "{}", job_args.to_string()).unwrap(); + // Protocol: exec:: + writeln!(stdin, "exec:{}:{}", TEST_SCRIPT_PATH, job_args.to_string()).unwrap(); stdin.flush().unwrap(); let mut response = String::new(); @@ -1043,12 +1049,7 @@ export function main(x: number, y: number): number { return x + y; } "#; - let results = run_worker_test( - "node", - script, - &["x", "y"], - vec![serde_json::json!({"x": 5, "y": 3})], - ); + let results = run_worker_test("node", script, vec![serde_json::json!({"x": 5, "y": 3})]); assert_eq!(results.len(), 1); assert_eq!(results[0], Ok(serde_json::json!(8))); @@ -1062,7 +1063,7 @@ export function main(n: number): number { } "#; let jobs: Vec = (1..=5).map(|i| serde_json::json!({"n": i})).collect(); - let results = run_worker_test("node", script, &["n"], jobs); + let results = run_worker_test("node", script, jobs); assert_eq!(results.len(), 5); for (i, result) in results.iter().enumerate() { @@ -1081,7 +1082,6 @@ export function main(msg: string): never { let results = run_worker_test( "node", script, - &["msg"], vec![serde_json::json!({"msg": "test error"})], ); @@ -1099,12 +1099,7 @@ export function main(x: number, y: number): number { return x + y; } "#; - let results = run_worker_test( - "bun", - script, - &["x", "y"], - vec![serde_json::json!({"x": 5, "y": 3})], - ); + let results = run_worker_test("bun", script, vec![serde_json::json!({"x": 5, "y": 3})]); assert_eq!(results.len(), 1); assert_eq!(results[0], Ok(serde_json::json!(8))); @@ -1118,7 +1113,7 @@ export function main(n: number): number { } "#; let jobs: Vec = (1..=5).map(|i| serde_json::json!({"n": i})).collect(); - let results = run_worker_test("bun", script, &["n"], jobs); + let results = run_worker_test("bun", script, jobs); assert_eq!(results.len(), 5); for (i, result) in results.iter().enumerate() { @@ -1137,7 +1132,6 @@ export function main(msg: string): never { let results = run_worker_test( "bun", script, - &["msg"], vec![serde_json::json!({"msg": "test error"})], ); @@ -1145,6 +1139,721 @@ export function main(msg: string): never { assert!(results[0].is_err()); assert_eq!(results[0], Err("test error".to_string())); } + + // ==================== Multi-Script (Runner Group) Tests ==================== + + /// Job to send to a specific script in a multi-script wrapper + struct MultiScriptJob { + script_path: String, + args: serde_json::Value, + } + + /// Creates a multi-script wrapper with multiple scripts as flat files, returns the wrapper path + fn create_multi_script_worker_files( + dir: &std::path::Path, + scripts: &[(&str, &str)], // (original_path, script_content) + ) -> std::path::PathBuf { + let mut entries_data = Vec::new(); + for (path, content) in scripts { + let safe_name = format!("_wm_{}", path.replace('/', "__")); + std::fs::write(dir.join(format!("{safe_name}.ts")), content).unwrap(); + entries_data.push((safe_name, path.to_string(), compute_ts_codegen(content))); + } + + let entries: Vec> = entries_data + .iter() + .map(|(safe, path, cg)| TsScriptEntry { + import_name: safe.as_str(), + original_path: path.as_str(), + codegen: cg, + }) + .collect(); + + let wrapper = generate_multi_script_wrapper(&entries, "ts"); + let wrapper_path = dir.join("wrapper.mjs"); + std::fs::write(&wrapper_path, &wrapper).unwrap(); + wrapper_path + } + + /// Helper to run a multi-script dedicated worker test + fn run_multi_script_worker_test( + scripts: &[(&str, &str)], + jobs: Vec, + ) -> Vec> { + let temp_dir = tempfile::tempdir().unwrap(); + let wrapper_path = create_multi_script_worker_files(temp_dir.path(), scripts); + let wrapper_str = wrapper_path.to_str().unwrap(); + + let mut cmd_args: Vec<&str> = BUN_DEDICATED_WORKER_ARGS.to_vec(); + cmd_args.push(wrapper_str); + + let mut child = Command::new(BUN_PATH.as_str()) + .args(cmd_args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn worker process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + // Wait for "start" signal + let mut start_line = String::new(); + reader.read_line(&mut start_line).unwrap(); + assert_eq!( + parse_dedicated_worker_line(start_line.trim()), + DedicatedWorkerResult::Start, + "Expected 'start', got: {}", + start_line.trim() + ); + + let mut results = Vec::new(); + + for job in &jobs { + writeln!(stdin, "exec:{}:{}", job.script_path, job.args.to_string()).unwrap(); + stdin.flush().unwrap(); + + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + + match parse_dedicated_worker_line(response.trim()) { + DedicatedWorkerResult::Success(value) => results.push(Ok(value)), + DedicatedWorkerResult::Error(err) => { + let msg = err["message"] + .as_str() + .unwrap_or("Unknown error") + .to_string(); + results.push(Err(msg)); + } + other => panic!("Unexpected response: {:?}", other), + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + + results + } + + #[test] + fn test_multi_script_routing_basic() { + let script_add = r#" +export function main(a: number, b: number): number { + return a + b; +} +"#; + let script_mul = r#" +export function main(x: number, y: number): number { + return x * y; +} +"#; + let results = run_multi_script_worker_test( + &[("f/math/add", script_add), ("f/math/mul", script_mul)], + vec![ + MultiScriptJob { + script_path: "f/math/add".to_string(), + args: serde_json::json!({"a": 3, "b": 4}), + }, + MultiScriptJob { + script_path: "f/math/mul".to_string(), + args: serde_json::json!({"x": 5, "y": 6}), + }, + // Route back to add + MultiScriptJob { + script_path: "f/math/add".to_string(), + args: serde_json::json!({"a": 10, "b": 20}), + }, + ], + ); + + assert_eq!(results.len(), 3); + assert_eq!(results[0], Ok(serde_json::json!(7))); // 3 + 4 + assert_eq!(results[1], Ok(serde_json::json!(30))); // 5 * 6 + assert_eq!(results[2], Ok(serde_json::json!(30))); // 10 + 20 + } + + #[test] + fn test_multi_script_interleaved_jobs() { + let script_upper = r#" +export function main(s: string): string { + return s.toUpperCase(); +} +"#; + let script_len = r#" +export function main(s: string): number { + return s.length; +} +"#; + let results = run_multi_script_worker_test( + &[("f/str/upper", script_upper), ("f/str/len", script_len)], + vec![ + MultiScriptJob { + script_path: "f/str/upper".to_string(), + args: serde_json::json!({"s": "hello"}), + }, + MultiScriptJob { + script_path: "f/str/len".to_string(), + args: serde_json::json!({"s": "hello"}), + }, + MultiScriptJob { + script_path: "f/str/upper".to_string(), + args: serde_json::json!({"s": "world"}), + }, + MultiScriptJob { + script_path: "f/str/len".to_string(), + args: serde_json::json!({"s": "ab"}), + }, + ], + ); + + assert_eq!(results.len(), 4); + assert_eq!(results[0], Ok(serde_json::json!("HELLO"))); + assert_eq!(results[1], Ok(serde_json::json!(5))); + assert_eq!(results[2], Ok(serde_json::json!("WORLD"))); + assert_eq!(results[3], Ok(serde_json::json!(2))); + } + + #[test] + fn test_multi_script_unknown_path_error() { + let script = r#" +export function main(x: number): number { + return x; +} +"#; + let results = run_multi_script_worker_test( + &[("f/known", script)], + vec![MultiScriptJob { + script_path: "f/unknown".to_string(), + args: serde_json::json!({"x": 1}), + }], + ); + + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + assert!(results[0] + .as_ref() + .unwrap_err() + .contains("Script not found")); + } + + #[test] + fn test_multi_script_error_doesnt_break_other_scripts() { + let script_ok = r#" +export function main(x: number): number { + return x * 2; +} +"#; + let script_err = r#" +export function main(msg: string): never { + throw new Error(msg); +} +"#; + let results = run_multi_script_worker_test( + &[("f/ok", script_ok), ("f/err", script_err)], + vec![ + MultiScriptJob { + script_path: "f/ok".to_string(), + args: serde_json::json!({"x": 5}), + }, + MultiScriptJob { + script_path: "f/err".to_string(), + args: serde_json::json!({"msg": "boom"}), + }, + // Should still work after error in other script + MultiScriptJob { + script_path: "f/ok".to_string(), + args: serde_json::json!({"x": 10}), + }, + ], + ); + + assert_eq!(results.len(), 3); + assert_eq!(results[0], Ok(serde_json::json!(10))); + assert!(results[1].is_err()); + assert_eq!(results[1], Err("boom".to_string())); + assert_eq!(results[2], Ok(serde_json::json!(20))); + } + + // ==================== exec_preprocess Tests ==================== + + /// Raw protocol command to send to a dedicated worker + enum ProtocolCmd { + Exec { path: String, args: serde_json::Value }, + ExecPreprocess { path: String, args: serde_json::Value }, + } + + /// Run a multi-script worker test with raw protocol commands, returning all protocol lines + fn run_raw_protocol_test( + scripts: &[(&str, &str)], + commands: Vec, + ) -> Vec { + let temp_dir = tempfile::tempdir().unwrap(); + let wrapper_path = create_multi_script_worker_files(temp_dir.path(), scripts); + let wrapper_str = wrapper_path.to_str().unwrap(); + + let mut cmd_args: Vec<&str> = BUN_DEDICATED_WORKER_ARGS.to_vec(); + cmd_args.push(wrapper_str); + + let mut child = Command::new(BUN_PATH.as_str()) + .args(cmd_args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn worker process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + let mut start_line = String::new(); + reader.read_line(&mut start_line).unwrap(); + assert_eq!( + parse_dedicated_worker_line(start_line.trim()), + DedicatedWorkerResult::Start, + ); + + let mut results = Vec::new(); + + for cmd in &commands { + let line = match cmd { + ProtocolCmd::Exec { path, args } => format!("exec:{}:{}", path, args), + ProtocolCmd::ExecPreprocess { path, args } => { + format!("exec_preprocess:{}:{}", path, args) + } + }; + writeln!(stdin, "{}", line).unwrap(); + stdin.flush().unwrap(); + + // exec_preprocess produces 2 response lines (preprocessed_args + success/error) + // exec produces 1 response line (success/error) + let expected_lines = match cmd { + ProtocolCmd::ExecPreprocess { .. } => 2, + ProtocolCmd::Exec { .. } => 1, + }; + + for _ in 0..expected_lines { + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + let parsed = parse_dedicated_worker_line(response.trim()); + // If it's an error, stop reading more lines for this command + if matches!(parsed, DedicatedWorkerResult::Error(_)) { + results.push(parsed); + break; + } + results.push(parsed); + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + + results + } + + #[test] + fn test_bun_exec_preprocess() { + let script = r#" +export function preprocessor(x: number) { + return { x: x * 10 }; +} +export function main(x: number): number { + return x + 1; +} +"#; + let results = run_raw_protocol_test( + &[("f/test/pre", script)], + vec![ProtocolCmd::ExecPreprocess { + path: "f/test/pre".to_string(), + args: serde_json::json!({"x": 5}), + }], + ); + // Should get preprocessed_args then success + assert_eq!(results.len(), 2); + assert_eq!( + results[0], + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50})) + ); + // main(50) => 51 + assert_eq!( + results[1], + DedicatedWorkerResult::Success(serde_json::json!(51)) + ); + } + + #[test] + fn test_bun_exec_preprocess_missing_preprocessor() { + let script = r#" +export function main(x: number): number { + return x; +} +"#; + let results = run_raw_protocol_test( + &[("f/test/nopre", script)], + vec![ProtocolCmd::ExecPreprocess { + path: "f/test/nopre".to_string(), + args: serde_json::json!({"x": 5}), + }], + ); + assert_eq!(results.len(), 1); + assert!(matches!(results[0], DedicatedWorkerResult::Error(_))); + } + + #[test] + fn test_bun_exec_preprocess_then_exec() { + let script = r#" +export function preprocessor(x: number) { + return { x: x * 2 }; +} +export function main(x: number): number { + return x + 100; +} +"#; + let results = run_raw_protocol_test( + &[("f/test/mixed", script)], + vec![ + ProtocolCmd::ExecPreprocess { + path: "f/test/mixed".to_string(), + args: serde_json::json!({"x": 5}), + }, + ProtocolCmd::Exec { + path: "f/test/mixed".to_string(), + args: serde_json::json!({"x": 7}), + }, + ], + ); + // preprocess: preprocessor(5) => {"x":10}, main(10) => 110 + // exec: main(7) => 107 + assert_eq!(results.len(), 3); + assert_eq!( + results[0], + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})) + ); + assert_eq!( + results[1], + DedicatedWorkerResult::Success(serde_json::json!(110)) + ); + assert_eq!( + results[2], + DedicatedWorkerResult::Success(serde_json::json!(107)) + ); + } + + // ==================== Argument Transformation Tests ==================== + + #[test] + fn test_bun_date_arg_transformation() { + let script = r#" +export function main(d: Date): string { + return d instanceof Date ? d.toISOString() : typeof d; +} +"#; + let results = run_worker_test( + "bun", + script, + vec![serde_json::json!({"d": "2024-01-15T10:30:00.000Z"})], + ); + assert_eq!(results.len(), 1); + assert_eq!( + results[0], + Ok(serde_json::json!("2024-01-15T10:30:00.000Z")) + ); + } + + #[test] + fn test_bun_null_and_undefined_args() { + let script = r#" +export function main(x?: number): string { + return x === null ? "null" : x === undefined ? "undefined" : String(x); +} +"#; + let results = run_worker_test( + "bun", + script, + vec![ + serde_json::json!({"x": null}), + serde_json::json!({"x": 42}), + serde_json::json!({}), + ], + ); + assert_eq!(results.len(), 3); + assert_eq!(results[0], Ok(serde_json::json!("null"))); + assert_eq!(results[1], Ok(serde_json::json!("42"))); + // Missing arg should be undefined + assert_eq!(results[2], Ok(serde_json::json!("undefined"))); + } +} + +// ============================================================================ +// Deno Dedicated Worker Protocol Tests +// ============================================================================ + +mod dedicated_worker_protocol_deno { + use std::io::{BufRead, BufReader, Write}; + use std::process::{Command, Stdio}; + use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult}; + use windmill_worker::{generate_deno_dedicated_worker_wrapper, DENO_PATH}; + + const TEST_SCRIPT_PATH: &str = "f/test/script"; + + fn run_deno_worker_test( + script: &str, + jobs: Vec, + ) -> Vec> { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::write(temp_dir.path().join("main.ts"), script).unwrap(); + + let wrapper = generate_deno_dedicated_worker_wrapper(script).unwrap(); + std::fs::write(temp_dir.path().join("wrapper.ts"), &wrapper).unwrap(); + + let mut child = Command::new(DENO_PATH.as_str()) + .args([ + "run", + "--no-check", + "--unstable-unsafe-proto", + "--unstable-bare-node-builtins", + "-A", + "wrapper.ts", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn deno process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + // Wait for "start" — deno outputs 'start\n' via console.log which adds + // its own newline, producing double newlines. Skip empty lines. + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line.trim().is_empty() { + continue; + } + assert_eq!( + parse_dedicated_worker_line(line.trim()), + DedicatedWorkerResult::Start, + "Expected 'start', got: {}", + line.trim() + ); + break; + } + + let mut results = Vec::new(); + for job_args in jobs { + writeln!(stdin, "exec:{}:{}", TEST_SCRIPT_PATH, job_args.to_string()).unwrap(); + stdin.flush().unwrap(); + + loop { + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + let trimmed = response.trim(); + if trimmed.is_empty() { + continue; + } + match parse_dedicated_worker_line(trimmed) { + DedicatedWorkerResult::Success(value) => results.push(Ok(value)), + DedicatedWorkerResult::Error(err) => { + let msg = err["message"] + .as_str() + .unwrap_or("Unknown error") + .to_string(); + results.push(Err(msg)); + } + other => panic!("Unexpected response: {:?}", other), + } + break; + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + results + } + + #[test] + fn test_deno_dedicated_worker_simple() { + let script = r#" +export function main(x: number, y: number): number { + return x + y; +} +"#; + let results = run_deno_worker_test(script, vec![serde_json::json!({"x": 5, "y": 3})]); + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(8))); + } + + #[test] + fn test_deno_dedicated_worker_multiple_jobs() { + let script = r#" +export function main(n: number): number { + return n * 2; +} +"#; + let jobs: Vec = (1..=5).map(|i| serde_json::json!({"n": i})).collect(); + let results = run_deno_worker_test(script, jobs); + assert_eq!(results.len(), 5); + for (i, result) in results.iter().enumerate() { + assert_eq!(*result, Ok(serde_json::json!(((i + 1) * 2) as i64))); + } + } + + #[test] + fn test_deno_dedicated_worker_error() { + let script = r#" +export function main(msg: string): never { + throw new Error(msg); +} +"#; + let results = run_deno_worker_test(script, vec![serde_json::json!({"msg": "test error"})]); + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + assert_eq!(results[0], Err("test error".to_string())); + } + + // ==================== exec_preprocess Tests ==================== + + /// Run a raw deno protocol test, reading all output lines per command + fn run_deno_raw_protocol_test( + script: &str, + commands: Vec<(&str, serde_json::Value)>, // ("exec" or "exec_preprocess", args) + ) -> Vec { + let temp_dir = tempfile::tempdir().unwrap(); + std::fs::write(temp_dir.path().join("main.ts"), script).unwrap(); + + let wrapper = generate_deno_dedicated_worker_wrapper(script).unwrap(); + std::fs::write(temp_dir.path().join("wrapper.ts"), &wrapper).unwrap(); + + let mut child = Command::new(DENO_PATH.as_str()) + .args([ + "run", + "--no-check", + "--unstable-unsafe-proto", + "--unstable-bare-node-builtins", + "-A", + "wrapper.ts", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn deno process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + // Wait for start, skip empty lines + loop { + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + if line.trim().is_empty() { + continue; + } + assert_eq!( + parse_dedicated_worker_line(line.trim()), + DedicatedWorkerResult::Start, + ); + break; + } + + let mut results = Vec::new(); + + for (cmd, args) in &commands { + writeln!(stdin, "{}:{}:{}", cmd, TEST_SCRIPT_PATH, args).unwrap(); + stdin.flush().unwrap(); + + let expected_lines = if *cmd == "exec_preprocess" { 2 } else { 1 }; + + for _ in 0..expected_lines { + loop { + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + if response.trim().is_empty() { + continue; + } + let parsed = parse_dedicated_worker_line(response.trim()); + if matches!(parsed, DedicatedWorkerResult::Error(_)) { + results.push(parsed); + break; + } + results.push(parsed); + break; + } + // If last result was an error, don't read more lines for this command + if matches!(results.last(), Some(DedicatedWorkerResult::Error(_))) { + break; + } + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + + results + } + + #[test] + fn test_deno_exec_preprocess() { + let script = r#" +export function preprocessor(x: number) { + return { x: x * 10 }; +} +export function main(x: number): number { + return x + 1; +} +"#; + let results = run_deno_raw_protocol_test( + script, + vec![("exec_preprocess", serde_json::json!({"x": 5}))], + ); + assert_eq!(results.len(), 2); + assert_eq!( + results[0], + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50})) + ); + assert_eq!( + results[1], + DedicatedWorkerResult::Success(serde_json::json!(51)) + ); + } + + // Note: no "missing preprocessor" test for Deno because the wrapper only generates + // the exec_preprocess handler when the script actually has a preprocessor function. + // Without one, exec_preprocess messages are unrecognized (by design — Rust never sends them). + + // ==================== Argument Transformation Tests ==================== + + #[test] + fn test_deno_date_arg_transformation() { + let script = r#" +export function main(d: Date): string { + return d instanceof Date ? d.toISOString() : typeof d; +} +"#; + let results = run_deno_worker_test( + script, + vec![serde_json::json!({"d": "2024-01-15T10:30:00.000Z"})], + ); + assert_eq!(results.len(), 1); + assert_eq!( + results[0], + Ok(serde_json::json!("2024-01-15T10:30:00.000Z")) + ); + } } // ============================================================================ diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 2ffb85a418..f04e189315 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1,9 +1,470 @@ use serde_json::json; +#[cfg(feature = "python")] use sqlx::postgres::Postgres; +#[cfg(feature = "python")] use sqlx::Pool; +#[cfg(feature = "python")] use windmill_common::scripts::ScriptLang; use windmill_test_utils::*; +// ============================================================================ +// Dedicated Worker Protocol Tests (Python) +// ============================================================================ + +#[cfg(feature = "python")] +mod dedicated_worker_protocol_python { + use std::io::{BufRead, BufReader, Write}; + use std::process::{Command, Stdio}; + use windmill_test_utils::{parse_dedicated_worker_line, DedicatedWorkerResult}; + use windmill_worker::{compute_py_codegen, generate_py_multi_script_wrapper, PyScriptEntry}; + + struct MultiScriptJob { + script_path: String, + args: serde_json::Value, + } + + /// Creates a multi-script Python wrapper, writes scripts to proper module paths + fn create_py_worker_files( + dir: &std::path::Path, + scripts: &[(&str, &str)], // (original_path, content) + ) -> std::path::PathBuf { + let mut codegens = Vec::new(); + for (path, content) in scripts { + let cg = compute_py_codegen(content, path); + let module_dir = dir.join(&cg.dirs); + std::fs::create_dir_all(&module_dir).unwrap(); + std::fs::write(module_dir.join(format!("{}.py", cg.module_name)), content).unwrap(); + codegens.push((path.to_string(), cg)); + } + + let entries: Vec> = codegens + .iter() + .map(|(path, cg)| PyScriptEntry { original_path: path.as_str(), codegen: cg }) + .collect(); + + let wrapper = generate_py_multi_script_wrapper(&entries, false, false); + let wrapper_path = dir.join("wrapper.py"); + std::fs::write(&wrapper_path, &wrapper).unwrap(); + wrapper_path + } + + fn run_py_multi_script_test( + scripts: &[(&str, &str)], + jobs: Vec, + ) -> Vec> { + let temp_dir = tempfile::tempdir().unwrap(); + create_py_worker_files(temp_dir.path(), scripts); + + let mut child = Command::new("python3") + .args(["-u", "-m", "wrapper"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn python3 process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + let mut start_line = String::new(); + reader.read_line(&mut start_line).unwrap(); + assert_eq!( + parse_dedicated_worker_line(start_line.trim()), + DedicatedWorkerResult::Start, + "Expected 'start', got: {}", + start_line.trim() + ); + + let mut results = Vec::new(); + for job in &jobs { + writeln!(stdin, "exec:{}:{}", job.script_path, job.args.to_string()).unwrap(); + stdin.flush().unwrap(); + + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + + match parse_dedicated_worker_line(response.trim()) { + DedicatedWorkerResult::Success(value) => results.push(Ok(value)), + DedicatedWorkerResult::Error(err) => { + let msg = err["message"] + .as_str() + .unwrap_or("Unknown error") + .to_string(); + results.push(Err(msg)); + } + other => panic!("Unexpected response: {:?}", other), + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + results + } + + fn run_py_single_script_test( + script_path: &str, + content: &str, + jobs: Vec, + ) -> Vec> { + run_py_multi_script_test( + &[(script_path, content)], + jobs.into_iter() + .map(|args| MultiScriptJob { script_path: script_path.to_string(), args }) + .collect(), + ) + } + + #[test] + fn test_python_dedicated_worker_simple() { + let results = run_py_single_script_test( + "f/test/add", + "def main(a: int, b: int):\n return a + b\n", + vec![serde_json::json!({"a": 3, "b": 4})], + ); + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(7))); + } + + #[test] + fn test_python_dedicated_worker_multiple_jobs() { + let results = run_py_single_script_test( + "f/test/double", + "def main(n: int):\n return n * 2\n", + (1..=5).map(|i| serde_json::json!({"n": i})).collect(), + ); + assert_eq!(results.len(), 5); + for (i, result) in results.iter().enumerate() { + assert_eq!(*result, Ok(serde_json::json!(((i + 1) * 2) as i64))); + } + } + + #[test] + fn test_python_multi_script_routing() { + let results = run_py_multi_script_test( + &[ + ( + "f/math/add", + "def main(a: int, b: int):\n return a + b\n", + ), + ( + "f/math/mul", + "def main(x: int, y: int):\n return x * y\n", + ), + ], + vec![ + MultiScriptJob { + script_path: "f/math/add".to_string(), + args: serde_json::json!({"a": 3, "b": 4}), + }, + MultiScriptJob { + script_path: "f/math/mul".to_string(), + args: serde_json::json!({"x": 5, "y": 6}), + }, + MultiScriptJob { + script_path: "f/math/add".to_string(), + args: serde_json::json!({"a": 10, "b": 20}), + }, + ], + ); + assert_eq!(results.len(), 3); + assert_eq!(results[0], Ok(serde_json::json!(7))); + assert_eq!(results[1], Ok(serde_json::json!(30))); + assert_eq!(results[2], Ok(serde_json::json!(30))); + } + + #[test] + fn test_python_multi_script_error_isolation() { + let results = run_py_multi_script_test( + &[ + ("f/ok", "def main(x: int):\n return x * 2\n"), + ("f/err", "def main(msg: str):\n raise Exception(msg)\n"), + ], + vec![ + MultiScriptJob { + script_path: "f/ok".to_string(), + args: serde_json::json!({"x": 5}), + }, + MultiScriptJob { + script_path: "f/err".to_string(), + args: serde_json::json!({"msg": "boom"}), + }, + MultiScriptJob { + script_path: "f/ok".to_string(), + args: serde_json::json!({"x": 10}), + }, + ], + ); + assert_eq!(results.len(), 3); + assert_eq!(results[0], Ok(serde_json::json!(10))); + assert!(results[1].is_err()); + assert_eq!(results[1], Err("boom".to_string())); + assert_eq!(results[2], Ok(serde_json::json!(20))); + } + + #[test] + fn test_python_multi_script_unknown_path() { + let results = run_py_multi_script_test( + &[("f/known", "def main(x: int):\n return x\n")], + vec![MultiScriptJob { + script_path: "f/unknown".to_string(), + args: serde_json::json!({"x": 1}), + }], + ); + assert_eq!(results.len(), 1); + assert!(results[0].is_err()); + assert!(results[0] + .as_ref() + .unwrap_err() + .contains("Script not found")); + } + + // ==================== exec_preprocess Tests ==================== + + /// Raw protocol command for Python + enum ProtocolCmd { + Exec { path: String, args: serde_json::Value }, + ExecPreprocess { path: String, args: serde_json::Value }, + } + + /// Run a Python worker test with raw protocol commands + fn run_py_raw_protocol_test( + scripts: &[(&str, &str)], + commands: Vec, + ) -> Vec { + let temp_dir = tempfile::tempdir().unwrap(); + create_py_worker_files(temp_dir.path(), scripts); + + let mut child = Command::new("python3") + .args(["-u", "-m", "wrapper"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .current_dir(temp_dir.path()) + .spawn() + .expect("Failed to spawn python3 process"); + + let mut stdin = child.stdin.take().unwrap(); + let stdout = child.stdout.take().unwrap(); + let mut reader = BufReader::new(stdout); + + let mut start_line = String::new(); + reader.read_line(&mut start_line).unwrap(); + assert_eq!( + parse_dedicated_worker_line(start_line.trim()), + DedicatedWorkerResult::Start, + ); + + let mut results = Vec::new(); + + for cmd in &commands { + let line = match cmd { + ProtocolCmd::Exec { path, args } => format!("exec:{}:{}", path, args), + ProtocolCmd::ExecPreprocess { path, args } => { + format!("exec_preprocess:{}:{}", path, args) + } + }; + writeln!(stdin, "{}", line).unwrap(); + stdin.flush().unwrap(); + + let expected_lines = match cmd { + ProtocolCmd::ExecPreprocess { .. } => 2, + ProtocolCmd::Exec { .. } => 1, + }; + + for _ in 0..expected_lines { + let mut response = String::new(); + reader.read_line(&mut response).unwrap(); + let parsed = parse_dedicated_worker_line(response.trim()); + if matches!(parsed, DedicatedWorkerResult::Error(_)) { + results.push(parsed); + break; + } + results.push(parsed); + } + } + + writeln!(stdin, "end").unwrap(); + stdin.flush().unwrap(); + let _ = child.wait().expect("Worker process failed to exit"); + + results + } + + #[test] + fn test_python_exec_preprocess() { + let script = r#" +def preprocessor(x: int): + return {"x": x * 10} + +def main(x: int): + return x + 1 +"#; + let results = run_py_raw_protocol_test( + &[("f/test/pre", script)], + vec![ProtocolCmd::ExecPreprocess { + path: "f/test/pre".to_string(), + args: serde_json::json!({"x": 5}), + }], + ); + assert_eq!(results.len(), 2); + assert_eq!( + results[0], + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 50})) + ); + // main(50) => 51 + assert_eq!( + results[1], + DedicatedWorkerResult::Success(serde_json::json!(51)) + ); + } + + #[test] + fn test_python_exec_preprocess_missing_preprocessor() { + let script = "def main(x: int):\n return x\n"; + let results = run_py_raw_protocol_test( + &[("f/test/nopre", script)], + vec![ProtocolCmd::ExecPreprocess { + path: "f/test/nopre".to_string(), + args: serde_json::json!({"x": 5}), + }], + ); + assert_eq!(results.len(), 1); + assert!(matches!(results[0], DedicatedWorkerResult::Error(_))); + } + + #[test] + fn test_python_exec_preprocess_then_exec() { + let script = r#" +def preprocessor(x: int): + return {"x": x * 2} + +def main(x: int): + return x + 100 +"#; + let results = run_py_raw_protocol_test( + &[("f/test/mixed", script)], + vec![ + ProtocolCmd::ExecPreprocess { + path: "f/test/mixed".to_string(), + args: serde_json::json!({"x": 5}), + }, + ProtocolCmd::Exec { + path: "f/test/mixed".to_string(), + args: serde_json::json!({"x": 7}), + }, + ], + ); + // preprocess: preprocessor(5) => {"x":10}, main(10) => 110 + // exec: main(7) => 107 + assert_eq!(results.len(), 3); + assert_eq!( + results[0], + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})) + ); + assert_eq!( + results[1], + DedicatedWorkerResult::Success(serde_json::json!(110)) + ); + assert_eq!( + results[2], + DedicatedWorkerResult::Success(serde_json::json!(107)) + ); + } + + // ==================== Argument Transformation Tests ==================== + + #[test] + fn test_python_datetime_arg_transformation() { + let script = r#" +from datetime import datetime + +def main(d: datetime): + return d.isoformat() +"#; + let results = run_py_single_script_test( + "f/test/dt", + script, + vec![serde_json::json!({"d": "2024-01-15T10:30:00+00:00"})], + ); + assert_eq!(results.len(), 1); + assert_eq!( + results[0], + Ok(serde_json::json!("2024-01-15T10:30:00+00:00")) + ); + } + + #[test] + fn test_python_bytes_arg_transformation() { + let script = r#" +def main(data: bytes): + return len(data) +"#; + // base64 of "hello" is "aGVsbG8=" + let results = run_py_single_script_test( + "f/test/bytes", + script, + vec![serde_json::json!({"data": "aGVsbG8="})], + ); + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(5))); + } + + #[test] + fn test_python_kwargs_filtering() { + // Test that extra kwargs are filtered out and only declared args are passed + let script = "def main(a: int, b: int):\n return a + b\n"; + let results = run_py_single_script_test( + "f/test/kwargs", + script, + vec![serde_json::json!({"a": 1, "b": 2, "extra": 99})], + ); + assert_eq!(results.len(), 1); + assert_eq!(results[0], Ok(serde_json::json!(3))); + } + + #[test] + fn test_python_function_call_sentinel_removal() { + // Test that '' sentinel values are removed from args + let script = "def main(a: int, b: int = 10):\n return a + b\n"; + let results = run_py_single_script_test( + "f/test/sentinel", + script, + vec![serde_json::json!({"a": 5, "b": ""})], + ); + assert_eq!(results.len(), 1); + // b should be removed (sentinel), default 10 used + assert_eq!(results[0], Ok(serde_json::json!(15))); + } + + // ==================== Relative Import Tests ==================== + + #[test] + fn test_python_dedicated_worker_with_relative_import_detection() { + // Test that the wrapper includes 'import loader' when scripts have relative imports + let script_with_relative = "from f.helper import util\ndef main(x: int):\n return x\n"; + let cg = compute_py_codegen(script_with_relative, "f/test/rel"); + let entries = [PyScriptEntry { original_path: "f/test/rel", codegen: &cg }]; + let wrapper = generate_py_multi_script_wrapper(&entries, false, true); + assert!( + wrapper.contains("import loader"), + "wrapper should contain 'import loader' when any_relative_imports=true" + ); + + // Without relative imports + let script_no_relative = "def main(x: int):\n return x\n"; + let cg2 = compute_py_codegen(script_no_relative, "f/test/norel"); + let entries2 = [PyScriptEntry { original_path: "f/test/norel", codegen: &cg2 }]; + let wrapper2 = generate_py_multi_script_wrapper(&entries2, false, false); + assert!( + !wrapper2.contains("import loader"), + "wrapper should NOT contain 'import loader' when any_relative_imports=false" + ); + } +} + #[cfg(feature = "python")] #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_requirements_python(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 56de4d9892..54ac78fade 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -239,6 +239,7 @@ pub fn workspaced_service() -> Router { "/history_update/h/:hash/p/*path", post(update_script_history), ) + .route("/list_dedicated_with_deps", get(list_dedicated_with_deps)) // Temporary raw script storage for CLI lock generation .route("/raw_temp/store", post(store_raw_script_temp)) .route("/raw_temp/diff", post(diff_raw_scripts_with_deployed)) @@ -2480,6 +2481,62 @@ async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { } } +#[derive(Serialize)] +struct DedicatedScriptDeps { + path: String, + language: ScriptLang, + workspace_dep_names: Vec, +} + +async fn list_dedicated_with_deps( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + + let rows = sqlx::query!( + "SELECT DISTINCT ON (path) path, language AS \"language: ScriptLang\", content FROM script + WHERE workspace_id = $1 + AND archived = false + AND dedicated_worker = true + AND language = ANY($2::SCRIPT_LANG[]) + ORDER BY path, created_at DESC", + &w_id, + &[ + ScriptLang::Python3, + ScriptLang::Bun, + ScriptLang::Bunnative, + ScriptLang::Deno, + ] as &[ScriptLang], + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + let result = rows + .into_iter() + .map(|row| { + let dep_names = + windmill_common::scripts::extract_workspace_dependencies_annotated_refs( + &row.language, + &row.content, + &row.path, + ) + .map(|refs| refs.external) + .unwrap_or_default(); + DedicatedScriptDeps { + path: row.path, + language: row.language, + workspace_dep_names: dep_names, + } + }) + .collect(); + + Ok(Json(result)) +} + // ============================================================================ // Temporary Raw Script Storage for CLI Lock Generation // ============================================================================ @@ -2508,11 +2565,9 @@ async fn store_raw_script_temp( .await?; // Clean up old entries (1 week TTL) - sqlx::query!( - "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'" - ) - .execute(&db) - .await?; + sqlx::query!("DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'") + .execute(&db) + .await?; Ok(Json(hash)) } @@ -2560,7 +2615,7 @@ async fn diff_raw_scripts_with_deployed( FROM script s \ WHERE s.path = local.path AND s.workspace_id = $3 AND s.archived = false \ ORDER BY s.created_at DESC LIMIT 1 \ - ) deployed ON deployed.deployed_hash = local.hash" + ) deployed ON deployed.deployed_hash = local.hash", ) .bind(&paths) .bind(&hashes) @@ -2582,7 +2637,7 @@ async fn diff_raw_scripts_with_deployed( AND wd.language = $3::SCRIPT_LANG \ AND wd.name IS NOT DISTINCT FROM $4 \ AND encode(sha256(convert_to(wd.content, 'UTF8')), 'hex') = $5 \ - )" + )", ) .bind(&dep.path) .bind(&w_id) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 825842f057..cb86c2a8b9 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6940,6 +6940,60 @@ paths: schema: type: string + /w/{workspace}/scripts/list_dedicated_with_deps: + get: + summary: list dedicated worker scripts with workspace dependency annotations + operationId: listDedicatedWithDeps + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of dedicated scripts with their workspace dependency names + content: + application/json: + schema: + type: array + items: + type: object + properties: + path: + type: string + language: + type: string + enum: + - python3 + - deno + - go + - bash + - powershell + - postgresql + - mysql + - bigquery + - snowflake + - mssql + - graphql + - nativets + - bun + - bunnative + - php + - rust + - ansible + - csharp + - oracledb + - duckdb + - java + - ruby + workspace_dep_names: + type: array + items: + type: string + required: + - path + - language + - workspace_dep_names + /w/{workspace}/scripts/raw/p/{path}: get: summary: raw script by path diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs index ccf34685e5..5f8255c10a 100644 --- a/backend/windmill-common/src/ai_google.rs +++ b/backend/windmill-common/src/ai_google.rs @@ -9,7 +9,10 @@ use serde::{Deserialize, Serialize}; -use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation}; +use crate::ai_types::{ + ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, + UrlCitation, +}; use crate::error::Error; // ============================================================================ @@ -87,7 +90,10 @@ pub struct GeminiTextRequest { /// Tool definition — function declarations and/or Google Search grounding. #[derive(Serialize)] pub struct GeminiTool { - #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")] + #[serde( + rename = "functionDeclarations", + skip_serializing_if = "Option::is_none" + )] pub function_declarations: Option>, #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] pub google_search: Option, @@ -115,7 +121,10 @@ pub struct GeminiToolConfig { #[derive(Serialize)] pub struct GeminiFunctionCallingConfig { pub mode: String, - #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")] + #[serde( + rename = "allowedFunctionNames", + skip_serializing_if = "Option::is_none" + )] pub allowed_function_names: Option>, } @@ -341,10 +350,8 @@ pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec { - parse_data_url(&image_url.url).map(|(mime_type, data)| { - GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - } + parse_data_url(&image_url.url).map(|(mime_type, data)| GeminiPart::InlineData { + inline_data: GeminiInlineData { mime_type, data }, }) } // S3Objects are handled by the worker @@ -372,15 +379,12 @@ pub fn openai_messages_to_gemini( if let Some(content) = &msg.content { let parts = convert_content_to_gemini_parts(content); if !parts.is_empty() { - system_instruction = - Some(GeminiContentMessage { role: None, parts }); + system_instruction = Some(GeminiContentMessage { role: None, parts }); } } } "tool" => { - if let (Some(tool_call_id), Some(content)) = - (&msg.tool_call_id, &msg.content) - { + if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) { let func_name = find_gemini_function_name(messages, tool_call_id); let response_text = match content { OpenAIContent::Text(text) => text.clone(), @@ -435,10 +439,8 @@ pub fn openai_messages_to_gemini( } if !parts.is_empty() { - contents.push(GeminiContentMessage { - role: Some(gemini_role.to_string()), - parts, - }); + contents + .push(GeminiContentMessage { role: Some(gemini_role.to_string()), parts }); } } } @@ -469,10 +471,8 @@ pub fn openai_tools_to_gemini( .collect(); if !declarations.is_empty() { - gemini_tools.push(GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }); + gemini_tools + .push(GeminiTool { function_declarations: Some(declarations), google_search: None }); } if has_websearch { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 8b20d331c0..b1ba371ea2 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1531,6 +1531,16 @@ pub fn dedicated_worker_tag(workspace_id: &str, path: &str) -> String { ) } +/// Configuration for a runner group — a single long-lived subprocess +/// that can execute multiple scripts sharing the same workspace dependency. +/// Auto-detected from script content annotations at worker startup. +#[derive(Clone, PartialEq, Debug)] +pub struct RunnerGroupConfig { + pub workspace_id: String, + pub dep_name: String, + pub language: String, +} + pub async fn load_worker_config( db: &DB, killpill_tx: KillpillSender, @@ -1638,7 +1648,7 @@ pub async fn load_worker_config( let worker_tags = config .worker_tags .or_else(|| { - // Check for multiple dedicated workers first + // Check for multiple dedicated workers if let Some(ref dws) = dedicated_workers.as_ref() { let mut dedi_tags: Vec = dws .iter() diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 8e83eae1da..ef2bf66513 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -899,6 +899,8 @@ pub enum DedicatedWorkerResult { Start, /// Worker returned a successful result Success(serde_json::Value), + /// Worker returned preprocessed args (from exec_preprocess) + PreprocessedArgs(serde_json::Value), /// Worker returned an error result Error(serde_json::Value), /// Line is not a protocol message (e.g., logs) @@ -908,6 +910,7 @@ pub enum DedicatedWorkerResult { /// Parse a line from dedicated worker stdout according to the protocol: /// - "start" -> Ready signal /// - "wm_res[success]:JSON" -> Success with result +/// - "wm_res[preprocessed_args]:JSON" -> Preprocessed args /// - "wm_res[error]:JSON" -> Error with details /// - anything else -> Other (logs) pub fn parse_dedicated_worker_line(line: &str) -> DedicatedWorkerResult { @@ -922,6 +925,13 @@ pub fn parse_dedicated_worker_line(line: &str) -> DedicatedWorkerResult { } } + if let Some(json_str) = line.strip_prefix("wm_res[preprocessed_args]:") { + match serde_json::from_str(json_str) { + Ok(value) => return DedicatedWorkerResult::PreprocessedArgs(value), + Err(_) => return DedicatedWorkerResult::Other(line.to_string()), + } + } + if let Some(json_str) = line.strip_prefix("wm_res[error]:") { match serde_json::from_str(json_str) { Ok(value) => return DedicatedWorkerResult::Error(value), diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index f2a00de0cf..d03f19b6ba 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -21,7 +21,7 @@ const p = { // On Windows, normalize path to POSIX format to match args.path from Bun's resolver const cdirPosix = cdir.replace(/\\/g, "/").replace(/^[a-zA-Z]:/, ""); const filterResolve = new RegExp( - `^(?!\\.\/main\\.ts)(?!${cdir}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + `^(?!\\.\/main\\.ts)(?!\\.\/_wm_)(?!${cdir}\/main\\.ts)(?!${cdir}\/_wm_)(?!${cdirPosix}\/main\\.ts)(?!${cdirPosix}\/_wm_)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` ); let cdirNodeModules = `${cdir}/node_modules/`; diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index 7d14ffcbc3..e89b70c26c 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -27,7 +27,7 @@ const p = { const cdirFwd = cdir.replace(/\\/g, "/"); const cdirPosix = cdirFwd.replace(/^[a-zA-Z]:/, ""); const filterResolve = new RegExp( - `^(?!\\.\/main\\.ts)(?!${cdirFwd}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + `^(?!\\.\/main\\.ts)(?!\\.\/_wm_)(?!${cdirFwd}\/main\\.ts)(?!${cdirFwd}\/_wm_)(?!${cdirPosix}\/main\\.ts)(?!${cdirPosix}\/_wm_)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` ); let cdirNodeModules = `${cdirFwd}/node_modules/`; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7dbaa6723d..6bc6cacc32 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -77,95 +77,236 @@ pub const EMPTY_FILE: &str = ""; /// Bun args for dedicated worker (without the script path) pub const BUN_DEDICATED_WORKER_ARGS: &[&str] = &["run", "-i", "--prefer-offline"]; -/// Generate the dedicated worker wrapper content. -/// - `arg_names`: The argument names for the main function (e.g., ["x", "y"]) -/// - `main_import`: The import path for the main module (e.g., "./main.ts") -/// - `date_conversions`: Optional date conversion statements for Datetime args -/// - `preprocessor_spread`: If the script has a preprocessor function, the comma-separated arg names for it -pub fn generate_dedicated_worker_wrapper( - arg_names: &[&str], - main_import: &str, - date_conversions: Option<&str>, - preprocessor_spread: Option<&str>, -) -> String { - let spread = arg_names.join(","); - let dates = date_conversions.unwrap_or(""); +/// Pre-computed codegen data for a TypeScript/Bun/Deno script. +/// Computed in Rust from the parsed signature, then baked into the wrapper template. +#[cfg(any(feature = "private", test))] +pub struct TsScriptCodegen { + pub spread: String, + pub date_conversions: String, + pub preprocessor_spread: Option, + pub preprocessor_date_conversions: Option, +} + +/// Parse a TS script and compute the codegen data (arg spread, date conversions, preprocessor). +/// This is the same logic that was used on main in `start_worker`. +#[cfg(any(feature = "private", test))] +pub fn compute_ts_codegen(content: &str) -> TsScriptCodegen { + let sig = + windmill_parser_ts::parse_deno_signature(content, true, false, None).unwrap_or_default(); + let arg_names: Vec<&str> = sig.args.iter().map(|a| a.name.as_str()).collect(); + let spread = arg_names.join(", "); + + let dates = sig + .args + .iter() + .filter(|a| matches!(a.typ, Typ::Datetime)) + .map(|a| { + format!( + "{name} = {name} ? new Date({name}) : undefined", + name = a.name + ) + }) + .join("\n "); + + let pre_sig = windmill_parser_ts::parse_deno_signature( + content, + true, + false, + Some("preprocessor".to_string()), + ) + .ok() + .filter(|s| !s.args.is_empty()); + + let preprocessor_spread = pre_sig + .as_ref() + .map(|s| s.args.iter().map(|a| a.name.as_str()).join(", ")); + let preprocessor_date_conversions = pre_sig.as_ref().map(|s| { + s.args + .iter() + .filter(|a| matches!(a.typ, Typ::Datetime)) + .map(|a| { + format!( + "{name} = {name} ? new Date({name}) : undefined", + name = a.name + ) + }) + .join("\n ") + }); + + TsScriptCodegen { + spread, + date_conversions: dates, + preprocessor_spread, + preprocessor_date_conversions, + } +} + +/// Script entry for the unified wrapper generator. +/// `import_name`: the file stem used in the import path (e.g., "main" → `./main.ts`, or "f__script" → `./f__script.ts`) +#[cfg(any(feature = "private", test))] +pub struct TsScriptEntry<'a> { + pub import_name: &'a str, + pub original_path: &'a str, + pub codegen: &'a TsScriptCodegen, +} + +/// Generate a wrapper for dedicated workers and runner groups. +/// All scripts are baked in at codegen time with static imports and inline arg handling. +/// Protocol: +/// exec:: -> wm_res[success]: | wm_res[error]: +/// exec_preprocess:: -> wm_res[preprocessed_args]: then wm_res[success]: | wm_res[error]: +/// end -> exit +#[cfg(any(feature = "private", test))] +pub fn generate_multi_script_wrapper(scripts: &[TsScriptEntry<'_>], ext: &str) -> String { let is_debug = std::env::var("RUST_LOG").is_ok_and(|x| x == "windmill=debug"); let print_lines = if is_debug { - r#"console.log(line);"# + r#"console.log("[debug] " + line);"# } else { "" }; - let preprocessor_logic = if let Some(pre_spread) = preprocessor_spread { - format!( + let imports: String = scripts + .iter() + .enumerate() + .map(|(i, e)| { + format!( + "import * as _s{i} from \"./{import_name}.{ext}\";", + import_name = e.import_name + ) + }) + .collect::>() + .join("\n"); + + // Generate per-script getArgs / getPreArgs functions + let mut functions = String::new(); + let mut registrations = String::new(); + + for (i, entry) in scripts.iter().enumerate() { + let cg = entry.codegen; + let spread = &cg.spread; + let dates = &cg.date_conversions; + + functions.push_str(&format!( r#" - if (rawLine.startsWith("preprocess:")) {{ - const preInput = rawLine.slice("preprocess:".length); - const parsedArgs = JSON.parse(preInput); - if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }})); - continue; - }} - try {{ - function preArgsObjToArr({{ {pre_spread} }}) {{ - return [ {pre_spread} ]; - }} - const preprocessedArgs = await Main.preprocessor(...preArgsObjToArr(parsedArgs)); - console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value)); - // Now call main with preprocessed args - const mainArgs = getArgs(JSON.stringify(preprocessedArgs ?? {{}})); - const res = await Main.main(...mainArgs); - console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); - }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: rawLine }})); - }} - continue; - }}"# - ) - } else { - String::new() - }; +function getArgs_{i}(line) {{ + let {{ {spread} }} = JSON.parse(line); + {dates} + return [ {spread} ]; +}} +"# + )); + + let pre_fn = if let Some(ref pre_spread) = cg.preprocessor_spread { + let pre_dates = cg.preprocessor_date_conversions.as_deref().unwrap_or(""); + functions.push_str(&format!( + r#" +function getPreArgs_{i}(line) {{ + let {{ {pre_spread} }} = JSON.parse(line); + {pre_dates} + return [ {pre_spread} ]; +}} +"# + )); + format!("getPreArgs_{i}") + } else { + "null".to_string() + }; + + registrations.push_str(&format!( + "scripts.set(\"{path}\", {{ module: _s{i}, getArgs: getArgs_{i}, getPreArgs: {pre_fn} }});\n", + path = entry.original_path, + )); + } format!( r#" -import * as Main from "{main_import}"; +{imports} import * as Readline from "node:readline" BigInt.prototype.toJSON = function () {{ return this.toString(); }}; -console.log('start'); +const scripts = new Map(); +{functions} +{registrations} -function getArgs(line) {{ - let {{ {spread} }} = JSON.parse(line) - {dates} - return [ {spread} ]; -}} +console.log('start'); for await (const line of Readline.createInterface({{ input: process.stdin }})) {{ {print_lines} - const rawLine = line; - if (rawLine === "end") {{ + if (line === "end") {{ process.exit(0); }} - {preprocessor_logic} - try {{ - const args = getArgs(rawLine); - const res = await Main.main(...args); - console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); - }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: rawLine }})); + + if (line.startsWith("exec_preprocess:")) {{ + const rest = line.slice("exec_preprocess:".length); + const colonIdx = rest.indexOf(":"); + if (colonIdx === -1) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Malformed exec_preprocess command: missing colon separator", name: "Error" }})); + continue; + }} + const scriptPath = rest.slice(0, colonIdx); + const argsJson = rest.slice(colonIdx + 1); + + const entry = scripts.get(scriptPath); + if (!entry) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Script not found: " + scriptPath, name: "Error" }})); + continue; + }} + + try {{ + if (!entry.getPreArgs) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }})); + continue; + }} + const preArgs = entry.getPreArgs(argsJson); + const preprocessedArgs = await entry.module.preprocessor(...preArgs); + console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value)); + const mainArgs = entry.getArgs(JSON.stringify(preprocessedArgs ?? {{}})); + const res = await entry.module.main(...mainArgs); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: argsJson }})); + }} + continue; }} + + if (line.startsWith("exec:")) {{ + const rest = line.slice("exec:".length); + const colonIdx = rest.indexOf(":"); + if (colonIdx === -1) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Malformed exec command: missing colon separator", name: "Error" }})); + continue; + }} + const scriptPath = rest.slice(0, colonIdx); + const argsJson = rest.slice(colonIdx + 1); + + const entry = scripts.get(scriptPath); + if (!entry) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Script not found: " + scriptPath, name: "Error" }})); + continue; + }} + + try {{ + const args = entry.getArgs(argsJson); + const res = await entry.module.main(...args); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value)); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: argsJson }})); + }} + continue; + }} + + console.error("Unknown command:", line); }} "# ) } /// Returns (package.json, bun.lock(b), is_empty, is_binary) -fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) { +pub(crate) fn split_lockfile(lockfile: &str) -> (&str, Option<&str>, bool, bool) { if let Some(index) = lockfile.find(BUN_LOCK_SPLIT) { // Split using "\n//bun.lock\n" let (before, after_with_sep) = lockfile.split_at(index); @@ -3587,53 +3728,18 @@ pub async fn start_worker( let main_code = remove_pinned_imports(inner_content)?; let _ = write_file(job_dir, "main.ts", &main_code)?; + let codegen = compute_ts_codegen(inner_content); + let wrapper_ext = if codebase.is_some() { "js" } else { "ts" }; { - // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args; - let dates = args - .iter() - .filter_map(|x| { - if matches!(x.typ, Typ::Datetime) { - Some(x.name.clone()) - } else { - None - } - }) - .map(|x| return format!("{x} = {x} ? new Date({x}) : undefined")) - .join("\n"); - - let arg_names: Vec<&str> = args.iter().map(|x| x.name.as_str()).collect(); - - // Parse preprocessor signature if it exists - let pre_spread = windmill_parser_ts::parse_deno_signature( - inner_content, - true, - false, - Some("preprocessor".to_string()), - ) - .ok() - .filter(|sig| !sig.args.is_empty()) - .map(|sig| sig.args.into_iter().map(|x| x.name).join(",")); - - // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); - // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud - - let main_import = if codebase.is_some() { - "./main.js" - } else { - "./main.ts" - }; - let dates_opt = if dates.is_empty() { - None - } else { - Some(dates.as_str()) - }; - let wrapper_content = generate_dedicated_worker_wrapper( - &arg_names, - main_import, - dates_opt, - pre_spread.as_deref(), - ); + let scripts = + [ + TsScriptEntry { + import_name: "main", + original_path: script_path, + codegen: &codegen, + }, + ]; + let wrapper_content = generate_multi_script_wrapper(&scripts, wrapper_ext); write_file(job_dir, "wrapper.mjs", &wrapper_content)?; } @@ -3832,4 +3938,48 @@ lockfile-content"#; assert!(!is_empty); assert!(!is_binary); } + + #[test] + fn test_compute_ts_codegen_basic_args() { + let code = r#"export function main(x: string, y: number) { return x; }"#; + let cg = compute_ts_codegen(code); + assert_eq!(cg.spread, "x, y"); + assert!(cg.date_conversions.is_empty()); + assert!(cg.preprocessor_spread.is_none()); + } + + #[test] + fn test_compute_ts_codegen_with_datetime() { + let code = r#"export function main(name: string, created_at: Date, count: number) { return name; }"#; + let cg = compute_ts_codegen(code); + assert_eq!(cg.spread, "name, created_at, count"); + assert!(cg.date_conversions.contains("created_at")); + assert!(cg.date_conversions.contains("new Date")); + } + + #[test] + fn test_compute_ts_codegen_with_preprocessor() { + let code = r#" +export function main(x: string, ts: Date) { return x; } +export function preprocessor(input: string, when: Date) { return { x: input, ts: when }; } +"#; + let cg = compute_ts_codegen(code); + assert_eq!(cg.spread, "x, ts"); + assert!(cg.date_conversions.contains("ts")); + assert_eq!(cg.preprocessor_spread.as_deref(), Some("input, when")); + assert!(cg + .preprocessor_date_conversions + .as_ref() + .unwrap() + .contains("when")); + } + + #[test] + fn test_compute_ts_codegen_no_args() { + let code = r#"export function main() { return 42; }"#; + let cg = compute_ts_codegen(code); + assert!(cg.spread.is_empty()); + assert!(cg.date_conversions.is_empty()); + assert!(cg.preprocessor_spread.is_none()); + } } diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 46f42af3bf..a0fd9fdde8 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -27,6 +27,16 @@ use windmill_common::{ }; use windmill_parser::Typ; +pub const DENO_UNSTABLE_ARGS: &[&str] = &[ + "--unstable-unsafe-proto", + "--unstable-bare-node-builtins", + "--unstable-webgpu", + "--unstable-ffi", + "--unstable-fs", + "--unstable-worker-options", + "--unstable-http", +]; + lazy_static::lazy_static! { static ref DENO_FLAGS: Option> = std::env::var("DENO_FLAGS") @@ -172,22 +182,14 @@ pub async fn generate_deno_lock( let mut child_cmd = Command::new(DENO_PATH.as_str()); child_cmd .current_dir(job_dir) - .args(vec![ - "cache", - "--unstable-unsafe-proto", - "--unstable-bare-node-builtins", - "--unstable-webgpu", - "--unstable-ffi", - "--unstable-fs", - "--unstable-worker-options", - "--unstable-http", + .args(["cache"].iter().chain(DENO_UNSTABLE_ARGS).chain(&[ "--lock=lock.json", "--frozen=false", "--allow-import", "--import-map", - &import_map_path, + import_map_path.as_str(), "main.ts", - ]) + ])) .envs(deno_envs) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -442,13 +444,7 @@ try {{ args.push("--import-map"); args.push(&import_map_path); args.push(&reload); - args.push("--unstable-unsafe-proto"); - args.push("--unstable-bare-node-builtins"); - args.push("--unstable-webgpu"); - args.push("--unstable-ffi"); - args.push("--unstable-fs"); - args.push("--unstable-worker-options"); - args.push("--unstable-http"); + args.extend_from_slice(DENO_UNSTABLE_ARGS); if !*DISABLE_DENO_LOCK { if let Some(reqs) = requirements_o { @@ -547,7 +543,7 @@ try {{ read_result(job_dir, handle_result.result_stream).await } -async fn build_import_map( +pub(crate) async fn build_import_map( w_id: &str, script_path: &str, base_internal_url: &str, @@ -592,6 +588,127 @@ async fn build_import_map( #[cfg(feature = "private")] use crate::{dedicated_worker_oss::handle_dedicated_process, JobCompletedSender}; +/// Generate the dedicated worker wrapper for Deno. +/// Parses the script signature and bakes in arg destructuring, date conversions, +/// and preprocessor logic. Uses the `exec::` protocol. +#[cfg(any(feature = "private", test))] +pub fn generate_dedicated_worker_wrapper(inner_content: &str) -> Result { + let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args; + let dates = args + .iter() + .filter_map(|x| { + if matches!(x.typ, Typ::Datetime) { + Some(x.name.clone()) + } else { + None + } + }) + .map(|x| format!("{x} = {x} ? new Date({x}) : undefined")) + .join("\n"); + + let spread = args.into_iter().map(|x| x.name).join(","); + + let pre_spread = windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + Some("preprocessor".to_string()), + ) + .ok() + .filter(|sig| !sig.args.is_empty()) + .map(|sig| sig.args.into_iter().map(|x| x.name).join(",")); + + let preprocessor_import = if pre_spread.is_some() { + r#"import { preprocessor } from "./main.ts";"# + } else { + "" + }; + + let preprocessor_logic = if let Some(ref pre_spread) = pre_spread { + format!( + r#" + if (line.startsWith("exec_preprocess:")) {{ + const rest = line.slice("exec_preprocess:".length); + const colonIdx = rest.indexOf(":"); + if (colonIdx === -1) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Malformed exec_preprocess command: missing colon separator", name: "Error" }}) + '\n'); + continue; + }} + const argsJson = rest.slice(colonIdx + 1); + const parsedArgs = JSON.parse(argsJson); + if (typeof preprocessor !== 'function') {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }}) + '\n'); + continue; + }} + try {{ + function preArgsObjToArr({{ {pre_spread} }}: any) {{ + return [ {pre_spread} ]; + }} + const preprocessedArgs: any = await preprocessor(...preArgsObjToArr(parsedArgs)); + console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); + let {{ {spread} }} = preprocessedArgs ?? {{}}; + {dates} + let res: any = await main(...[ {spread} ]); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n'); + }} + continue; + }}"# + ) + } else { + String::new() + }; + + Ok(format!( + r#" +import {{ main }} from "./main.ts"; +{preprocessor_import} + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +console.log('start\n'); + +const decoder = new TextDecoder(); +for await (const chunk of Deno.stdin.readable) {{ + const lines = decoder.decode(chunk); + let exit = false; + for (const line of lines.trim().split("\n")) {{ + if (line === "end") {{ + exit = true; + break; + }} + {preprocessor_logic} + if (line.startsWith("exec:")) {{ + const rest = line.slice("exec:".length); + const colonIdx = rest.indexOf(":"); + if (colonIdx === -1) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: "Malformed exec command: missing colon separator", name: "Error" }}) + '\n'); + continue; + }} + const argsJson = rest.slice(colonIdx + 1); + try {{ + let {{ {spread} }} = JSON.parse(argsJson) + {dates} + let res: any = await main(...[ {spread} ]); + console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); + }} catch (e) {{ + console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: argsJson }}) + '\n'); + }} + continue; + }} + console.error("Unknown command:", line); + }} + if (exit) {{ + break; + }} +}} +"#, + )) +} + #[cfg(feature = "private")] use tokio::sync::mpsc::Receiver; #[cfg(feature = "private")] @@ -650,115 +767,19 @@ pub async fn start_worker( let context_envs = build_envs_map(context.to_vec()).await; { - // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature(inner_content, true, false, None)?.args; - let dates = args - .iter() - .filter_map(|x| { - if matches!(x.typ, Typ::Datetime) { - Some(x.name.clone()) - } else { - None - } - }) - .map(|x| return format!("{x} = {x} ? new Date({x}) : undefined")) - .join("\n"); - - let spread = args.into_iter().map(|x| x.name).join(","); - - // Parse preprocessor signature if it exists - let pre_spread = windmill_parser_ts::parse_deno_signature( - inner_content, - true, - false, - Some("preprocessor".to_string()), - ) - .ok() - .filter(|sig| !sig.args.is_empty()) - .map(|sig| sig.args.into_iter().map(|x| x.name).join(",")); - - let preprocessor_import = if pre_spread.is_some() { - r#"import { preprocessor } from "./main.ts";"# - } else { - "" - }; - - let preprocessor_logic = if let Some(ref pre_spread) = pre_spread { - format!( - r#" - if (line.startsWith("preprocess:")) {{ - const preInput = line.slice("preprocess:".length); - const parsedArgs = JSON.parse(preInput); - if (typeof preprocessor !== 'function') {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: "preprocessor function is missing", name: "Error" }}) + '\n'); - continue; - }} - try {{ - function preArgsObjToArr({{ {pre_spread} }}: any) {{ - return [ {pre_spread} ]; - }} - const preprocessedArgs: any = await preprocessor(...preArgsObjToArr(parsedArgs)); - console.log("wm_res[preprocessed_args]:" + JSON.stringify(preprocessedArgs ?? {{}}, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); - // Now call main with preprocessed args - let {{ {spread} }} = preprocessedArgs ?? {{}}; - {dates} - let res: any = await main(...[ {spread} ]); - console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); - }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n'); - }} - continue; - }}"# - ) - } else { - String::new() - }; - - // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); - // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud - let wrapper_content: String = format!( - r#" -import {{ main }} from "./main.ts"; -{preprocessor_import} - -BigInt.prototype.toJSON = function () {{ - return this.toString(); -}}; - -{dates} - -console.log('start\n'); - -const decoder = new TextDecoder(); -for await (const chunk of Deno.stdin.readable) {{ - const lines = decoder.decode(chunk); - let exit = false; - for (const line of lines.trim().split("\n")) {{ - if (line === "end") {{ - exit = true; - break; - }} - {preprocessor_logic} - try {{ - let {{ {spread} }} = JSON.parse(line) - {dates} - let res: any = await main(...[ {spread} ]); - console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); - }} catch (e) {{ - console.log("wm_res[error]:" + JSON.stringify({{ message: e.message, name: e.name, stack: e.stack, line: line }}) + '\n'); - }} - }} - if (exit) {{ - break; - }} -}} -"#, - ); + let wrapper_content = generate_dedicated_worker_wrapper(inner_content)?; write_file(job_dir, "wrapper.ts", &wrapper_content)?; } build_import_map(w_id, script_path, base_internal_url, job_dir).await?; + let import_map = format!("{job_dir}/import_map.json"); + let reload = format!("--reload={base_internal_url}"); + let wrapper = format!("{job_dir}/wrapper.ts"); + let mut deno_args = vec!["run", "--no-check", "--import-map", &import_map, &reload]; + deno_args.extend_from_slice(DENO_UNSTABLE_ARGS); + deno_args.extend_from_slice(&["-A", &wrapper]); + handle_dedicated_process( &*DENO_PATH, job_dir, @@ -766,22 +787,7 @@ for await (const chunk of Deno.stdin.readable) {{ envs, context, common_deno_proc_envs, - vec![ - "run", - "--no-check", - "--import-map", - &format!("{job_dir}/import_map.json"), - &format!("--reload={base_internal_url}"), - "--unstable-unsafe-proto", - "--unstable-bare-node-builtins", - "--unstable-webgpu", - "--unstable-ffi", - "--unstable-fs", - "--unstable-worker-options", - "--unstable-http", - "-A", - &format!("{job_dir}/wrapper.ts"), - ], + deno_args, killpill_rx, job_completed_tx, token, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 2fe56f50b9..e97ec8ff9a 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -86,12 +86,23 @@ pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZ pub use worker::*; pub use bun_executor::{ - build_loader, compute_bundle_local_and_remote_path, generate_dedicated_worker_wrapper, - get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir, - LoaderMode, BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, + build_loader, compute_bundle_local_and_remote_path, get_common_bun_proc_envs, + install_bun_lockfile, prebundle_bun_script, prepare_job_dir, LoaderMode, + BUN_DEDICATED_WORKER_ARGS, RELATIVE_BUN_BUILDER, RELATIVE_BUN_LOADER, }; -pub use deno_executor::generate_deno_lock; +#[cfg(any(feature = "private", test))] +pub use bun_executor::{ + compute_ts_codegen, generate_multi_script_wrapper, TsScriptCodegen, TsScriptEntry, +}; +#[cfg(any(feature = "private", test))] +pub use deno_executor::generate_dedicated_worker_wrapper as generate_deno_dedicated_worker_wrapper; +pub use deno_executor::{generate_deno_lock, DENO_UNSTABLE_ARGS}; pub use prepare_deps::run_prepare_deps_cli; +#[cfg(all(feature = "python", any(feature = "private", test)))] +pub use python_executor::{ + compute_py_codegen, generate_multi_script_wrapper as generate_py_multi_script_wrapper, + PyScriptCodegen, PyScriptEntry, +}; #[cfg(feature = "python")] pub use python_versions::PyV; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3b6507a7e0..78d3d54e15 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -117,7 +117,12 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver bool { + RELATIVE_IMPORT_REGEX.is_match(content) +} #[cfg(all(feature = "enterprise", feature = "parquet", unix))] use crate::global_cache::pull_from_tar; @@ -1084,6 +1089,320 @@ fn python_preprocessor_spread(sig: windmill_parser::MainArgSignature, indent: &s } } +/// Pre-computed codegen data for a Python script. +/// Computed in Rust from the parsed signature, then baked into the wrapper template. +#[cfg(any(feature = "private", test))] +pub struct PyScriptCodegen { + /// Python module directory dot notation (e.g., "f.my") + pub module_dir_dot: String, + /// Python module name / last path component (e.g., "script") + pub module_name: String, + /// Directory path for the module (e.g., "f/my") + pub dirs: String, + /// Inline Python code for type transforms (dates, bytes, etc.) + pub transforms: String, + /// Inline Python code for arg spread / filtering + pub spread: String, + /// Inline Python code for preprocessor arg spread (if applicable) + pub pre_spread: Option, +} + +/// Parse a Python script and compute the codegen data. +/// This reuses the same logic that was used on main in `prepare_wrapper`. +#[cfg(any(feature = "private", test))] +pub fn compute_py_codegen(content: &str, script_path: &str) -> PyScriptCodegen { + let dirs = compute_python_module_dir(script_path); + let last = script_path + .split("/") + .map(|x| { + if x.starts_with(|x: char| x.is_ascii_digit()) { + format!("_{}", x) + } else { + x.to_string() + } + }) + .last() + .unwrap() + .replace("-", "_") + .replace(" ", "_") + .to_lowercase(); + + let sig = windmill_parser_py::parse_python_signature(content, None, false).unwrap_or_default(); + let pre_sig = windmill_parser_py::parse_python_signature( + content, + Some("preprocessor".to_string()), + false, + ) + .ok() + .filter(|s| !s.args.is_empty()); + + let init_sig = pre_sig.as_ref().unwrap_or(&sig); + + let transforms = init_sig + .args + .iter() + .map(|x| match x.typ { + windmill_parser::Typ::Bytes => { + let name = &x.name; + format!( + "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \ + kwargs[\"{name}\"] = base64.b64decode(kwargs[\"{name}\"])\n", + ) + } + windmill_parser::Typ::Datetime => { + let name = &x.name; + format!( + "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \ + kwargs[\"{name}\"] = datetime.fromisoformat(kwargs[\"{name}\"])\n", + ) + } + windmill_parser::Typ::Date => { + let name = &x.name; + format!( + "if \"{name}\" in kwargs and kwargs[\"{name}\"] is not None:\n \ + try:\n \ + kwargs[\"{name}\"] = date.fromisoformat(kwargs[\"{name}\"])\n \ + except ValueError:\n \ + for _fmt in (\"%d-%m-%Y\", \"%m/%d/%Y\", \"%d/%m/%Y\", \"%Y/%m/%d\"):\n \ + try:\n \ + kwargs[\"{name}\"] = datetime.strptime(kwargs[\"{name}\"], _fmt).date()\n \ + break\n \ + except ValueError:\n \ + continue\n", + ) + } + _ => "".to_string(), + }) + .collect::>() + .join(""); + + let spread = if sig.star_kwargs { + "args = kwargs".to_string() + } else { + sig.args + .into_iter() + .map(|x| { + let name = &x.name; + if x.default.is_none() { + format!("args[\"{name}\"] = kwargs.get(\"{name}\")") + } else { + format!( + r#"args["{name}"] = kwargs.get("{name}") + if args["{name}"] is None: + del args["{name}"]"# + ) + } + }) + .join("\n ") + }; + + let pre_spread = pre_sig.map(|sig| python_preprocessor_spread(sig, " ")); + + let module_dir_dot = dirs.replace("/", ".").replace("-", "_"); + + PyScriptCodegen { module_dir_dot, module_name: last, dirs, transforms, spread, pre_spread } +} + +/// Script entry for the Python unified wrapper generator. +#[cfg(any(feature = "private", test))] +pub struct PyScriptEntry<'a> { + pub original_path: &'a str, + pub codegen: &'a PyScriptCodegen, +} + +/// Generate a wrapper for Python dedicated workers and runner groups. +/// All scripts are baked in at codegen time with proper Python imports and inline arg handling. +/// Protocol: +/// exec:: -> wm_res[success]: | wm_res[error]: +/// exec_preprocess:: -> wm_res[preprocessed_args]: then wm_res[success]: | wm_res[error]: +/// end -> exit +#[cfg(any(feature = "private", test))] +pub fn generate_multi_script_wrapper( + scripts: &[PyScriptEntry<'_>], + skip_result_postprocessing: bool, + any_relative_imports: bool, +) -> String { + let postprocessor = get_result_postprocessor(skip_result_postprocessing); + let res_to_json_body = python_res_to_json_body(postprocessor); + + let imports: String = scripts + .iter() + .enumerate() + .map(|(i, e)| { + format!( + "from {module_dir_dot} import {module_name} as _s{i}", + module_dir_dot = e.codegen.module_dir_dot, + module_name = e.codegen.module_name, + ) + }) + .collect::>() + .join("\n"); + + let mut functions = String::new(); + let mut registrations = String::new(); + + for (i, entry) in scripts.iter().enumerate() { + let cg = entry.codegen; + let indented_transforms = cg + .transforms + .split('\n') + .map(|line| { + if line.is_empty() { + String::new() + } else { + format!(" {}", line) + } + }) + .collect::>() + .join("\n"); + + functions.push_str(&format!( + r#" +def transform_{i}(kwargs): +{indented_transforms} + args = dict() + {spread} + for k, v in list(args.items()): + if v == '': + del args[k] + return args +"#, + spread = cg.spread + )); + + let pre_fn = if let Some(ref pre_spread) = cg.pre_spread { + functions.push_str(&format!( + r#" +def pre_transform_{i}(kwargs): + pre_args = dict() + {pre_spread} + for k, v in list(pre_args.items()): + if v == '': + del pre_args[k] + return pre_args +"#, + )); + format!("pre_transform_{i}") + } else { + "None".to_string() + }; + + registrations.push_str(&format!( + "scripts[\"{path}\"] = {{ 'mod': _s{i}, 'transform': transform_{i}, 'pre_transform': {pre_fn} }}\n", + path = entry.original_path, + )); + } + + let import_loader = if any_relative_imports { + "import loader" + } else { + "" + }; + + format!( + r#" +import json +import sys +import traceback +import re +import base64 +from datetime import datetime, date +{import_loader} + +{imports} + +scripts = {{}} + +def to_b_64(v: bytes): + b64 = base64.b64encode(v) + return b64.decode('ascii') + +replace_invalid_fields = re.compile(r'(?:\bNaN\b|\\u0000|Infinity|\-Infinity)') + +def res_to_json(res, typ): +{res_to_json_body} +{functions} +{registrations} + +sys.stdout.write('start\n') +sys.stdout.flush() + +for line in sys.stdin: + line = line.strip() + if line == 'end': + break + + if line.startswith('exec_preprocess:'): + try: + rest = line[len('exec_preprocess:'):] + colon_idx = rest.index(':') + script_path = rest[:colon_idx] + args_json = rest[colon_idx + 1:] + + entry = scripts.get(script_path) + if not entry: + err_json = json.dumps({{ "message": "Script not found: " + script_path, "name": "Error" }}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + + mod = entry['mod'] + if not hasattr(mod, 'preprocessor') or not callable(mod.preprocessor): + err_json = json.dumps({{"message": "preprocessor function is missing", "name": "Error"}}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + kwargs = json.loads(args_json, strict=False) + pre_args = entry['pre_transform'](kwargs) + preprocessed = mod.preprocessor(**pre_args) + preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") + main_args = entry['transform'](preprocessed if preprocessed else {{}}) + res = mod.main(**main_args) + typ = type(res) + res_json = res_to_json(res, typ) + sys.stdout.write("wm_res[success]:" + res_json + "\n") + except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + + if line.startswith('exec:'): + try: + rest = line[len('exec:'):] + colon_idx = rest.index(':') + script_path = rest[:colon_idx] + args_json = rest[colon_idx + 1:] + + entry = scripts.get(script_path) + if not entry: + err_json = json.dumps({{ "message": "Script not found: " + script_path, "name": "Error" }}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + + kwargs = json.loads(args_json, strict=False) + args = entry['transform'](kwargs) + res = entry['mod'].main(**args) + typ = type(res) + res_json = res_to_json(res, typ) + sys.stdout.write("wm_res[success]:" + res_json + "\n") + except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '') + sys.stdout.write("wm_res[error]:" + err_json + "\n") + sys.stdout.flush() + continue + + sys.stderr.write("Unknown command: " + line + "\n") +"# + ) +} + async fn prepare_wrapper( job_dir: &str, job_flow_step_id: Option<&str>, @@ -1303,7 +1622,7 @@ async fn replace_pip_secret( } } -async fn handle_python_deps( +pub(crate) async fn handle_python_deps( job_dir: &str, requirements_o: Option<&String>, inner_content: &str, @@ -2419,6 +2738,22 @@ pub async fn start_worker( use crate::PyV; tracing::info!("script path: {}", script_path); + let codegen = compute_py_codegen(inner_content, script_path); + + // Write script to proper module path (e.g., f/my/script.py) + let module_dir = format!("{}/{}", job_dir, codegen.dirs); + tokio::fs::create_dir_all(&module_dir).await?; + write_file( + &module_dir, + &format!("{}.py", codegen.module_name), + inner_content, + )?; + + let any_relative_imports = RELATIVE_IMPORT_REGEX.is_match(inner_content); + if any_relative_imports { + let _ = write_file(job_dir, "loader.py", RELATIVE_PYTHON_LOADER)?; + } + let mut mem_peak: i32 = 0; let mut canceled_by: Option = None; let context = variables::get_reserved_variables( @@ -2463,122 +2798,12 @@ pub async fn start_worker( ) .await?; - let ( - import_loader, - import_base64, - import_datetime, - module_dir_dot, - _dirs, - last, - transforms, - spread, - _, - _, - ) = prepare_wrapper(job_dir, None, None, None, inner_content, script_path).await?; - - // Parse preprocessor signature if the script has one - let pre_spread = windmill_parser_py::parse_python_signature( - inner_content, - Some("preprocessor".to_string()), - false, - ) - .ok() - .filter(|sig| !sig.args.is_empty()) - .map(|sig| python_preprocessor_spread(sig, " ")); - { - let postprocessor = get_result_postprocessor(annotations.skip_result_postprocessing); - let indented_transforms = transforms - .lines() - .map(|x| format!(" {}", x)) - .collect::>() - .join("\n"); - - let preprocessor_logic = if let Some(ref pre_spread) = pre_spread { - format!( - r#" - if line.startswith('preprocess:'): - pre_input = line[len('preprocess:'):] - kwargs = json.loads(pre_input, strict=False) - if not hasattr(inner_script, 'preprocessor') or not callable(inner_script.preprocessor): - err_json = json.dumps({{"message": "preprocessor function is missing", "name": "Error"}}, separators=(',', ':'), default=str).replace('\n', '') - sys.stdout.write("wm_res[error]:" + err_json + "\n") - sys.stdout.flush() - continue - try: - pre_args = {{}} - {pre_spread} - for k, v in list(pre_args.items()): - if v == '': - del pre_args[k] - preprocessed_kwargs = inner_script.preprocessor(**pre_args) - preprocessed_json = json.dumps(preprocessed_kwargs, separators=(',', ':'), default=str).replace('\n', '') - sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") - transform_and_run(preprocessed_kwargs) - except BaseException as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - tb = traceback.format_tb(exc_traceback) - err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '') - sys.stdout.write("wm_res[error]:" + err_json + "\n") - sys.stdout.flush() - continue -"# - ) - } else { - String::new() - }; - - let res_to_json_body = python_res_to_json_body(postprocessor); - let wrapper_content: String = format!( - r#" -import json -{import_loader} -{import_base64} -{import_datetime} -import traceback -import sys -from {module_dir_dot} import {last} as inner_script -import re - - -def to_b_64(v: bytes): - import base64 - b64 = base64.b64encode(v) - return b64.decode('ascii') - -def res_to_json(res, typ): -{res_to_json_body} - -def transform_and_run(kwargs): - args = {{}} -{indented_transforms} - {spread} - for k, v in list(args.items()): - if v == '': - del args[k] - res = inner_script.main(**args) - typ = type(res) - res_json = res_to_json(res, typ) - sys.stdout.write("wm_res[success]:" + res_json + "\n") - -replace_invalid_fields = re.compile(r'(?:\bNaN\b|\\u0000|Infinity|\-Infinity)') -sys.stdout.write('start\n') - -for line in sys.stdin: - if line == 'end\n': - break - line = line.strip() - {preprocessor_logic} - kwargs = json.loads(line, strict=False) - try: - transform_and_run(kwargs) - except BaseException as e: - exc_type, exc_value, exc_traceback = sys.exc_info() - tb = traceback.format_tb(exc_traceback) - err_json = json.dumps({{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}, separators=(',', ':'), default=str).replace('\n', '') - sys.stdout.write("wm_res[error]:" + err_json + "\n") - sys.stdout.flush() -"#, + let scripts = [PyScriptEntry { original_path: script_path, codegen: &codegen }]; + let wrapper_content = generate_multi_script_wrapper( + &scripts, + annotations.skip_result_postprocessing, + any_relative_imports, ); write_file(job_dir, "wrapper.py", &wrapper_content)?; } @@ -2632,6 +2857,7 @@ for line in sys.stdin: &mut None, ) .await?; + handle_dedicated_process( &python_path, job_dir, @@ -2704,4 +2930,43 @@ mod tests { // @ is replaced with . assert_eq!(compute_python_module_dir("u/@admin/script"), "u/.admin"); } + + #[test] + fn test_compute_py_codegen_basic_args() { + let code = "def main(x: str, y: int):\n return x\n"; + let cg = compute_py_codegen(code, "f/test/script"); + assert!(cg.spread.contains("args[\"x\"]")); + assert!(cg.spread.contains("args[\"y\"]")); + assert!(cg.transforms.is_empty()); + assert!(cg.pre_spread.is_none()); + assert_eq!(cg.module_name, "script"); + } + + #[test] + fn test_compute_py_codegen_with_datetime_and_bytes() { + let code = "import datetime\n\ndef main(name: str, created_at: datetime.datetime, file: bytes):\n return name\n"; + let cg = compute_py_codegen(code, "f/my/handler"); + assert!(cg.transforms.contains("datetime.fromisoformat")); + assert!(cg.transforms.contains("base64.b64decode")); + assert!(cg.spread.contains("args[\"name\"]")); + assert_eq!(cg.module_dir_dot, "f.my"); + assert_eq!(cg.module_name, "handler"); + } + + #[test] + fn test_compute_py_codegen_star_kwargs() { + let code = "def main(**kwargs):\n return kwargs\n"; + let cg = compute_py_codegen(code, "f/test/star"); + assert_eq!(cg.spread, "args = kwargs"); + } + + #[test] + fn test_compute_py_codegen_with_preprocessor() { + let code = "import datetime\n\ndef main(x: str, ts: datetime.datetime):\n return x\n\ndef preprocessor(input: str, when: datetime.datetime):\n return {\"x\": input, \"ts\": when}\n"; + let cg = compute_py_codegen(code, "f/test/pre"); + assert!(cg.spread.contains("args[\"x\"]")); + assert!(cg.pre_spread.is_some()); + let pre = cg.pre_spread.as_ref().unwrap(); + assert!(pre.contains("pre_args[\"input\"]")); + } } diff --git a/frontend/src/lib/components/DedicatedWorkersSelector.svelte b/frontend/src/lib/components/DedicatedWorkersSelector.svelte index 1ed9d36a7c..94086982fb 100644 --- a/frontend/src/lib/components/DedicatedWorkersSelector.svelte +++ b/frontend/src/lib/components/DedicatedWorkersSelector.svelte @@ -1,6 +1,22 @@ +{#snippet depBadge(dep: string)} + {#if existingDeps.has(dep)} + + {dep} + + + {:else} + + Workspace dependency '{dep}' not found. Create it in workspace settings to enable shared + runners. + + + + {dep} + + {/if} +{/snippet} + +{#snippet tagRow(tag: string, info: SelectedTagInfo | undefined)} +
    +
    + {#if info?.type === 'flow' && info.runners && info.runners.length > 0} + + {:else} +
    + {/if} +
    + {#if info} + {#if info.type === 'flow'} + + {:else} + + {/if} + {info.path} + ({info.workspace}) + + {#if !tagRunnerGroup.has(tag)} + {#if info.workspaceDeps} + {#each info.workspaceDeps as dep} + {@render depBadge(dep)} + {/each} + {/if} + {#if info.type === 'flow' && info.runners} + + {info.runners.length} runner{info.runners.length !== 1 ? 's' : ''} + + {:else if info.language} + {info.language} + {/if} + {/if} + {:else} + {tag} + {/if} +
    + {#if !disabled} + + {/if} +
    + + {#if info?.type === 'flow' && info.expanded && info.runners} +
    + {#each info.runners as runner (runner.stepId)} +
    + {runner.stepId} + {#if runner.stepSummary} + {runner.stepSummary} + {/if} + + {runner.isInline ? runner.language : runner.scriptPath} + +
    + {/each} +
    + {/if} +
    +{/snippet} +
    - {#if selectedTags.length > 0}
    -
    - {#each selectedTags as tag (tag)} - {@const info = selectedTagsInfo.get(tag)} +
    + + {#each runnerGroups as group (`${group.depName}:${group.language}`)}
    -
    - {#if info?.type === 'flow' && info.runners && info.runners.length > 0} - - {:else} -
    - {/if} -
    -
    - {#if info} - {#if info.type === 'flow'} - - {:else} - - {/if} - {info.path} - ({info.workspace}) - {#if info.type === 'flow' && info.runners} - - {info.runners.length} runner{info.runners.length !== 1 ? 's' : ''} - - {:else if info.type === 'script'} - 1 runner - {/if} - {:else} - {tag} - {/if} -
    -
    - {#if !disabled} - - {/if} +
    + + Shared runner + + {@render depBadge(group.depName)} + {group.language}
    +
    + {#each group.tags as tag (tag)} + {@const info = selectedTagsInfo.get(tag)} + {@render tagRow(tag, info)} + {/each} +
    +
    + {/each} - {#if info?.type === 'flow' && info.expanded && info.runners} -
    - {#each info.runners as runner (runner.stepId)} -
    - {runner.stepId} - {#if runner.stepSummary} - {runner.stepSummary} - {/if} - - {runner.isInline ? runner.language : runner.scriptPath} - -
    - {/each} -
    + + {#each standaloneTags as tag (tag)} + {@const info = selectedTagsInfo.get(tag)} +
    + {#if info?.type === 'flow'} + + {:else} + + {/if} + {info?.path ?? tag} + ({info?.workspace ?? ''}) + + {#if info?.workspaceDeps} + {#each info.workspaceDeps as dep} + {@render depBadge(dep)} + {/each} + {/if} + {#if info?.type === 'flow' && info.runners} + + {info.runners.length} runner{info.runners.length !== 1 ? 's' : ''} + + {:else if info?.language} + {info.language} + {/if} + {#if !disabled} + {/if}
    {/each} @@ -585,15 +824,18 @@ {/if}
    - {runnable.displayName} + {runnable.displayName} {#if runnable.type === 'flow' && runnable.runners} {runnable.runners.length} {/if} - + {#if runnable.workspaceDeps} + {#each runnable.workspaceDeps as dep} + {@render depBadge(dep)} + {/each} + {/if} + {runnable.type === 'flow' ? 'flow' : runnable.language} From 7f48704cfdbc2b2b0391f9643a35ed1d7d49c641 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 15:50:45 +0000 Subject: [PATCH 46/48] add missing grants on app_bundles for windmill_user and windmill_admin (#8527) Co-authored-by: Claude Opus 4.6 (1M context) --- backend/migrations/20260325000000_app_bundles_grants.down.sql | 3 +++ backend/migrations/20260325000000_app_bundles_grants.up.sql | 3 +++ 2 files changed, 6 insertions(+) create mode 100644 backend/migrations/20260325000000_app_bundles_grants.down.sql create mode 100644 backend/migrations/20260325000000_app_bundles_grants.up.sql diff --git a/backend/migrations/20260325000000_app_bundles_grants.down.sql b/backend/migrations/20260325000000_app_bundles_grants.down.sql new file mode 100644 index 0000000000..92e34705fe --- /dev/null +++ b/backend/migrations/20260325000000_app_bundles_grants.down.sql @@ -0,0 +1,3 @@ +-- Revoke grants for app_bundles table +REVOKE ALL ON app_bundles FROM windmill_user; +REVOKE ALL ON app_bundles FROM windmill_admin; diff --git a/backend/migrations/20260325000000_app_bundles_grants.up.sql b/backend/migrations/20260325000000_app_bundles_grants.up.sql new file mode 100644 index 0000000000..2ad56563db --- /dev/null +++ b/backend/migrations/20260325000000_app_bundles_grants.up.sql @@ -0,0 +1,3 @@ +-- Add grants for app_bundles table +GRANT ALL ON app_bundles TO windmill_user; +GRANT ALL ON app_bundles TO windmill_admin; From 0bd756839c0261f255111d62088bdaaecb838085 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 25 Mar 2026 18:10:20 +0100 Subject: [PATCH 47/48] feat: SCIM user deprovisioning (active:false) + instance-level user disable (#8484) * [ee] feat: handle active:false in SCIM user PATCH/PUT for deprovisioning Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref for SCIM active:false deprovision fix Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * nit sqlx * [ee] feat: add password.disabled column for SCIM user deactivation Co-Authored-By: Claude Opus 4.6 (1M context) * [ee] feat: enforce password.disabled in auth checks Co-Authored-By: Claude Opus 4.6 (1M context) * [ee] refactor: use scim_deactivated_user table instead of password.disabled Co-Authored-By: Claude Opus 4.6 (1M context) * [ee] fix: apply SCIM filters to deactivated users, add name column Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * chore: add down migration for scim_deactivated_user Co-Authored-By: Claude Opus 4.6 (1M context) * fix: rename migration to avoid timestamp conflict, update sqlx cache Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * [ee] refactor: use password.disabled for SCIM deactivation, block login for disabled users Co-Authored-By: Claude Opus 4.6 (1M context) * [ee] feat: show disabled toggle in superadmin user list, add disabled field to API Co-Authored-By: Claude Opus 4.6 (1M context) * feat: add confirmation modal when disabling instance user Co-Authored-By: Claude Opus 4.6 (1M context) * fix: improve disable user confirmation text Co-Authored-By: Claude Opus 4.6 (1M context) * fix: revert toggle state when disable confirmation is cancelled Co-Authored-By: Claude Opus 4.6 (1M context) * fix: properly revert toggle on disable cancel using reset key Co-Authored-By: Claude Opus 4.6 (1M context) * refactor: move disable/enable to dropdown menu, add disabled badge on email Co-Authored-By: Claude Opus 4.6 (1M context) * fix: rename 'Show active users only' to 'Recently active only' to avoid confusion with disabled state Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref Co-Authored-By: Claude Opus 4.6 (1M context) * chore: remove accidentally committed gen files Co-Authored-By: Claude Opus 4.6 (1M context) * fix: use .catch() for enable user error handling in dropdown action Co-Authored-By: Claude Opus 4.6 (1M context) * fix: delete tokens on user removal, improve confirmation modal texts Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update sqlx cache for non-enterprise code paths Co-Authored-By: Claude Opus 4.6 (1M context) * fix: restore sqlx cache files deleted by incorrect prepare run Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add missing sqlx cache for non-enterprise git sync query Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to a1274aa11a83f608eacc32c0d449ca3527d98c15 This commit updates the EE repository reference after PR #473 was merged in windmill-ee-private. Previous ee-repo-ref: 30f8c53b101b9e25107e793cdc038b0e07061739 New ee-repo-ref: a1274aa11a83f608eacc32c0d449ca3527d98c15 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- ...1410d32a8d672cfa4929e9e3763c51daa1bc.json} | 10 +- ...024d9826a328bf0416c22daf06fff5ced08f6.json | 14 +++ ...ef901cd3c417b9f3af03f35009213143bd443.json | 28 ++++++ ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json | 15 +++ ...2d14755474cba82b3b388a47585a8bb325b1a.json | 17 ---- ...5b31f0efc6d8ef73f691009c73f833dcee10.json} | 10 +- ...bdc0e1934d67d3f2b14047d434b77d370af21.json | 22 +++++ ...ac9019767074158e0c027988e5b0d51a3656.json} | 4 +- ...504b7b5cb7a39538ab9abeb44f781c711493.json} | 10 +- ...78447d0aa3d143e94e49924ff7ac8b7abf924.json | 22 +++++ backend/ee-repo-ref.txt | 2 +- ...60324000000_scim_deactivated_user.down.sql | 1 + ...0260324000000_scim_deactivated_user.up.sql | 1 + backend/windmill-api-users/src/users.rs | 32 ++++++- backend/windmill-api/openapi.yaml | 5 + .../components/SuperadminSettingsInner.svelte | 96 ++++++++++++++++--- 17 files changed, 248 insertions(+), 43 deletions(-) rename backend/.sqlx/{query-05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab.json => query-115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc.json} (83%) create mode 100644 backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json create mode 100644 backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json create mode 100644 backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json delete mode 100644 backend/.sqlx/query-9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a.json rename backend/.sqlx/{query-60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce.json => query-a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10.json} (82%) create mode 100644 backend/.sqlx/query-ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21.json rename backend/.sqlx/{query-638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6.json => query-daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656.json} (59%) rename backend/.sqlx/{query-65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9.json => query-f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493.json} (85%) create mode 100644 backend/.sqlx/query-fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924.json create mode 100644 backend/migrations/20260324000000_scim_deactivated_user.down.sql create mode 100644 backend/migrations/20260324000000_scim_deactivated_user.up.sql diff --git a/backend/.sqlx/query-05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab.json b/backend/.sqlx/query-115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc.json similarity index 83% rename from backend/.sqlx/query-05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab.json rename to backend/.sqlx/query-115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc.json index 53a3863587..d2c85b0e53 100644 --- a/backend/.sqlx/query-05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab.json +++ b/backend/.sqlx/query-115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2", + "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -57,6 +57,11 @@ "ordinal": 10, "name": "role_source", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "disabled", + "type_info": "Bool" } ], "parameters": { @@ -76,8 +81,9 @@ true, null, false, + false, false ] }, - "hash": "05027983ffdb11824190543754d0be922e1463d2046753cf80377369a90013ab" + "hash": "115a9cb44d0a41952c08dc36e0331410d32a8d672cfa4929e9e3763c51daa1bc" } diff --git a/backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json b/backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json new file mode 100644 index 0000000000..dc7c41cfd3 --- /dev/null +++ b/backend/.sqlx/query-192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE email = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "192ddae8c3c82a8f099a4944483024d9826a328bf0416c22daf06fff5ced08f6" +} diff --git a/backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json b/backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json new file mode 100644 index 0000000000..d6946e80d0 --- /dev/null +++ b/backend/.sqlx/query-23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, disabled FROM password WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "23b9c862d050b00aaa332527b62ef901cd3c417b9f3af03f35009213143bd443" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json b/backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json new file mode 100644 index 0000000000..fc86915946 --- /dev/null +++ b/backend/.sqlx/query-8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE password SET disabled = $1 WHERE email = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bool", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8bd266705fc8272f3d8941922ad7d18161eb6f5ec1ba9f1b55feffe8b6518c67" +} diff --git a/backend/.sqlx/query-9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a.json b/backend/.sqlx/query-9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a.json deleted file mode 100644 index 25a32e5338..0000000000 --- a/backend/.sqlx/query-9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, deployment_msg) VALUES ($1, $2, $3, $4) ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET deployment_msg = EXCLUDED.deployment_msg", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "9f07510019ebe6f0c5fa17bf31c2d14755474cba82b3b388a47585a8bb325b1a" -} diff --git a/backend/.sqlx/query-60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce.json b/backend/.sqlx/query-a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10.json similarity index 82% rename from backend/.sqlx/query-60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce.json rename to backend/.sqlx/query-a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10.json index 6d2382f494..dab14d9f1d 100644 --- a/backend/.sqlx/query-60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce.json +++ b/backend/.sqlx/query-a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", + "query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", "describe": { "columns": [ { @@ -57,6 +57,11 @@ "ordinal": 10, "name": "role_source", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "disabled", + "type_info": "Bool" } ], "parameters": { @@ -76,8 +81,9 @@ true, true, false, + false, false ] }, - "hash": "60118de85463098220b1c74f667b6fedb0f3f0040844c3774145e8f1f4c023ce" + "hash": "a5fd115e7be5129d623543bbfa7b5b31f0efc6d8ef73f691009c73f833dcee10" } diff --git a/backend/.sqlx/query-ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21.json b/backend/.sqlx/query-ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21.json new file mode 100644 index 0000000000..a9348dfa8b --- /dev/null +++ b/backend/.sqlx/query-ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT super_admin FROM password WHERE email = $1 AND disabled = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "super_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ccc49a2a6e11f874825365de758bdc0e1934d67d3f2b14047d434b77d370af21" +} diff --git a/backend/.sqlx/query-638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6.json b/backend/.sqlx/query-daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656.json similarity index 59% rename from backend/.sqlx/query-638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6.json rename to backend/.sqlx/query-daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656.json index d37bc73dad..5b18e66ed4 100644 --- a/backend/.sqlx/query-638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6.json +++ b/backend/.sqlx/query-daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT DO NOTHING", + "query": "INSERT INTO password (email, login_type, verified, username, name) VALUES ($1, 'saml', true, $2, $3) ON CONFLICT (email) DO UPDATE SET disabled = false", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "638d3c2ba1198dce5b5b0e47df59a92ff8011e19fbefcc3960d6f0fe167e55b6" + "hash": "daa1a6bf3d4a1001da88301932a7ac9019767074158e0c027988e5b0d51a3656" } diff --git a/backend/.sqlx/query-65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9.json b/backend/.sqlx/query-f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493.json similarity index 85% rename from backend/.sqlx/query-65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9.json rename to backend/.sqlx/query-f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493.json index 2ccf7bfdc8..c1d113ee27 100644 --- a/backend/.sqlx/query-65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9.json +++ b/backend/.sqlx/query-f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password WHERE email = $1", + "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE email = $1", "describe": { "columns": [ { @@ -57,6 +57,11 @@ "ordinal": 10, "name": "role_source", "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "disabled", + "type_info": "Bool" } ], "parameters": { @@ -75,8 +80,9 @@ true, null, false, + false, false ] }, - "hash": "65c59e224e460351c2f88261f8b1b1e7ce2bb160270b59c0f359b7952453b2b9" + "hash": "f0c9c54740cc1c0c2a6fa4e79d4d504b7b5cb7a39538ab9abeb44f781c711493" } diff --git a/backend/.sqlx/query-fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924.json b/backend/.sqlx/query-fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924.json new file mode 100644 index 0000000000..c85c557a90 --- /dev/null +++ b/backend/.sqlx/query-fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT disabled FROM password WHERE email = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "disabled", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fc6c6310ae8ac5eb351d7e2af1678447d0aa3d143e94e49924ff7ac8b7abf924" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 263ec1de9a..7a74b353a3 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -182943e5ad9bf2a905ccdf07d4e346437fb329a9 +a1274aa11a83f608eacc32c0d449ca3527d98c15 diff --git a/backend/migrations/20260324000000_scim_deactivated_user.down.sql b/backend/migrations/20260324000000_scim_deactivated_user.down.sql new file mode 100644 index 0000000000..1af6e77c62 --- /dev/null +++ b/backend/migrations/20260324000000_scim_deactivated_user.down.sql @@ -0,0 +1 @@ +ALTER TABLE password DROP COLUMN IF EXISTS disabled; diff --git a/backend/migrations/20260324000000_scim_deactivated_user.up.sql b/backend/migrations/20260324000000_scim_deactivated_user.up.sql new file mode 100644 index 0000000000..7410481bdd --- /dev/null +++ b/backend/migrations/20260324000000_scim_deactivated_user.up.sql @@ -0,0 +1 @@ +ALTER TABLE password ADD COLUMN disabled BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 75db0e2d24..8790a9c541 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -157,6 +157,7 @@ pub struct GlobalUserInfo { operator_only: Option, first_time_user: bool, role_source: String, + disabled: bool, } #[derive(Serialize, Debug)] @@ -213,6 +214,7 @@ pub struct EditUser { pub is_super_admin: Option, pub is_devops: Option, pub name: Option, + pub disabled: Option, } #[derive(Deserialize)] @@ -396,7 +398,7 @@ async fn list_users_as_super_admin( GlobalUserInfo, "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')), authors as (SELECT distinct email FROM usr WHERE usr.operator IS false) - SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source + SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled FROM password WHERE email IN (SELECT email FROM active_users) ORDER BY super_admin DESC, devops DESC @@ -409,7 +411,7 @@ async fn list_users_as_super_admin( } else { sqlx::query_as!( GlobalUserInfo, - "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ + "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \ $1 OFFSET $2", per_page as i32, offset as i32 @@ -657,7 +659,7 @@ async fn global_whoami( ) -> JsonResult { let user = sqlx::query_as!( GlobalUserInfo, - "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source FROM password WHERE \ + "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE \ email = $1", email ) @@ -680,6 +682,7 @@ async fn global_whoami( operator_only: None, first_time_user: false, role_source: "manual".to_string(), + disabled: false, })) } else { Err(user.unwrap_err()) @@ -1439,6 +1442,22 @@ async fn update_user( .await?; } + if let Some(d) = eu.disabled { + sqlx::query_scalar!( + "UPDATE password SET disabled = $1 WHERE email = $2", + d, + &email_to_update + ) + .execute(&mut *tx) + .await?; + if d { + // Delete all tokens for immediate session revocation + sqlx::query!("DELETE FROM token WHERE email = $1", &email_to_update) + .execute(&mut *tx) + .await?; + } + } + audit_log( &mut *tx, &authed, @@ -1461,6 +1480,9 @@ async fn delete_user( require_super_admin(&db, &authed.email).await?; let mut tx = db.begin().await?; + sqlx::query!("DELETE FROM token WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; sqlx::query!("DELETE FROM password WHERE email = $1", &email_to_delete) .execute(&mut *tx) .await?; @@ -1719,7 +1741,7 @@ async fn login( }; let email_w_h: Option<(String, String, bool)> = sqlx::query_as( "SELECT email, password_hash, super_admin FROM password WHERE email = $1 AND login_type = \ - 'password'", + 'password' AND disabled = false", ) .bind(&email) .fetch_optional(&mut *tx) @@ -1808,7 +1830,7 @@ async fn refresh_token( } let super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", + "SELECT super_admin FROM password WHERE email = $1 AND disabled = false", &authed.email ) .fetch_optional(&mut *tx) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index cb86c2a8b9..7b90c64d12 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -588,6 +588,8 @@ paths: type: boolean name: type: string + disabled: + type: boolean responses: "200": description: user updated @@ -23470,6 +23472,8 @@ components: role_source: type: string enum: ["manual", "instance_group"] + disabled: + type: boolean required: - email @@ -23478,6 +23482,7 @@ components: - verified - first_time_user - role_source + - disabled Flow: allOf: diff --git a/frontend/src/lib/components/SuperadminSettingsInner.svelte b/frontend/src/lib/components/SuperadminSettingsInner.svelte index 4a1d376f8c..683db57724 100644 --- a/frontend/src/lib/components/SuperadminSettingsInner.svelte +++ b/frontend/src/lib/components/SuperadminSettingsInner.svelte @@ -17,7 +17,7 @@ import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import { userStore, workspaceStore } from '$lib/stores' - import { ExternalLink, Pencil, UserMinus, UserPlus } from 'lucide-svelte' + import { Ban, CheckCircle2, ExternalLink, Pencil, UserMinus, UserPlus } from 'lucide-svelte' import DropdownV2 from './DropdownV2.svelte' import Popover from './meltComponents/Popover.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' @@ -67,6 +67,8 @@ let filteredUsers: GlobalUserInfo[] = $state([]) let deleteConfirmedCallback: (() => void) | undefined = $state(undefined) let deleteUserEmail: string = $state('') + let disableConfirmedCallback: (() => void) | undefined = $state(undefined) + let disableUserEmail: string = $state('') let editWrappers: Record = $state({}) let activeOnly = $state(false) @@ -293,9 +295,9 @@ /> @@ -347,13 +349,25 @@ {#if filteredUsers && users} - {#each filteredUsers.slice(0, nbDisplayed) as { email, super_admin, devops, login_type, name, username, operator_only, role_source }, i (email)} - - {email} + {#each filteredUsers.slice(0, nbDisplayed) as { email, super_admin, devops, login_type, name, username, operator_only, role_source, disabled }, i (email)} + + +
    + {email} + {#if disabled} + Disabled + {/if} +
    +     {#if automateUsernameCreation} {#if username} @@ -514,6 +528,39 @@ if (btn instanceof HTMLElement) btn.click() } }, + { + displayName: disabled ? 'Enable' : 'Disable', + icon: disabled ? CheckCircle2 : Ban, + action: () => { + if (!disabled) { + disableUserEmail = email + disableConfirmedCallback = async () => { + try { + await UserService.globalUserUpdate({ + email, + requestBody: { disabled: true } + }) + sendUserToast('User disabled') + listUsers(activeOnly) + } catch (e) { + sendUserToast('Failed to disable user', true) + } + } + } else { + UserService.globalUserUpdate({ + email, + requestBody: { disabled: false } + }) + .then(() => { + sendUserToast('User enabled') + listUsers(activeOnly) + }) + .catch(() => { + sendUserToast('Failed to enable user', true) + }) + } + } + }, { displayName: 'Remove', icon: UserMinus, @@ -578,6 +625,33 @@ }} >
    - Are you sure you want to remove {deleteUserEmail}? + Are you sure you want to remove {deleteUserEmail}? They will be removed from all + workspaces and instance groups, and all their sessions and tokens will be revoked. This action + is irreversible. Their workspace content (scripts, flows, apps) will not be deleted. +
    + + { + disableConfirmedCallback = undefined + listUsers(activeOnly) + }} + on:confirmed={() => { + if (disableConfirmedCallback) { + disableConfirmedCallback() + } + disableConfirmedCallback = undefined + }} +> +
    + Are you sure you want to disable {disableUserEmail}? All their active sessions and + tokens will be revoked immediately. They will be unable to log in until re-enabled. Their + workspace memberships and content will be preserved.
    From ead1ea73af59215a84fb08a58b5d30115acb529f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 25 Mar 2026 17:51:37 +0000 Subject: [PATCH 48/48] sqlx --- ...3e69e4ef8821c6cbf3b4f296b3853d95692af.json | 22 ------------------- 1 file changed, 22 deletions(-) delete mode 100644 backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json diff --git a/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json b/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json deleted file mode 100644 index a78e67067f..0000000000 --- a/backend/.sqlx/query-21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM usr WHERE workspace_id = $1 AND disabled = false", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "21f4840f60e8310d7b7efcba7483e69e4ef8821c6cbf3b4f296b3853d95692af" -}