From 34a392fed3452b7701b913d1d8699d2282d81787 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:55:47 +0100 Subject: [PATCH 01/25] add AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS env var (#8482) * feat: add AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS env var Add workspace restrictions by Azure account name, similar to the existing S3_BUCKETS_WORKSPACE_RESTRICTIONS for bucket names. Refactored parsing into a shared parse_restrictions_from_str function. Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref.txt Co-Authored-By: Claude Opus 4.6 (1M context) * chore: update ee-repo-ref to a997285e976d0642b72584e1966a70a79d84e7dc This commit updates the EE repository reference after PR #472 was merged in windmill-ee-private. Previous ee-repo-ref: 5718dc7deca18ad52ffb413813e97b8ca75805b8 New ee-repo-ref: a997285e976d0642b72584e1966a70a79d84e7dc Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-object-store/src/lib.rs | 192 +++++++++++++---------- backend/windmill-worker/src/common.rs | 8 + 3 files changed, 115 insertions(+), 87 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 27fc8090fd..820e457354 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -563877bf1c8b4184f638bab51be89b1c0aec6dad +a997285e976d0642b72584e1966a70a79d84e7dc diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 1449412a83..4caf67f348 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -65,8 +65,8 @@ pub mod object_store_reexports { pub use object_store::memory::InMemory; pub use object_store::path::Path; pub use object_store::{ - Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, - PutMultipartOpts, PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart, + Attribute, Attributes, Error as ObjectStoreError, GetResult, ObjectStore, PutMultipartOpts, + PutPayload, PutResult, Result as ObjectStoreResult, WriteMultipart, }; } @@ -530,10 +530,7 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result error::Result> { let store = object_store::local::LocalFileSystem::new_with_prefix(root_path).map_err(|e| { - error::Error::internal_err(format!( - "Error building filesystem object store: {:?}", - e - )) + error::Error::internal_err(format!("Error building filesystem object store: {:?}", e)) })?; Ok(Arc::new(store)) } @@ -644,36 +641,45 @@ lazy_static::lazy_static! { static ref S3_BUCKET_RESTRICTIONS: Option>> = { parse_bucket_restrictions() }; + static ref AZ_ACCOUNT_NAME_RESTRICTIONS: Option>> = { + parse_az_account_name_restrictions() + }; } fn parse_bucket_restrictions() -> Option>> { let env_var = std::env::var("S3_BUCKETS_WORKSPACE_RESTRICTIONS").ok()?; - parse_bucket_restrictions_from_str(&env_var) + parse_restrictions_from_str(&env_var, "S3 bucket") } -fn parse_bucket_restrictions_from_str(input: &str) -> Option>> { +fn parse_az_account_name_restrictions() -> Option>> { + let env_var = std::env::var("AZ_ACCOUNT_NAME_WORKSPACE_RESTRICTIONS").ok()?; + parse_restrictions_from_str(&env_var, "Azure account name") +} + +fn parse_restrictions_from_str(input: &str, label: &str) -> Option>> { if input.trim().is_empty() { return None; } let mut restrictions = HashMap::new(); - for bucket_rule in input.split(';') { - let bucket_rule = bucket_rule.trim(); - if bucket_rule.is_empty() { + for rule in input.split(';') { + let rule = rule.trim(); + if rule.is_empty() { continue; } - let parts: Vec<&str> = bucket_rule.splitn(2, ':').collect(); + let parts: Vec<&str> = rule.splitn(2, ':').collect(); if parts.len() != 2 { tracing::warn!( - "Invalid bucket restriction format: '{}'. Expected 'bucket:workspace1,workspace2'", - bucket_rule + "Invalid {} restriction format: '{}'. Expected 'name:workspace1,workspace2'", + label, + rule ); continue; } - let bucket_name = parts[0].trim().to_string(); + let name = parts[0].trim().to_string(); let workspaces: Vec = parts[1] .split(',') .map(|w| w.trim().to_string()) @@ -682,26 +688,33 @@ fn parse_bucket_restrictions_from_str(input: &str) -> Option Option>> { + parse_restrictions_from_str(input, "S3 bucket") +} + pub fn check_bucket_workspace_restriction( bucket_name: &str, workspace_id: &str, @@ -719,6 +732,23 @@ pub fn check_bucket_workspace_restriction( Ok(()) } +pub fn check_az_account_name_workspace_restriction( + account_name: &str, + workspace_id: &str, +) -> error::Result<()> { + if let Some(ref restrictions) = *AZ_ACCOUNT_NAME_RESTRICTIONS { + if let Some(allowed_workspaces) = restrictions.get(account_name) { + if !allowed_workspaces.contains(&workspace_id.to_string()) { + return Err(error::Error::NotAuthorized(format!( + "Workspace '{}' is not authorized to access Azure account '{}'", + workspace_id, account_name + ))); + } + } + } + Ok(()) +} + pub const DEFAULT_STORAGE: &str = "_default_"; pub fn bundle(w_id: &str, hash: &str) -> String { @@ -739,7 +769,8 @@ pub async fn upload_artifact_to_store( #[cfg(not(all(feature = "enterprise", feature = "parquet")))] let object_store: Option<()> = None; Ok( - if &windmill_common::utils::MODE_AND_ADDONS.mode == &windmill_common::utils::Mode::Standalone + if &windmill_common::utils::MODE_AND_ADDONS.mode + == &windmill_common::utils::Mode::Standalone && object_store.is_none() { let path = format!("{}/{}", standalone_dir, path); @@ -821,11 +852,11 @@ pub fn lfs_to_object_store_resource( })?; Ok(ObjectStoreResource::Gcs(gcs_resource)) } - LargeFileStorage::FilesystemStorage(fs) => Ok(ObjectStoreResource::Filesystem( - FilesystemSettings { + LargeFileStorage::FilesystemStorage(fs) => { + Ok(ObjectStoreResource::Filesystem(FilesystemSettings { root_path: fs.root_path.clone(), - }, - )), + })) + } } } @@ -995,13 +1026,9 @@ pub async fn convert_json_line_stream>( drop(file); let ctx = SessionContext::new(); - ctx.register_json( - "my_table", - path_str, - NdJsonReadOptions::default(), - ) - .await - .map_err(to_anyhow)?; + ctx.register_json("my_table", path_str, NdJsonReadOptions::default()) + .await + .map_err(to_anyhow)?; let df = ctx.sql("SELECT * FROM my_table").await.map_err(to_anyhow)?; let schema = df.schema().clone().into(); @@ -1252,8 +1279,7 @@ mod tests { advanced_permissions: None, }); // resource_value is ignored for filesystem - let result = - lfs_to_object_store_resource(&lfs, serde_json::Value::Null).unwrap(); + let result = lfs_to_object_store_resource(&lfs, serde_json::Value::Null).unwrap(); match result { ObjectStoreResource::Filesystem(fs) => { assert_eq!(fs.root_path, "/tmp/mydata"); @@ -1279,10 +1305,18 @@ mod tests { port: None, }; let result = duckdb_connection_settings_internal(s3).unwrap(); - assert!(result.connection_settings_str.contains("SET s3_region='eu-west-1'")); - assert!(result.connection_settings_str.contains("SET s3_access_key_id='AKIA123'")); - assert!(result.connection_settings_str.contains("SET s3_secret_access_key='secret456'")); - assert!(result.connection_settings_str.contains("SET s3_url_style='path'")); + assert!(result + .connection_settings_str + .contains("SET s3_region='eu-west-1'")); + assert!(result + .connection_settings_str + .contains("SET s3_access_key_id='AKIA123'")); + assert!(result + .connection_settings_str + .contains("SET s3_secret_access_key='secret456'")); + assert!(result + .connection_settings_str + .contains("SET s3_url_style='path'")); assert!(!result.connection_settings_str.contains("SET s3_use_ssl=0")); assert_eq!(result.s3_bucket, Some("test-bucket".to_string())); assert!(result.azure_container_path.is_none()); @@ -1304,7 +1338,9 @@ mod tests { }; let result = duckdb_connection_settings_internal(s3).unwrap(); assert!(result.connection_settings_str.contains("SET s3_use_ssl=0")); - assert!(!result.connection_settings_str.contains("SET s3_url_style='path'")); + assert!(!result + .connection_settings_str + .contains("SET s3_url_style='path'")); } #[test] @@ -1320,8 +1356,12 @@ mod tests { federated_token_file: None, }); let result = format_duckdb_connection_settings(resource).unwrap(); - assert!(result.connection_settings_str.contains("AccountName=myaccount")); - assert!(result.connection_settings_str.contains("AccountKey=base64key==")); + assert!(result + .connection_settings_str + .contains("AccountName=myaccount")); + assert!(result + .connection_settings_str + .contains("AccountKey=base64key==")); assert_eq!( result.azure_container_path, Some("az://mycontainer".to_string()) @@ -1337,7 +1377,10 @@ mod tests { }); let result = format_duckdb_connection_settings(resource); assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("GCS is not supported")); + assert!(result + .unwrap_err() + .to_string() + .contains("GCS is not supported")); } #[test] @@ -1361,10 +1404,8 @@ mod tests { use windmill_common::error::Error; // NotFound - let err = object_store::Error::NotFound { - path: "test/path".into(), - source: "missing".into(), - }; + let err = + object_store::Error::NotFound { path: "test/path".into(), source: "missing".into() }; let mapped = object_store_error_to_error(err); assert!(matches!(mapped, Error::NotFound(_))); @@ -1388,10 +1429,8 @@ mod tests { assert!(matches!(mapped, Error::BadRequest(_))); // Unauthenticated - let err = object_store::Error::Unauthenticated { - path: "obj".into(), - source: "no creds".into(), - }; + let err = + object_store::Error::Unauthenticated { path: "obj".into(), source: "no creds".into() }; let mapped = object_store_error_to_error(err); assert!(matches!(mapped, Error::NotAuthorized(_))); } @@ -1533,15 +1572,10 @@ mod tests { .await .unwrap(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root.to_string(), - }); - let s3_obj = S3Object { - s3: "etag.txt".to_string(), - storage: None, - filename: None, - presigned: None, - }; + let resource = + ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root.to_string() }); + let s3_obj = + S3Object { s3: "etag.txt".to_string(), storage: None, filename: None, presigned: None }; let etag = get_etag_or_empty(&resource, s3_obj).await; // LocalFileSystem should return an etag based on file metadata @@ -1554,9 +1588,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root.to_string(), - }); + let resource = + ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root.to_string() }); let s3_obj = S3Object { s3: "nonexistent.txt".to_string(), storage: None, @@ -1575,9 +1608,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap().to_string(); - let settings = ObjectSettings::Filesystem(FilesystemSettings { - root_path: root, - }); + let settings = ObjectSettings::Filesystem(FilesystemSettings { root_path: root }); let expirable = build_object_store_from_settings(settings, None) .await @@ -1585,10 +1616,7 @@ mod tests { let data = bytes::Bytes::from("end to end via settings"); expirable .store - .put( - &Path::from("e2e.txt"), - PutPayload::from(data.clone()), - ) + .put(&Path::from("e2e.txt"), PutPayload::from(data.clone())) .await .unwrap(); let result = expirable @@ -1610,9 +1638,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let root = dir.path().to_str().unwrap().to_string(); - let resource = ObjectStoreResource::Filesystem(FilesystemSettings { - root_path: root, - }); + let resource = ObjectStoreResource::Filesystem(FilesystemSettings { root_path: root }); let client = build_object_store_client(&resource).await.unwrap(); let data = bytes::Bytes::from("end to end via resource"); @@ -1634,7 +1660,10 @@ mod tests { #[test] fn test_bundle_path_format() { - assert_eq!(bundle("my_workspace", "abc123"), "script_bundle/my_workspace/abc123"); + assert_eq!( + bundle("my_workspace", "abc123"), + "script_bundle/my_workspace/abc123" + ); } #[test] @@ -1646,8 +1675,7 @@ mod tests { #[test] fn test_parse_bucket_restrictions_single_bucket() { - let result = - parse_bucket_restrictions_from_str("my-bucket:workspace1,workspace2").unwrap(); + let result = parse_bucket_restrictions_from_str("my-bucket:workspace1,workspace2").unwrap(); assert_eq!(result.len(), 1); assert_eq!( result.get("my-bucket").unwrap(), @@ -1657,19 +1685,13 @@ mod tests { #[test] fn test_parse_bucket_restrictions_multiple_buckets() { - let result = parse_bucket_restrictions_from_str( - "bucket-a:ws1,ws2;bucket-b:ws3", - ) - .unwrap(); + let result = parse_bucket_restrictions_from_str("bucket-a:ws1,ws2;bucket-b:ws3").unwrap(); assert_eq!(result.len(), 2); assert_eq!( result.get("bucket-a").unwrap(), &vec!["ws1".to_string(), "ws2".to_string()] ); - assert_eq!( - result.get("bucket-b").unwrap(), - &vec!["ws3".to_string()] - ); + assert_eq!(result.get("bucket-b").unwrap(), &vec!["ws3".to_string()]); } #[test] @@ -1681,16 +1703,14 @@ mod tests { #[test] fn test_parse_bucket_restrictions_invalid_format_skipped() { // "no-colon" is invalid, only "valid:ws1" should be parsed - let result = - parse_bucket_restrictions_from_str("no-colon;valid:ws1").unwrap(); + let result = parse_bucket_restrictions_from_str("no-colon;valid:ws1").unwrap(); assert_eq!(result.len(), 1); assert!(result.contains_key("valid")); } #[test] fn test_parse_bucket_restrictions_trailing_semicolons() { - let result = - parse_bucket_restrictions_from_str(";bucket:ws1;;").unwrap(); + let result = parse_bucket_restrictions_from_str(";bucket:ws1;;").unwrap(); assert_eq!(result.len(), 1); assert!(result.contains_key("bucket")); } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index bdecb0526f..f62964c60c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1000,6 +1000,14 @@ pub(crate) async fn get_workspace_s3_resource_path( windmill_object_store::check_bucket_workspace_restriction(bucket, workspace_id)?; } + // Check Azure account name workspace restrictions + if let ObjectStoreResource::Azure(azure_resource) = &object_store_resource { + windmill_object_store::check_az_account_name_workspace_restriction( + &azure_resource.account_name, + workspace_id, + )?; + } + Ok(Some(object_store_resource)) } From f329ee7aaefbae0ad344743c40825440a936bd30 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 23 Mar 2026 17:01:01 +0000 Subject: [PATCH 02/25] fix: respect NO_COLOR env variable for stdout log output (#8483) Co-authored-by: Claude Opus 4.6 (1M context) --- backend/windmill-common/src/tracing_init.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index f4892b67c9..3b009a36eb 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -68,7 +68,11 @@ pub fn initialize_tracing( mode: &Mode, environment: &str, ) -> (WorkerGuard, crate::otel_oss::OtelProvider) { - let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); + let style = if std::env::var("NO_COLOR").is_ok() { + "never".into() + } else { + std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()) + }; let rust_log_env = std::env::var("RUST_LOG"); let rust_log_stdout_env = std::env::var("RUST_LOG_STDOUT"); From 010753c73ac85237af50acadf9c08567b1bc993c Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 23 Mar 2026 18:59:46 +0100 Subject: [PATCH 03/25] fix: skip debounce arg accumulation when batch table is empty (CE) (#8485) On CE (without private feature), v2_job_debounce_batch is never populated because maybe_debounce_post_preprocessing is EE-only. The accumulation query returns zero rows, producing an empty array that replaces the original nodes_to_relock value. This causes flow modules to never get relocked when triggered by relative imports. Fix: only replace the original value when the batch query actually returned entries to accumulate. Co-authored-by: Claude Opus 4.6 --- backend/windmill-queue/src/jobs.rs | 69 ++++++++++++++++-------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1aff43a388..a6ba745c24 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3100,41 +3100,46 @@ impl PulledJobResult { "Accumulated arguments from debounced jobs in batch" ); - let new_value = to_raw_value(&accumulated_arg); + // If the batch query returned no entries (e.g. CE where + // v2_job_debounce_batch is never populated), keep the + // original value unchanged instead of replacing it with []. + if !accumulated_arg.is_empty() { + let new_value = to_raw_value(&accumulated_arg); - let original_value = j - .args - .as_ref() - .and_then(|a| a.get(arg_name_to_accumulate)) - .map(|v| v.get().to_string()) - .unwrap_or_else(|| "null".to_string()); + let original_value = j + .args + .as_ref() + .and_then(|a| a.get(arg_name_to_accumulate)) + .map(|v| v.get().to_string()) + .unwrap_or_else(|| "null".to_string()); - append_logs( - &j_id, - &j.workspace_id, - format!( - "Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n", - &new_value - ), - &(db.into()), - ) - .await; - - j.args - .get_or_insert(Json(Default::default())) - .as_mut() - .insert(arg_name_to_accumulate.to_owned(), new_value); - - // Persist accumulated args to v2_job so that flow steps - // re-reading from the DB (via get_mini_pulled_job) see them - if let Some(ref args) = j.args { - sqlx::query!( - "UPDATE v2_job SET args = $2 WHERE id = $1", - j_id, - args as &Json>>, + append_logs( + &j_id, + &j.workspace_id, + format!( + "Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n", + &new_value + ), + &(db.into()), ) - .execute(db) - .await?; + .await; + + j.args + .get_or_insert(Json(Default::default())) + .as_mut() + .insert(arg_name_to_accumulate.to_owned(), new_value); + + // Persist accumulated args to v2_job so that flow steps + // re-reading from the DB (via get_mini_pulled_job) see them + if let Some(ref args) = j.args { + sqlx::query!( + "UPDATE v2_job SET args = $2 WHERE id = $1", + j_id, + args as &Json>>, + ) + .execute(db) + .await?; + } } } From 9643006f1e90b991b334bb58caf62301bc26d09d Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Mon, 23 Mar 2026 19:20:19 +0100 Subject: [PATCH 04/25] feat(cli): better stale scripts detection #3 (#8480) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix Signed-off-by: pyranota * reduce tests Signed-off-by: pyranota * update Signed-off-by: pyranota * fix Signed-off-by: pyranota * update Signed-off-by: pyranota * WIP: stash changes after merge with origin/main * Delete backend/parsers/windmill-parser-wasm/Cargo.lock * reset cargo.toml * feat(cli): integrate dependency tree into generate-metadata command - Add isDirectlyStale field to DependencyNode for staleness tracking - Update addScript to accept itemType, folder, isRawApp, isDirectlyStale - Update propagateStaleness to use isDirectlyStale field instead of parameter - Handlers now determine staleness and pass it to tree.addScript - generate-metadata calls propagateStaleness() and populates staleItems from tree - Pass legacyBehaviour=false and tree to handlers during generation phase 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * fix(cli): store originalPath in tree for correct handler invocation Scripts need the path with extension to be passed to the handler. Added originalPath field to DependencyNode to track this. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 * fix parsers Signed-off-by: pyranota * rever sqlx removal * update sqlx * feat: make py-imports parser WASM-compatible and add as separate WASM package Gate heavy deps (sqlx, windmill-common, async-recursion, toml, pep440_rs, tracing) behind cfg(not(wasm32)). Make parse_code_for_imports, parse_relative_imports, NImport, and ImportPin public. Remove duplicate import_parser from parser-py (reset to origin/main). Add py-imports-parser feature to windmill-parser-wasm and py-imports target to build.nu. Co-Authored-By: Claude Opus 4.6 * safer return * update * fix: CLI metadata fixes - folder filter, staleness detection, WASM py-imports setup - Fix lazy_static cfg gating for WASM compatibility (split into separate blocks) - Fix folder argument filter to match specific file paths (not just directories) - Fix staleness detection to use checkHash with conf (includes module hashes) - Convert relative_imports_skip tests from Deno to bun APIs - Add windmill-parser-wasm-py-imports to CLI and build-npm dependencies - Relax module stale test to not require per-module change detail in output Co-Authored-By: Claude Opus 4.6 * fix: restore temp_script_refs parameter in parse_python_imports Re-adds the temp_script_refs parameter that was lost when resetting py-imports crate to origin/main. This enables resolving relative imports from not-yet-deployed scripts during CLI lock generation. * fixes * extend testsuit * update ee repo ref * fix: diff endpoint bytea cast, upload only mismatched scripts - Add POST /scripts/raw_temp/diff endpoint to batch-compare local content hashes against deployed versions using Postgres sha256() - Use convert_to(content, 'UTF8') instead of content::bytea to avoid failure on scripts containing backslash sequences (e.g. \n) - CLI now diffs all scripts against deployed, uploads only mismatched ones - propagateStaleness no longer deletes non-stale nodes (needed for diff) - Suppress verbose log.info messages during metadata generation - Add E2E tests for locally modified and unpushed helper scripts Co-Authored-By: Claude Opus 4.6 * rework * sqlx * fixes * add index * expand tests * fix flows * archive script before executing * disable tests for ci * skip Python-dependent E2E tests on CI Tests requiring the python backend feature are skipped when CI_MINIMAL_FEATURES=true since CI builds with zip-only features. Co-Authored-By: Claude Opus 4.6 * fix: make flow fixture lock optional and reset nonDottedPaths after tests Flow fixtures no longer emit an empty lock file by default. The lockContent parameter controls whether a lock: "!inline ..." line appears in flow.yaml. This prevents flows from appearing "up-to-date" when they should be processed by generate-metadata. Also adds afterAll to reset setNonDottedPaths(false) so global state doesn't leak between test files when run together. Co-Authored-By: Claude Opus 4.6 * debug: add error logging in withTestBackend to diagnose CI failures Co-Authored-By: Claude Opus 4.6 * debug: add --bail 1 to CI test runner to show full error on first failure Co-Authored-By: Claude Opus 4.6 * debug: include CLI stdout/stderr in assertion message for workspace deps test Co-Authored-By: Claude Opus 4.6 * fix: set WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES in test backend The workspace deps feature requires workers to report their version, but in test/CI there are no separate workers (standalone mode). The version check fails because workers haven't had time to ping yet. Setting this env var bypasses the version check. Also reverts --bail 1 from CI workflow now that the root cause is fixed. Co-Authored-By: Claude Opus 4.6 * debug: add --bail 1 to Windows CI and assertion messages for Windows failure diagnosis Co-Authored-By: Claude Opus 4.6 * fix: replace TEMP_SCRIPT_REFS_PLACEHOLDER in bun builder tests The loader.bun.js now includes a TEMP_SCRIPT_REFS_PLACEHOLDER that must be replaced before execution. The builder tests were missing this replacement, causing all 6 bun_builder_tests to fail. Co-Authored-By: Claude Opus 4.6 * fix: use cdirFwd in Windows loader filterLoad regex Raw cdir (with backslashes) interpolated into RegExp causes \r to become carriage return and \w to become word-char, so filterLoad never matches main.ts. This prevents replaceRelativeImports from running, leaving bare relative imports like "./script_b" in the bundled output, which scanImports then misparses as package ".". Co-Authored-By: Claude Opus 4.6 * fix: Windows filterLoad regex + graceful fallback for old backends - Fix filterLoad in loader.bun.windows.js to match both native backslash and forward-slash paths from Bun's resolver by escaping cdir for regex - Wrap uploadScripts in try/catch so generate-metadata degrades gracefully when the backend lacks /raw_temp endpoints (locks use deployed versions) - Add TODO for missing TEMP_SCRIPT_REFS support in Windows loader Co-Authored-By: Claude Opus 4.6 * debug: add loader/builder debug logging for Windows CI diagnosis Temporary console.log statements to understand: - What path Bun passes to onLoad for main.ts - Whether filterLoad regex matches - Whether replaceRelativeImports fires - What the bundled output contains - What imports scanImports extracts Co-Authored-By: Claude Opus 4.6 * chore: trigger CI for cli path Co-Authored-By: Claude Opus 4.6 * chore: trigger CI via workflow file change Co-Authored-By: Claude Opus 4.6 * Add TEMP_SCRIPT_REFS to Windows loader, use .ts extensions in test imports - Add TEMP_SCRIPT_REFS_PLACEHOLDER support to loader.bun.windows.js (mirrors loader.bun.js) so CLI lock generation can resolve imports from locally-modified scripts on Windows - Use .ts extensions in all test relative imports to work around the Windows filterLoad regex bug (replaceRelativeImports doesn't fire on Windows, so extensionless imports fail) - Remove unused uploadSucceeded variable Co-Authored-By: Claude Opus 4.6 * Remove debug logging from loader_builder.bun.js Co-Authored-By: Claude Opus 4.6 * Remove windmill-parser-wasm-py-imports from frontend package.json This dependency is only needed by the CLI, not the frontend. Co-Authored-By: Claude Opus 4.6 * debug: add temp_script_refs logging for Windows CI investigation Co-Authored-By: Claude Opus 4.6 * ci: remove --bail 1 from Windows CLI tests Co-Authored-By: Claude Opus 4.6 * fix: normalize backslashes in folder filter treePath lookup (Windows) On Windows, item.path (originalPath) uses backslashes but tree keys use forward slashes. The isRelevant filter's touchesFolder call passed the unnormalized path to traverseTransitive, which couldn't find the node. This caused cross-folder importers to be excluded from generate-metadata when a folder argument was specified. Also removes debug logging from previous commit. Co-Authored-By: Claude Opus 4.6 * Update cli-tests.yml * fix: normalize backslashes in strict-folder-boundaries warning message (Windows) Co-Authored-By: Claude Opus 4.6 * chore: update ee-repo-ref to fe8f0d1d7448464c98474d994e6492c0a45e8e38 This commit updates the EE repository reference after PR #467 was merged in windmill-ee-private. Previous ee-repo-ref: 03e6eaf950776c96b9581848a583af9ad735be60 New ee-repo-ref: fe8f0d1d7448464c98474d994e6492c0a45e8e38 Automated by sync-ee-ref workflow. * revert cli-tests.yml --------- Signed-off-by: pyranota Co-authored-by: Claude Opus 4.5 Co-authored-by: windmill-internal-app[bot] --- ...32b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json | 12 + ...40af492d4c8a8871cef972980150f319fe6ff.json | 16 + ...7b932800e33cd462a651f7e6716929ee9b6f2.json | 22 + ...a158db101f54f0908551b5a4f5e6655e122b.json} | 4 +- backend/Cargo.lock | 4 +- backend/ee-repo-ref.txt | 2 +- .../20260304000000_raw_script_temp.down.sql | 2 + .../20260304000000_raw_script_temp.up.sql | 11 + .../windmill-parser-py-imports/Cargo.toml | 20 +- .../windmill-parser-py-imports/src/lib.rs | 80 +- .../windmill-parser-py-imports/tests/tests.rs | 3 + .../parsers/windmill-parser-sql/Cargo.toml | 1 - .../parsers/windmill-parser-sql/src/lib.rs | 3 +- backend/parsers/windmill-parser-ts/src/lib.rs | 82 ++ .../parsers/windmill-parser-ts/tests/tests.rs | 84 +- .../parsers/windmill-parser-wasm/Cargo.toml | 2 + backend/parsers/windmill-parser-wasm/build.nu | 6 + backend/parsers/windmill-parser-wasm/dev.nu | 7 +- .../windmill-parser-wasm/publish-pkgs.sh | 3 + .../parsers/windmill-parser-wasm/src/lib.rs | 19 + backend/parsers/windmill-parser/src/lib.rs | 17 + backend/src/main.rs | 1 + backend/tests/bun_jobs.rs | 2 + backend/tests/nativets_dedicated.rs | 1 + backend/windmill-api-scripts/src/scripts.rs | 142 ++ backend/windmill-api/openapi.yaml | 77 + backend/windmill-api/src/jobs.rs | 12 + backend/windmill-common/src/cache.rs | 35 +- backend/windmill-types/Cargo.toml | 1 + backend/windmill-types/src/s3.rs | 17 +- backend/windmill-worker/loader.bun.js | 17 +- backend/windmill-worker/loader.bun.windows.js | 23 +- backend/windmill-worker/src/bun_executor.rs | 26 +- .../windmill-worker/src/python_executor.rs | 1 + .../windmill-worker/src/worker_lockfiles.rs | 48 +- cli/build-npm.ts | 1 + cli/bun.lock | 7 +- cli/package.json | 3 +- cli/src/commands/app/app_metadata.ts | 148 +- cli/src/commands/flow/flow.ts | 6 + cli/src/commands/flow/flow_metadata.ts | 147 +- .../generate-metadata/generate-metadata.ts | 257 ++-- cli/src/commands/script/script.ts | 4 +- cli/src/commands/sync/sync.ts | 10 +- cli/src/utils/dependency_tree.ts | 373 +++++ cli/src/utils/metadata.ts | 104 +- cli/src/utils/relative_imports.ts | 39 + cli/src/utils/resource_folders.ts | 20 + cli/test/cargo_backend.ts | 29 +- cli/test/relative_imports_skip.test.ts | 420 ++++++ cli/test/relative_imports_wasm.test.ts | 1235 +++++++++++++++++ cli/test/resource_folders_unit.test.ts | 34 + cli/test/sync_pull_push.test.ts | 3 +- cli/test/test_backend.ts | 121 +- cli/test/test_fixtures.ts | 99 +- cli/test/unified_generate_metadata.test.ts | 5 +- 56 files changed, 3565 insertions(+), 303 deletions(-) create mode 100644 backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json create mode 100644 backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json create mode 100644 backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json rename backend/.sqlx/{query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json => query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json} (50%) create mode 100644 backend/migrations/20260304000000_raw_script_temp.down.sql create mode 100644 backend/migrations/20260304000000_raw_script_temp.up.sql create mode 100644 cli/src/utils/dependency_tree.ts create mode 100644 cli/src/utils/relative_imports.ts create mode 100644 cli/test/relative_imports_skip.test.ts create mode 100644 cli/test/relative_imports_wasm.test.ts diff --git a/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json new file mode 100644 index 0000000000..3c7f23d1af --- /dev/null +++ b/backend/.sqlx/query-25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "25ac66a1022c41267df199a95f532b0f778c25fbe0f7a7f9734c1f7e536ed6ce" +} diff --git a/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json new file mode 100644 index 0000000000..10ec0a5c49 --- /dev/null +++ b/backend/.sqlx/query-2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at)\n VALUES ($1, $2, $3, NOW())\n ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bpchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2d523cd0d5b7107b15846b885fa40af492d4c8a8871cef972980150f319fe6ff" +} diff --git a/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json new file mode 100644 index 0000000000..ed1b03fd77 --- /dev/null +++ b/backend/.sqlx/query-88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT content FROM raw_script_temp WHERE hash = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "88ec0ddcc86fb67089b551ccbce7b932800e33cd462a651f7e6716929ee9b6f2" +} diff --git a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json similarity index 50% rename from backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json rename to backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json index 0946fa4006..c910be0b82 100644 --- a/backend/.sqlx/query-c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623.json +++ b/backend/.sqlx/query-96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", + "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND archived = false ORDER BY created_at DESC LIMIT 1\n ", "describe": { "columns": [ { @@ -19,5 +19,5 @@ false ] }, - "hash": "c7cae4cf872fce0a989cf89aa35929218a9d459ee1c2b36a28b110e9741ab623" + "hash": "96f6163a164b9ffb4ec52372d7fea158db101f54f0908551b5a4f5e6655e122b" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 25c4233bbd..e40478cc4b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16898,8 +16898,6 @@ dependencies = [ "async-recursion", "itertools 0.14.0", "lazy_static", - "malachite", - "malachite-bigint", "pep440_rs", "phf 0.11.3", "regex", @@ -16956,7 +16954,6 @@ dependencies = [ "serde", "serde_json", "windmill-parser", - "windmill-types", ] [[package]] @@ -17465,6 +17462,7 @@ dependencies = [ "strum 0.27.2", "tracing", "uuid", + "windmill-parser", ] [[package]] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 820e457354..f30dcc7a1f 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a997285e976d0642b72584e1966a70a79d84e7dc +fe8f0d1d7448464c98474d994e6492c0a45e8e38 diff --git a/backend/migrations/20260304000000_raw_script_temp.down.sql b/backend/migrations/20260304000000_raw_script_temp.down.sql new file mode 100644 index 0000000000..05703e4f18 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_raw_script_temp_created_at; +DROP TABLE IF EXISTS raw_script_temp; diff --git a/backend/migrations/20260304000000_raw_script_temp.up.sql b/backend/migrations/20260304000000_raw_script_temp.up.sql new file mode 100644 index 0000000000..c7b5a23754 --- /dev/null +++ b/backend/migrations/20260304000000_raw_script_temp.up.sql @@ -0,0 +1,11 @@ +-- Temporary storage for raw script content during CLI lock generation +-- Content is stored with hash as key, cleaned up after 1 week +CREATE TABLE raw_script_temp ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id), + hash CHAR(64) NOT NULL, + content TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, hash) +); + +CREATE INDEX IF NOT EXISTS idx_raw_script_temp_created_at ON raw_script_temp (created_at); diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index deea68c2ef..565e1422bb 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -13,21 +13,19 @@ regex-lite.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] regex.workspace = true - -[dependencies] -windmill-parser.workspace = true windmill-common.workspace = true -rustpython-parser.workspace = true -malachite.workspace = true -malachite-bigint.workspace = true -phf.workspace = true -itertools.workspace = true -serde_json.workspace = true -anyhow.workspace = true -lazy_static.workspace = true sqlx.workspace = true async-recursion.workspace = true toml.workspace = true serde.workspace = true pep440_rs.workspace = true tracing.workspace = true + +[dependencies] +windmill-parser.workspace = true +rustpython-parser.workspace = true +phf.workspace = true +itertools.workspace = true +serde_json.workspace = true +anyhow.workspace = true +lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 06762dbc9e..de7f51bc79 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -8,10 +8,14 @@ mod mapping; +#[cfg(not(target_arch = "wasm32"))] use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; -use std::{collections::HashMap, str::FromStr}; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; +#[cfg(not(target_arch = "wasm32"))] +use std::collections::HashMap; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -24,7 +28,9 @@ use rustpython_parser::{ text_size::TextRange, Parse, }; +#[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; +#[cfg(not(target_arch = "wasm32"))] use windmill_common::{ error::{self, to_anyhow}, worker::{ @@ -46,10 +52,14 @@ fn replace_full_import(x: &str) -> Option { FULL_IMPORTS_MAP.get(x).map(|x| (*x).to_owned()) } +#[cfg(not(target_arch = "wasm32"))] lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); - static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); static ref PKG_RE: Regex = Regex::new(r"^([^!=<>]+)(?:[!=<>]|$)").unwrap(); +} + +lazy_static! { + static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); // Regex to properly match main function definition at line start, // capturing both sync and async variants static ref DEF_MAIN_RE: Regex = Regex::new(r"(?m)^(async\s+)?def\s+main\s*\(").unwrap(); @@ -82,7 +92,7 @@ fn process_import(module: Option, path: &str, level: usize) -> Vec error::Result> { +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { let nimports = parse_code_for_imports(code, path)?; return Ok(nimports .into_iter() @@ -94,7 +104,7 @@ pub fn parse_relative_imports(code: &str, path: &str) -> error::Result error::Result> { +pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -175,7 +187,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> let code_with_fake_main = format!("{}\n\ndef main(): pass", code); let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) + anyhow::anyhow!("Error parsing code for imports: {}", e.to_string()) })?; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main @@ -256,6 +268,7 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> return Ok(nimports); } +#[cfg(not(target_arch = "wasm32"))] pub async fn parse_python_imports( code: &str, w_id: &str, @@ -264,6 +277,7 @@ pub async fn parse_python_imports( version_specifiers: &mut Vec, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -276,6 +290,7 @@ pub async fn parse_python_imports( &mut None, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -313,6 +328,7 @@ pub async fn parse_python_imports( Ok((imports, compile_error_hint)) } +#[cfg(not(target_arch = "wasm32"))] fn extract_pkg_name(requirement: &str) -> String { PKG_RE .captures(requirement) @@ -320,6 +336,7 @@ fn extract_pkg_name(requirement: &str) -> String { .unwrap_or_default() } +#[cfg(not(target_arch = "wasm32"))] #[async_recursion] async fn parse_python_imports_inner( code: &str, @@ -331,6 +348,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv: &mut Option, locked_v: &mut Option, raw_workspace_dependencies_o: &Option, + temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); @@ -494,17 +512,37 @@ async fn parse_python_imports_inner( for n in nimports.into_iter() { let mut nested = match n { NImport::Relative(rpath) => { - let code = sqlx::query_scalar!( - r#" - SELECT content FROM script WHERE path = $1 AND workspace_id = $2 - AND archived = false ORDER BY created_at DESC LIMIT 1 - "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()); + // First try to get content from temp_script_refs cache if available + let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { + tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { + Ok(content) => Some(content), + Err(e) => { + tracing::warn!("temp_script_refs hash '{}' not found in cache: {}, falling back to deployed script", hash, e); + None + } + } + } else { + None + }; + + // Use cached content if available, otherwise fall back to deployed script + let code = match code_from_cache { + Some(content) => content, + None => { + sqlx::query_scalar!( + r#" + SELECT content FROM script WHERE path = $1 AND workspace_id = $2 + AND archived = false ORDER BY created_at DESC LIMIT 1 + "#, + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()) + } + }; if already_visited.contains(&rpath) { vec![] @@ -522,6 +560,7 @@ async fn parse_python_imports_inner( path_where_annotated_pyv, locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .into_values() @@ -646,6 +685,7 @@ async fn parse_python_imports_inner( Ok(final_imports) } +#[cfg(not(target_arch = "wasm32"))] fn extract_nimports_from_content( content: &str, hm: &mut HashMap, diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index e61b4cc7db..c853c8e5b4 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -26,6 +26,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -67,6 +68,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -98,6 +100,7 @@ def main(): &mut vec![], &mut None, &None, + &None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/parsers/windmill-parser-sql/Cargo.toml b/backend/parsers/windmill-parser-sql/Cargo.toml index e84888280c..143b02a2cc 100644 --- a/backend/parsers/windmill-parser-sql/Cargo.toml +++ b/backend/parsers/windmill-parser-sql/Cargo.toml @@ -16,7 +16,6 @@ regex.workspace = true [dependencies] windmill-parser.workspace = true -windmill-types.workspace = true anyhow.workspace = true lazy_static.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 4938a8e194..c8aef3eed5 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -15,7 +15,7 @@ use std::{ iter::Peekable, str::CharIndices, }; -pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ}; +pub use windmill_parser::{s3_mode_extension, Arg, MainArgSignature, ObjectType, S3ModeFormat, Typ}; pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__"; pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__"; @@ -143,7 +143,6 @@ pub fn parse_db_resource(code: &str) -> Option { cap.map(|x| x.get(1).map(|x| x.as_str().to_string()).unwrap()) } -pub use windmill_types::s3::{s3_mode_extension, S3ModeFormat}; pub struct S3ModeArgs { pub prefix: Option, pub storage: Option, diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 53ecaf277c..125834a3ac 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -117,6 +117,12 @@ impl Visit for ImportsFinder { } } +/// Parse TypeScript/JavaScript code and extract all import paths as raw strings. +/// +/// Returns import paths exactly as written in the code (e.g., `"./module"`, `"../utils"`, `"lodash"`). +/// Does not resolve relative paths to absolute Windmill paths. +/// +/// See also: [`parse_relative_imports`] for resolved absolute paths. pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result> { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.into()); @@ -151,6 +157,82 @@ pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Resul Ok(imports) } +/// Parse TypeScript/JavaScript code and extract relative imports resolved to absolute Windmill paths. +/// +/// Takes the script's Windmill path (e.g., `"f/folder/script"`) and resolves relative imports +/// like `"./module"` or `"../utils"` to absolute paths like `"f/folder/module"` or `"f/utils"`. +/// +/// Only returns relative imports (those starting with `./`, `../`, or `/`). +/// External package imports (e.g., `"lodash"`) are filtered out. +/// +/// See also: [`parse_expr_for_imports`] for raw import strings without resolution. +/// +/// # Arguments +/// * `code` - The TypeScript/JavaScript source code +/// * `path` - The Windmill path of the script (e.g., `"f/folder/script"`) +/// +/// # Returns +/// A sorted, deduplicated list of resolved absolute Windmill paths. +/// +/// # Examples +/// ```ignore +/// // Script at "f/folder/script" with: import { x } from "../utils" +/// // Returns: ["f/utils"] +/// ``` +pub fn parse_relative_imports(code: &str, path: &str) -> anyhow::Result> { + let imports = parse_expr_for_imports(code, false)?; + let script_dir = path.rsplit_once('/').map(|(dir, _)| dir).unwrap_or(""); + + let mut resolved: Vec = imports + .into_iter() + .filter(|imp| is_relative_import(imp)) + .map(|imp| { + // Remove .ts extension if present + let imp = imp.strip_suffix(".ts").unwrap_or(&imp); + + if imp.starts_with("/") { + // Absolute path (e.g., /f/folder/script) - remove leading slash + imp[1..].to_string() + } else { + // Relative path (e.g., ./script or ../folder/script) + let combined = format!("{}/{}", script_dir, imp); + normalize_path(&combined) + } + }) + .collect(); + + resolved.sort(); + resolved.dedup(); + Ok(resolved) +} + +/// Check if an import path is a relative import (starts with `./`, `../`, or `/`) +fn is_relative_import(import_path: &str) -> bool { + import_path.starts_with("./") + || import_path.starts_with("../") + || import_path.starts_with("/") +} + +/// Normalize a path by resolving `.` and `..` components +fn normalize_path(input_path: &str) -> String { + let parts: Vec<&str> = input_path.split('/').filter(|p| !p.is_empty()).collect(); + let mut result: Vec<&str> = Vec::new(); + + for part in parts { + if part == "." { + continue; + } else if part == ".." { + if !result.is_empty() { + result.pop(); + } + } else { + result.push(part); + } + } + + result.join("/") +} + struct OutputFinder { idents: HashSet<(String, String)>, } diff --git a/backend/parsers/windmill-parser-ts/tests/tests.rs b/backend/parsers/windmill-parser-ts/tests/tests.rs index 7a685fbb77..b50169ca68 100644 --- a/backend/parsers/windmill-parser-ts/tests/tests.rs +++ b/backend/parsers/windmill-parser-ts/tests/tests.rs @@ -2,7 +2,7 @@ mod tests { use serde_json::json; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, ObjectType, Typ}; - use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports}; + use windmill_parser_ts::{parse_deno_signature, parse_expr_for_imports, parse_relative_imports}; #[test] fn test_imports_basic() { @@ -798,7 +798,7 @@ mod tests { // Test case where there are exports but no preprocessor let code = r#" export { foo, bar } from "./utils"; - + export async function main(param: string) { return param; } @@ -806,4 +806,84 @@ mod tests { let sig = parse_deno_signature(code, false, false, None).unwrap(); assert_eq!(sig.has_preprocessor, Some(false)); } + + // ========================================================================== + // Tests for parse_relative_imports + // ========================================================================== + + #[test] + fn test_relative_imports_dot() { + let code = r#" + import { helper } from "./helper"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_double_dot() { + let code = r#" + import { utils } from "../utils/helper"; + export async function main() { return utils(); } + "#; + let result = parse_relative_imports(code, "f/folder/subfolder/script").unwrap(); + assert_eq!(result, vec!["f/folder/utils/helper"]); + } + + #[test] + fn test_relative_imports_absolute_path() { + let code = r#" + import { shared } from "/f/shared/utils"; + export async function main() { return shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/shared/utils"]); + } + + #[test] + fn test_relative_imports_mixed() { + let code = r#" + import { helper } from "./helper"; + import { utils } from "../utils"; + import { shared } from "/f/shared/lib"; + import lodash from "lodash"; + export async function main() { return helper() + utils() + shared(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + // Should only include relative imports, not external packages like lodash + assert_eq!(result, vec!["f/folder/helper", "f/shared/lib", "f/utils"]); + } + + #[test] + fn test_relative_imports_with_ts_extension() { + let code = r#" + import { helper } from "./helper.ts"; + export async function main() { return helper(); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert_eq!(result, vec!["f/folder/helper"]); + } + + #[test] + fn test_relative_imports_external_only() { + let code = r#" + import lodash from "lodash"; + import { something } from "@scope/package"; + export async function main() { return lodash.map([]); } + "#; + let result = parse_relative_imports(code, "f/folder/script").unwrap(); + assert!(result.is_empty()); + } + + #[test] + fn test_relative_imports_deeply_nested() { + let code = r#" + import { a } from "../../a"; + import { b } from "../../../b"; + export async function main() { return a() + b(); } + "#; + let result = parse_relative_imports(code, "f/one/two/three/script").unwrap(); + assert_eq!(result, vec!["f/b", "f/one/a"]); + } } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 8372b1838c..895a35713e 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -40,6 +40,7 @@ java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] wac-parser = [ "dep:windmill-parser-wac"] asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"] +py-imports-parser = [ "dep:windmill-parser-py-imports"] [dependencies] anyhow.workspace = true @@ -61,6 +62,7 @@ windmill-parser-wac = { workspace = true, optional = true } windmill-parser-ts-asset = { workspace = true, optional = true } windmill-parser-py-asset = { workspace = true, optional = true } windmill-parser-sql-asset = { workspace = true, optional = true } +windmill-parser-py-imports = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index 4e08ce95d3..212c5f3a37 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -67,6 +67,12 @@ const targets = [ features: "asset-parser", env: "default", }, + { + ident: "py-imports", + desc: "Python imports" + features: "py-imports-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/dev.nu b/backend/parsers/windmill-parser-wasm/dev.nu index f4d7ef6495..ec865f36dc 100755 --- a/backend/parsers/windmill-parser-wasm/dev.nu +++ b/backend/parsers/windmill-parser-wasm/dev.nu @@ -1,6 +1,6 @@ #!/usr/bin/env nu - -# Build in debug mode specified lang parser to wasm + +# Build in debug mode specified lang parser to wasm # and perform installation to frontend def "main" [ lang: string # Example: nu @@ -9,4 +9,7 @@ def "main" [ ( cd ../../../frontend; npm install ../backend/parsers/windmill-parser-wasm/pkg-($lang) ) + ( + cd ../../../cli; bun install ../backend/parsers/windmill-parser-wasm/pkg-($lang) + ) } diff --git a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh index 80f31650f3..3ac0ceef18 100755 --- a/backend/parsers/windmill-parser-wasm/publish-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/publish-pkgs.sh @@ -36,3 +36,6 @@ popd pushd "pkg-asset" && npm publish ${args} popd + +pushd "pkg-py-imports" && npm publish ${args} +popd diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2af0a3bc9a..a2cef1b0ef 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -38,6 +38,8 @@ pub fn parse_outputs(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return raw import strings. +/// See [`parse_ts_relative_imports`] for resolved absolute paths. #[cfg(feature = "ts-parser")] #[wasm_bindgen] pub fn parse_ts_imports(code: &str) -> String { @@ -50,6 +52,15 @@ pub fn parse_ts_imports(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } +/// Parse TypeScript imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +/// See [`parse_ts_imports`] for raw import strings. +#[cfg(feature = "ts-parser")] +#[wasm_bindgen] +pub fn parse_ts_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_ts::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "bash-parser")] #[wasm_bindgen] pub fn parse_bash(code: &str) -> String { @@ -214,6 +225,14 @@ pub fn parse_assets_py(code: &str) -> String { } } +/// Parse Python imports and return relative imports resolved to absolute Windmill paths. +/// Throws JS error on parse failure. +#[cfg(feature = "py-imports-parser")] +#[wasm_bindgen] +pub fn parse_py_relative_imports(code: &str, path: &str) -> Result, String> { + windmill_parser_py_imports::parse_relative_imports(code, path).map_err(|e| e.to_string()) +} + #[cfg(feature = "ansible-parser")] #[wasm_bindgen] pub fn parse_assets_ansible(code: &str) -> String { diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index dd2eada7cf..80019d044f 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -14,6 +14,23 @@ use serde_json::Value; pub mod asset_parser; +/// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) +#[derive(Clone, Copy, Debug)] +pub enum S3ModeFormat { + Json, + Csv, + Parquet, +} + +/// Returns the file extension for the given S3 mode format +pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { + match format { + S3ModeFormat::Json => "json", + S3ModeFormat::Csv => "csv", + S3ModeFormat::Parquet => "parquet", + } +} + #[derive(Serialize, Debug, PartialEq, Default)] pub struct MainArgSignature { pub star_args: bool, diff --git a/backend/src/main.rs b/backend/src/main.rs index 6d977c2db4..17567167a3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -312,6 +312,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { "cache_init", "", &mut None, + &None, ) .await { diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index b2eb89112c..2edfef8989 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -921,6 +921,7 @@ mod dedicated_worker_protocol { "test-workspace", "f/test/script", LoaderMode::Node, + &None, )) .expect("build_loader failed"); @@ -1299,6 +1300,7 @@ mod bun_builder_tests { // Write build.js using the loader and builder constants directly // Parameters are dummy values since tests don't use Windmill relative imports let loader = RELATIVE_BUN_LOADER + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", "{}") .replace("W_ID", "test-workspace") .replace("BASE_INTERNAL_URL", "http://localhost:8000") .replace("TOKEN", "test-token") diff --git a/backend/tests/nativets_dedicated.rs b/backend/tests/nativets_dedicated.rs index f13759e5bc..1c6987a480 100644 --- a/backend/tests/nativets_dedicated.rs +++ b/backend/tests/nativets_dedicated.rs @@ -32,6 +32,7 @@ mod prewarmed_isolate_tests { "test-workspace", "f/test/script", LoaderMode::BrowserBundle, + &None, ) .await .expect("build_loader failed"); diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index fcbb159887..56de4d9892 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -239,6 +239,9 @@ pub fn workspaced_service() -> Router { "/history_update/h/:hash/p/*path", post(update_script_history), ) + // Temporary raw script storage for CLI lock generation + .route("/raw_temp/store", post(store_raw_script_temp)) + .route("/raw_temp/diff", post(diff_raw_scripts_with_deployed)) } #[derive(Serialize, FromRow)] @@ -1614,6 +1617,9 @@ struct RawScriptByPathQuery { cache_key: Option, // used specifically for python to cache folders on import success to avoid extra db calls on package fetch cache_folders: Option, + // If provided, load content from raw_script_temp table using this hash instead of deployed script. + // Used by CLI lock generation to resolve imports from not-yet-deployed scripts. + temp_script_hash: Option, } struct StringWithLength(String); @@ -1672,6 +1678,16 @@ async fn raw_script_by_path_internal( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + + // If temp_script_hash is provided, try loading from temp storage first. + // This is used by CLI lock generation to resolve imports from not-yet-deployed scripts. + // Falls back to the normal deployed script lookup if not found in temp storage. + if let Some(hash) = query.temp_script_hash { + if let Ok(content) = windmill_common::cache::raw_script_temp::load(hash, &db).await { + return Ok(content); + } + } + let cache_path = query .cache_key .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); @@ -2463,3 +2479,129 @@ async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { Ok(()) } } + +// ============================================================================ +// Temporary Raw Script Storage for CLI Lock Generation +// ============================================================================ + +/// Store raw script content temporarily for CLI lock generation. +async fn store_raw_script_temp( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(content): Json, +) -> Result> { + check_scopes(&authed, || "scripts:write".to_string())?; + + let hash = windmill_common::cache::raw_script_temp::compute_hash(&w_id, &content); + + // Store to DB + sqlx::query!( + "INSERT INTO raw_script_temp (workspace_id, hash, content, created_at) + VALUES ($1, $2, $3, NOW()) + ON CONFLICT (workspace_id, hash) DO UPDATE SET created_at = NOW()", + &w_id, + &hash, + &content + ) + .execute(&db) + .await?; + + // Clean up old entries (1 week TTL) + sqlx::query!( + "DELETE FROM raw_script_temp WHERE created_at < NOW() - INTERVAL '1 week'" + ) + .execute(&db) + .await?; + + Ok(Json(hash)) +} + +/// Compare local script content hashes with deployed versions. +/// Receives a map of path → SHA256(content), returns paths where the hash +/// differs from the deployed script (or the script doesn't exist on remote). +/// Hash comparison is done entirely in Postgres to avoid transferring content. +#[derive(Deserialize)] +struct WorkspaceDepDiff { + path: String, + language: ScriptLang, + name: Option, + hash: String, +} + +#[derive(Deserialize)] +struct DiffRequest { + scripts: std::collections::HashMap, + #[serde(default)] + workspace_deps: Vec, +} + +async fn diff_raw_scripts_with_deployed( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result>> { + check_scopes(&authed, || "scripts:read".to_string())?; + + let mut matching_set: std::collections::HashSet = std::collections::HashSet::new(); + let mut all_paths: Vec = Vec::new(); + + // --- Scripts --- + if !req.scripts.is_empty() { + let paths: Vec = req.scripts.keys().cloned().collect(); + let hashes: Vec = paths.iter().map(|p| req.scripts[p].clone()).collect(); + + let matching: Vec = sqlx::query_scalar( + "SELECT local.path FROM \ + unnest($1::text[], $2::text[]) AS local(path, hash) \ + INNER JOIN LATERAL ( \ + SELECT encode(sha256(convert_to(s.content, 'UTF8')), 'hex') AS deployed_hash \ + FROM script s \ + WHERE s.path = local.path AND s.workspace_id = $3 AND s.archived = false \ + ORDER BY s.created_at DESC LIMIT 1 \ + ) deployed ON deployed.deployed_hash = local.hash" + ) + .bind(&paths) + .bind(&hashes) + .bind(&w_id) + .fetch_all(&db) + .await?; + + matching_set.extend(matching); + all_paths.extend(paths); + } + + // --- Workspace dependencies --- + for dep in &req.workspace_deps { + let matching: Option = sqlx::query_scalar( + "SELECT $1::text \ + WHERE EXISTS ( \ + SELECT 1 FROM workspace_dependencies wd \ + WHERE wd.workspace_id = $2 AND wd.archived = false \ + AND wd.language = $3::SCRIPT_LANG \ + AND wd.name IS NOT DISTINCT FROM $4 \ + AND encode(sha256(convert_to(wd.content, 'UTF8')), 'hex') = $5 \ + )" + ) + .bind(&dep.path) + .bind(&w_id) + .bind(dep.language.as_str()) + .bind(&dep.name) + .bind(&dep.hash) + .fetch_optional(&db) + .await?; + + if let Some(path) = matching { + matching_set.insert(path); + } + all_paths.push(dep.path.clone()); + } + + let mismatched: Vec = all_paths + .into_iter() + .filter(|p| !matching_set.contains(p)) + .collect(); + + Ok(Json(mismatched)) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7d15eea003..930cb94949 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6988,6 +6988,83 @@ paths: type: string format: uuid + /w/{workspace}/scripts/raw_temp/store: + post: + summary: store raw script content temporarily for CLI lock generation + operationId: storeRawScriptTemp + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: script content to store + required: true + content: + application/json: + schema: + type: string + responses: + "200": + description: hash of stored content + content: + application/json: + schema: + type: string + + /w/{workspace}/scripts/raw_temp/diff: + post: + summary: diff local script hashes against deployed versions + operationId: diffRawScriptsWithDeployed + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: scripts and workspace deps to diff against deployed versions + required: true + content: + application/json: + schema: + type: object + required: + - scripts + properties: + scripts: + description: map of script path to SHA256 content hash + type: object + additionalProperties: + type: string + workspace_deps: + description: workspace dependencies to diff + type: array + items: + type: object + required: + - path + - language + - hash + properties: + path: + description: CLI path (e.g. dependencies/package.json) + type: string + language: + $ref: "#/components/schemas/ScriptLang" + name: + description: named workspace dependency (null for default) + type: string + hash: + description: SHA256 content hash + type: string + responses: + "200": + description: list of paths that differ from deployed versions + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/jobs/list_selected_job_groups: # We use post because sending a huge array as a query param can produce # URLs that may be too long diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 29b9765480..7f9a3f36db 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -5074,6 +5074,10 @@ pub struct RunDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option, + /// Map of script path -> content hash for resolving imports from temp storage. + /// Used by CLI to provide local script content during lock generation. + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Deserialize, Clone, Debug)] @@ -5133,6 +5137,8 @@ async fn run_dependencies_job( let mut hm = HashMap::new(); req.raw_workspace_dependencies .map(|v| hm.insert("raw_workspace_dependencies".to_owned(), to_raw_value(&v))); + req.temp_script_refs + .map(|v| hm.insert("temp_script_refs".to_owned(), to_raw_value(&v))); let (uuid, tx) = push( &db, @@ -5183,6 +5189,8 @@ pub struct RunFlowDependenciesRequest { pub raw_workspace_dependencies: Option, #[serde(default)] pub raw_deps: Option>, + #[serde(default)] + pub temp_script_refs: Option>, } #[derive(Serialize)] @@ -5226,6 +5234,10 @@ async fn run_flow_dependencies_job( req.raw_workspace_dependencies .map(|v| args_map.insert("raw_workspace_dependencies".to_string(), to_raw_value(&v))); + // Add temp_script_refs to args if present (for CLI local import resolution) + req.temp_script_refs + .map(|v| args_map.insert("temp_script_refs".to_string(), to_raw_value(&v))); + let (uuid, tx) = push( &db, PushIsolationLevel::IsolatedRoot(db.clone()), diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 8901055889..eae1648657 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -991,6 +991,38 @@ pub mod workspace_dependencies { } } +/// Temporary raw script content cache for CLI lock generation. +pub mod raw_script_temp { + use super::*; + use crate::DB; + + make_static! { + static ref CACHE: { String => String } in "raw_script_temp" <= 10000; + } + + /// Compute hash for raw script content (includes workspace_id for isolation). + pub fn compute_hash(workspace_id: &str, content: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(workspace_id.as_bytes()); + hasher.update(content.as_bytes()); + format!("{:x}", hasher.finalize()) + } + + /// Load content from cache, falling back to DB. + pub fn load(hash: String, db: &DB) -> impl Future> + '_ { + CACHE.get_or_insert_async(hash.clone(), async move { + sqlx::query_scalar!( + "SELECT content FROM raw_script_temp WHERE hash = $1", + &hash + ) + .fetch_optional(db) + .await? + .ok_or_else(|| error::Error::NotFound(format!("raw_script_temp hash: {}", hash))) + }) + } +} + const _: () = { impl Import for RawFlow { fn import(src: &impl Storage) -> error::Result { @@ -1183,7 +1215,8 @@ const _: () = { ((u8, ScriptHash), |x| format!("{:02x}-{:016x}", x.0, x.1.0)), (FlowNodeId, |x| format!("{:016x}", x.0)), (AppScriptId, |x| format!("{:016x}", x.0)), - ((i64, String), |x| format!("{}-{}", x.1, x.0)) + ((i64, String), |x| format!("{}-{}", x.1, x.0)), + (String, |x| x.as_str()) } #[cfg(feature = "scoped_cache")] diff --git a/backend/windmill-types/Cargo.toml b/backend/windmill-types/Cargo.toml index 9619a10ea7..2472d6bf02 100644 --- a/backend/windmill-types/Cargo.toml +++ b/backend/windmill-types/Cargo.toml @@ -9,6 +9,7 @@ name = "windmill_types" path = "src/lib.rs" [dependencies] +windmill-parser.workspace = true serde.workspace = true serde_json.workspace = true chrono.workspace = true diff --git a/backend/windmill-types/src/s3.rs b/backend/windmill-types/src/s3.rs index e3cbd1f9a3..0ab1794d7e 100644 --- a/backend/windmill-types/src/s3.rs +++ b/backend/windmill-types/src/s3.rs @@ -346,20 +346,9 @@ pub struct DuckdbConnectionSettingsQueryV2 { pub storage: Option, } -#[derive(Clone, Copy, Debug)] -pub enum S3ModeFormat { - Json, - Csv, - Parquet, -} - -pub fn s3_mode_extension(format: S3ModeFormat) -> &'static str { - match format { - S3ModeFormat::Json => "json", - S3ModeFormat::Csv => "csv", - S3ModeFormat::Parquet => "parquet", - } -} +// Re-export from windmill-parser to keep a single type definition +// (windmill-parser is WASM-compatible, windmill-types is not due to sqlx) +pub use windmill_parser::{s3_mode_extension, S3ModeFormat}; #[cfg(test)] mod tests { diff --git a/backend/windmill-worker/loader.bun.js b/backend/windmill-worker/loader.bun.js index f64e6d769b..f2a00de0cf 100644 --- a/backend/windmill-worker/loader.bun.js +++ b/backend/windmill-worker/loader.bun.js @@ -1,8 +1,11 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + const p = { name: "windmill-relative-resolver", async setup(build) { const { writeFileSync, readFileSync, mkdirSync } = await import("fs"); - const { dirname, resolve } = await import("node:path"); + const { dirname, resolve, join } = await import("node:path"); const base_internal_url = "BASE_INTERNAL_URL".replace( "localhost", @@ -95,11 +98,17 @@ const p = { : args.importer.replace(cdir + "/", ""); const isRelative = !args.path.startsWith("/"); + const endExt = args.path.endsWith(".ts") ? "" : ".ts"; + const pathNoExt = args.path.replace(/\.ts$/, ""); - let endExt = args.path.endsWith(".ts") ? "" : ".ts"; - const url = isRelative + // Lookup temp script hash + const normalized = (isRelative ? join(dirname(file_path), pathNoExt) : pathNoExt.slice(1)).replace(/\\/g, "/"); + const hash = TEMP_SCRIPT_REFS?.[normalized]; + + const url = (isRelative ? `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${file_path}/../${args.path}${endExt}` - : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}`; + : `${base_internal_url}/api/w/${w_id}/scripts/raw_unpinned/p/${args.path}${endExt}` + ) + (hash ? `?temp_script_hash=${hash}` : ""); const file = isRelative ? resolve("./" + file_path + "/../" + args.path + ".url") : resolve("./" + args.path + ".url"); diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js index fedef5fc5a..7d14ffcbc3 100644 --- a/backend/windmill-worker/loader.bun.windows.js +++ b/backend/windmill-worker/loader.bun.windows.js @@ -1,3 +1,6 @@ +// Injected by backend: maps normalized paths to temp storage hashes (or null) +const TEMP_SCRIPT_REFS = TEMP_SCRIPT_REFS_PLACEHOLDER; + // Windows-specific bun loader that uses a virtual "windmill-url" namespace instead // of writing .url files to disk. This avoids Windows path issues (backslashes in // resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace @@ -70,7 +73,13 @@ const p = { const rawScriptPath = isAbsolute ? `${path}${endExt}` : `${importerPath}/../${path}${endExt}`; - return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + const normalized = normalizePath(rawScriptPath); + // Look up temp script hash (keys are extensionless paths) + const lookupPath = normalized.replace(/\.ts$/, ""); + const hash = TEMP_SCRIPT_REFS?.[lookupPath]; + // Encode hash in the path so onLoad can extract it and append to fetch URL + const resolvedPath = hash ? `${normalized}?temp_script_hash=${hash}` : normalized; + return { path: resolvedPath, namespace: "windmill-url" }; } build.onLoad({ filter: filterLoad }, async (args) => { @@ -80,8 +89,13 @@ const p = { // Load windmill scripts by fetching from the API build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { - const path = args.path.replace(/^windmill-url:/, ""); - const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + // Extract temp_script_hash if embedded in the path by resolveWindmillImport + const [scriptPath, queryString] = args.path.replace(/^windmill-url:/, "").split("?"); + const hashParam = queryString?.startsWith("temp_script_hash=") + ? queryString.replace("temp_script_hash=", "") + : undefined; + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${scriptPath}` + + (hashParam ? `?temp_script_hash=${hashParam}` : ""); const req = await fetch(url, { method: "GET", headers: { @@ -124,7 +138,8 @@ const p = { // Resolve nested imports from within windmill-url modules build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { - const importer = args.importer.replace(/^windmill-url:/, ""); + // Strip any query string from the importer path before resolving + const importer = args.importer.replace(/^windmill-url:/, "").split("?")[0]; return resolveWindmillImport(importer, args.path); }); }, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index c3065be7d2..fb07806fd6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -206,6 +206,7 @@ pub async fn gen_bun_lockfile( workspace_dependencies: &WorkspaceDependenciesPrefetched, npm_mode: bool, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, quiet: bool, ) -> Result> { let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; @@ -216,6 +217,11 @@ pub async fn gen_bun_lockfile( gen_bunfig(job_dir, job_id, w_id, db).await?; write_file(job_dir, "package.json", package_json_content.as_str())?; } else { + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -224,7 +230,8 @@ pub async fn gen_bun_lockfile( "CURRENT_PATH", &crate::common::use_flow_root_path(script_path), ) - .replace("RAW_GET_ENDPOINT", "raw"); + .replace("RAW_GET_ENDPOINT", "raw") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); write_file( &job_dir, @@ -615,9 +622,15 @@ pub async fn build_loader( w_id: &str, current_path: &str, mode: LoaderMode, + temp_script_refs: &Option>, ) -> Result<()> { // Use forward slashes in JS strings to avoid backslash escape issues on Windows let job_dir_js = job_dir.replace('\\', "/"); + let temp_refs_json = temp_script_refs + .as_ref() + .and_then(|m| serde_json::to_string(m).ok()) + .unwrap_or_else(|| "null".to_string()); + let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -626,7 +639,8 @@ pub async fn build_loader( "CURRENT_PATH", &crate::common::use_flow_root_path(current_path), ) - .replace("RAW_GET_ENDPOINT", "raw_unpinned"); + .replace("RAW_GET_ENDPOINT", "raw_unpinned") + .replace("TEMP_SCRIPT_REFS_PLACEHOLDER", &temp_refs_json); if mode == LoaderMode::Node { write_file( @@ -924,6 +938,7 @@ pub async fn prebundle_bun_script( worker_name: &str, token: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + temp_script_refs: &Option>, ) -> Result<()> { let (local_path, remote_path) = compute_bundle_local_and_remote_path(inner_content, lock, script_path, db, w_id).await; @@ -950,6 +965,7 @@ pub async fn prebundle_bun_script( } else { LoaderMode::BunBundle }, + temp_script_refs, ) .await?; @@ -1271,6 +1287,7 @@ pub async fn handle_bun_job( workspace_dependencies, annotation.npm, &mut Some(occupancy_metrics), + &None, wac_replay_info.is_some(), ) .await?; @@ -1639,6 +1656,7 @@ try {{ } else { LoaderMode::BunBundle }, + &None, ) .await?; @@ -1655,6 +1673,7 @@ try {{ } else { LoaderMode::Bun }, + &None, ) .await } else { @@ -3358,6 +3377,7 @@ pub async fn start_worker( w_id, script_path, LoaderMode::BrowserBundle, + &None, ) .await?; generate_bun_bundle( @@ -3470,6 +3490,7 @@ pub async fn start_worker( .await?, annotation.npm, &mut None, + &None, false, ) .await?; @@ -3544,6 +3565,7 @@ pub async fn start_worker( } else { LoaderMode::Bun }, + &None, ) .await?; } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3aa7580220..3b6507a7e0 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1348,6 +1348,7 @@ async fn handle_python_deps( &mut version_specifiers, &mut locked_v, &None, + &None, // temp_script_refs: only used during CLI lock generation )) .await?; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index daba455f1c..93b48cf818 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -139,6 +139,13 @@ pub async fn handle_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let content = capture_dependency_job( &job.id, job.script_lang.as_ref().map(|v| Ok(v)).unwrap_or_else(|| { @@ -164,6 +171,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &temp_script_refs, ) .await; @@ -210,6 +218,7 @@ pub async fn handle_dependency_job( script_path, None, "script", + &None, ) .await { @@ -368,6 +377,13 @@ pub async fn handle_flow_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + let version = if skip_flow_update { None } else { @@ -467,6 +483,7 @@ pub async fn handle_flow_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -681,6 +698,7 @@ async fn lock_flow_value<'c>( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( FlowValue, sqlx::Transaction<'c, sqlx::Postgres>, @@ -711,6 +729,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -742,6 +761,7 @@ async fn lock_flow_value<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -779,6 +799,7 @@ async fn lock_flow_value<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?; @@ -817,6 +838,7 @@ async fn lock_modules<'c>( dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result<( Vec, sqlx::Transaction<'c, sqlx::Postgres>, @@ -872,6 +894,7 @@ async fn lock_modules<'c>( dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::ForloopFlow { @@ -911,6 +934,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -942,6 +966,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; e.value = FlowModuleValue::WhileloopFlow { @@ -978,6 +1003,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; nmodified_ids.extend(inner_modified_ids); @@ -1008,6 +1034,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; errors.extend(ninner_errors); @@ -1078,6 +1105,7 @@ async fn lock_modules<'c>( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, )) .await?; @@ -1179,6 +1207,7 @@ async fn lock_modules<'c>( job_path, Some(&e.id), "flow", + &temp_script_refs, ) .await; // @@ -1586,6 +1615,7 @@ async fn lock_modules_app( dependency_map: &mut ScopedDependencyMap, raw_workspace_dependencies_o: &Option, triggered_by_relative_import: bool, + temp_script_refs: &Option>, ) -> Result { match value { Value::Object(mut m) => { @@ -1696,6 +1726,7 @@ async fn lock_modules_app( &job.runnable_path(), container_id.as_deref(), "app", + temp_script_refs, ) .await; match new_lock { @@ -1773,6 +1804,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1801,6 +1833,7 @@ async fn lock_modules_app( dependency_map, raw_workspace_dependencies_o, triggered_by_relative_import, + temp_script_refs, ) .await?, ); @@ -1852,6 +1885,13 @@ pub async fn handle_app_dependency_job( .map(|x| x.get("triggered_by_relative_import").is_some()) .unwrap_or_default(); + // Extract temp_script_refs from job args (path -> hash mapping for temp storage) + let temp_script_refs: Option> = job + .args + .as_ref() + .and_then(|x| x.get("temp_script_refs")) + .and_then(|v| serde_json::from_str(v.get()).ok()); + sqlx::query!( "DELETE FROM workspace_runnable_dependencies WHERE app_path = $1 AND workspace_id = $2", job_path, @@ -1899,6 +1939,7 @@ pub async fn handle_app_dependency_job( &mut dependency_map, &raw_workspace_dependencies_o, triggered_by_relative_import, + &temp_script_refs, ) .await?; @@ -2398,6 +2439,8 @@ async fn capture_dependency_job( base_path: &str, step_id: Option<&str>, runnable_type: &str, // "script", "flow", or "app" + // Map of script path -> content hash for resolving imports from temp storage (CLI). + temp_script_refs: &Option>, ) -> error::Result { // Check if we can skip relocking: // - Must be triggered by relative import @@ -2456,12 +2499,13 @@ async fn capture_dependency_job( let (mut version_specifiers, mut locked_v) = (vec![], None); let reqs = windmill_parser_py_imports::parse_python_imports( job_raw_code, - &w_id, + w_id, script_path, &db, &mut version_specifiers, &mut locked_v, raw_workspace_dependencies_o, + temp_script_refs, ) .await? .0 @@ -2585,6 +2629,7 @@ async fn capture_dependency_job( &workspace_dependencies, windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm, &mut Some(occupancy_metrics), + temp_script_refs, false, ) .await? @@ -2602,6 +2647,7 @@ async fn capture_dependency_job( worker_name, &token, &mut Some(occupancy_metrics), + temp_script_refs, ) .await?; } diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 72be8f1ca9..865091104c 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -12,6 +12,7 @@ const parserPackages = [ "windmill-parser-wasm-yaml", "windmill-parser-wasm-csharp", "windmill-parser-wasm-nu", "windmill-parser-wasm-java", "windmill-parser-wasm-ruby", + "windmill-parser-wasm-py-imports", ]; const parserExternals = parserPackages.flatMap(p => ["--external", p]); diff --git a/cli/bun.lock b/cli/bun.lock index e400420a63..6c421141b4 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -24,10 +24,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", @@ -294,13 +295,15 @@ "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="], + "windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.659.1", "", {}, "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="], + "windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.639.0", "", {}, "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="], "windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="], "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="], - "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.623.1", "", {}, "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="], + "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="], "windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="], diff --git a/cli/package.json b/cli/package.json index 6102f815f2..e44a215631 100644 --- a/cli/package.json +++ b/cli/package.json @@ -32,10 +32,11 @@ "windmill-parser-wasm-nu": "*", "windmill-parser-wasm-php": "*", "windmill-parser-wasm-py": "*", + "windmill-parser-wasm-py-imports": "*", "windmill-parser-wasm-regex": "*", "windmill-parser-wasm-ruby": "*", "windmill-parser-wasm-rust": "*", - "windmill-parser-wasm-ts": "*", + "windmill-parser-wasm-ts": "^1.659.1", "windmill-parser-wasm-yaml": "*", "windmill-yaml-validator": "1.1.1", "ws": "8.18.0", diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 253dc99f3b..82e2a1ac22 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts"; import { stringify as yamlStringify } from "yaml"; import { GlobalOptions } from "../../types.ts"; import { + readLockfile, checkifMetadataUptodate, blueColor, clearGlobalLock, @@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; import { resolveWorkspace } from "../../core/context.ts"; import { requireLogin } from "../../core/auth.ts"; import { getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__app_hash"; export const APP_BACKEND_FOLDER = "backend"; @@ -113,7 +116,9 @@ export async function generateAppLocksInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (appFolder.endsWith(SEP)) { appFolder = appFolder.substring(0, appFolder.length - 1); @@ -125,9 +130,6 @@ export async function generateAppLocksInternal( log.info(`Generating locks for app ${appFolder} at ${remote_path}`); } - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - // Read the app file first to filter workspace dependencies const appFilePath = path.join( appFolder, @@ -135,35 +137,80 @@ export async function generateAppLocksInternal( ); const appFile = (await yamlParseFile(appFilePath)) as AppFile; - // Filter workspace dependencies based on inline scripts' languages and annotations const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value; - const filteredDeps = await filterWorkspaceDependenciesForApp( - appValue, - rawWorkspaceDependencies, - appFolder - ); + const folderNormalized = appFolder.replaceAll(SEP, "/"); - let hashes = await generateAppHash( - filteredDeps, - appFolder, - rawApp, - opts.defaultTs - ); + let filteredDeps: Record = {}; + const conf = await readLockfile(); - const conf = await import("../../utils/metadata.ts").then((m) => - m.readLockfile() - ); - if ( - await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH) - ) { - if (!noStaleMessage) { - log.info( - colors.green(`App ${remote_path} metadata is up-to-date, skipping`) - ); + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + // For raw apps in new format, runnables are in separate files under backend/ + let treeAppValue = structuredClone(appValue); + if (rawApp) { + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER); + const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath); + if (Object.keys(runnablesFromFiles).length > 0) { + treeAppValue = runnablesFromFiles; + } + } + + // First pass: add inline scripts as separate nodes, then add app node importing them + const inlineScriptPaths: string[] = []; + await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => { + if (!inlineScript.content || !inlineScript.language) { + return inlineScript; + } + + let content = inlineScript.content; + // Resolve !inline references + if (typeof content === "string" && content.startsWith("!inline ")) { + const filePath = appFolder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + return inlineScript; + } + } + + const treePath = folderNormalized + "/" + context.path.join("/"); + const language = inlineScript.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false); + inlineScriptPaths.push(treePath); + + return inlineScript; + }); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp); + return; + } + // Second pass: get mismatched workspace deps from tree + // TODO: pass raw workspace deps more precisely to every inline script lock generation call + // (currently we pass the union of all mismatched deps filtered for the whole app) + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder); + } else { + // Legacy behaviour + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder); + + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`App ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -179,6 +226,8 @@ export async function generateAppLocksInternal( let updatedScripts: string[] = []; if (!justUpdateMetadataLock) { + const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); + const changedScripts = []; // Find hashes that do not correspond to previous hashes for (const [scriptPath, hash] of Object.entries(hashes)) { @@ -190,7 +239,13 @@ export async function generateAppLocksInternal( } } - if (changedScripts.length > 0) { + // Get temp_script_refs from tree for relative import resolution + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); + + // In tree mode, the tree already verified this app is stale (possibly via dependency change). + // Per-script hashes only detect content changes, not transitive dependency changes, + // so we must regenerate locks for all inline scripts regardless. + if (changedScripts.length > 0 || (tree && !legacyBehaviour)) { if (!noStaleMessage) { log.info( `Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}` @@ -219,7 +274,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); // Note: updateRawAppRunnables now writes each runnable to its own file } else { @@ -236,7 +292,8 @@ export async function generateAppLocksInternal( appFolder, filteredDeps, opts.defaultTs, - noStaleMessage + noStaleMessage, + tempScriptRefs ); normalAppFile.value = result.value; updatedScripts = result.updatedScripts; @@ -252,15 +309,16 @@ export async function generateAppLocksInternal( } } - // Regenerate hashes after updates - hashes = await generateAppHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateAppHash( + depsForHash, appFolder, rawApp, opts.defaultTs ); await clearGlobalLock(appFolder); - for (const [scriptPath, hash] of Object.entries(hashes)) { + for (const [scriptPath, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(appFolder, hash, scriptPath); } if (!noStaleMessage) { @@ -366,7 +424,8 @@ async function updateRawAppRunnables( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise { const updatedRunnables: string[] = []; const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); @@ -446,7 +505,8 @@ async function updateRawAppRunnables( content, language, `${remotePath}/${runnableId}`, - rawDeps + rawDeps, + tempScriptRefs ); // Determine file extension for this language @@ -513,7 +573,8 @@ async function updateAppInlineScripts( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun", - noStaleMessage?: boolean + noStaleMessage?: boolean, + tempScriptRefs?: Record ): Promise<{ value: any; updatedScripts: string[] }> { const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() }); const updatedScripts: string[] = []; @@ -561,7 +622,8 @@ async function updateAppInlineScripts( content, language, scriptPath, - rawDeps + rawDeps, + tempScriptRefs ); } // Determine file extension for this language (following extractInlineScriptsForApps pattern) @@ -626,7 +688,8 @@ async function generateInlineScriptLock( content: string, language: string, scriptPath: string, - rawWorkspaceDependencies: Record | undefined + rawWorkspaceDependencies: Record | undefined, + tempScriptRefs?: Record ): Promise { // Filter workspace dependencies to only include those matching this script's language and annotations const filteredDeps = rawWorkspaceDependencies @@ -657,6 +720,9 @@ async function generateInlineScriptLock( ? filteredDeps : null, entrypoint: scriptPath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index 4d92405d01..58d0777f90 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -437,6 +437,7 @@ async function preview( export async function generateLocks( opts: GlobalOptions & { yes?: boolean; + dryRun?: boolean; } & SyncOptions, folder: string | undefined ) { @@ -487,6 +488,10 @@ export async function generateLocks( } if (hasAny) { + if (opts.dryRun) { + log.info(colors.gray("Dry run complete.")); + return; + } if ( !opts.yes && !(await Confirm.prompt({ @@ -592,6 +597,7 @@ const command = new Command() ) .arguments("[flow:file]") .option("--yes", "Skip confirmation prompt") + .option("--dry-run", "Perform a dry run without making changes") .option( "-i --includes ", "Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)" diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 6a9040094f..1f5d86b8ea 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -29,10 +29,9 @@ import { FlowFile } from "./flow.ts"; import { FlowValue } from "../../../gen/types.gen.ts"; import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts"; import { workspaceDependenciesLanguages } from "../../utils/script_common.ts"; -import { - extractNameFromFolder, - getNonDottedPaths, -} from "../../utils/resource_folders.ts"; +import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts"; +import { extractRelativeImports } from "../../utils/relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; const TOP_HASH = "__flow_hash"; async function generateFlowHash( @@ -70,7 +69,9 @@ export async function generateFlowLockInternal( defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean + noStaleMessage?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); @@ -80,33 +81,67 @@ export async function generateFlowLockInternal( log.info(`Generating lock for flow ${folder} at ${remote_path}`); } - // Always get out-of-sync workspace dependencies - const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); - const flowValue = (await yamlParseFile( folder! + SEP + "flow.yaml" )) as FlowFile; - // Filter workspace dependencies based on inline scripts' languages and annotations - const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); - - let hashes = await generateFlowHash( - filteredDeps, - folder, + const folderNormalized = folder.replaceAll(SEP, "/"); + const inlineScriptsForTree = extractInlineScriptsForFlows( + structuredClone(flowValue.value.modules), + {}, + SEP, opts.defaultTs - ); + ).filter(s => !s.is_lock); + let filteredDeps: Record = {}; const conf = await readLockfile(); - if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { - if (!noStaleMessage) { - log.info( - colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) - ); + + if (!legacyBehaviour && tree) { + if (dryRun) { + const inlineScriptPaths: string[] = []; + for (const script of inlineScriptsForTree) { + let content = script.content; + if (content.startsWith("!inline ")) { + const filePath = folder + SEP + content.replace("!inline ", ""); + try { + content = await readFile(filePath, "utf-8"); + } catch { + continue; + } + } + + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + const language = script.language as ScriptLanguage; + const imports = await extractRelativeImports(content, treePath, language); + await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false); + inlineScriptPaths.push(treePath); + } + + const hashes = await generateFlowHash({}, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale); + return; + } + // Second pass: get mismatched workspace deps from tree + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder); + } else { + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); + filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); + + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + + if (!isDirectlyStale) { + if (!noStaleMessage) { + log.info( + colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + return remote_path; } - return; - } else if (dryRun) { - return remote_path; } if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) { @@ -122,7 +157,23 @@ export async function generateFlowLockInternal( let changedScripts: string[] = []; + // Build mapping from on-disk file names (hash keys like "a.py") to tree paths + // (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without + // a path assigner, so paths always have .inline_script suffix, but on-disk files + // may not (non-dotted mode). + const fileToTreePath = new Map(); + for (const script of inlineScriptsForTree) { + const c = script.content; + if (c.startsWith("!inline ")) { + const fileName = c.replace("!inline ", ""); + const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path)); + fileToTreePath.set(fileName, treePath); + } + } + if (!justUpdateMetadataLock) { + const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); + //find hashes that do not correspond to previous hashes for (const [path, hash] of Object.entries(hashes)) { if (path == TOP_HASH) { @@ -137,27 +188,39 @@ export async function generateFlowLockInternal( log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`); } const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8"); + // In tree mode, use the tree's staleness info (which includes transitive dependency changes) + // to determine which scripts need relocking, instead of only content-changed ones. + const locksToRemove = (tree && !legacyBehaviour) + ? Object.keys(hashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; await replaceInlineScripts( flowValue.value.modules, fileReader, log, folder + SEP!, SEP, - changedScripts + locksToRemove ); if (flowValue.value.failure_module) { - await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } if (flowValue.value.preprocessor_module) { - await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, changedScripts); + await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove); } //removeChangedLocks + const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized); flowValue.value = await updateFlow( workspace, flowValue.value, remote_path, - filteredDeps + filteredDeps, + tempScriptRefs ); const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", { @@ -187,13 +250,15 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash( - filteredDeps, + // Non-legacy mode excludes workspace deps from hash (tracked via tree instead) + const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps; + const finalHashes = await generateFlowHash( + depsForHash, folder, opts.defaultTs ); await clearGlobalLock(folder); - for (const [path, hash] of Object.entries(hashes)) { + for (const [path, hash] of Object.entries(finalHashes)) { await updateMetadataGlobalLock(folder, hash, path); } if (!noStaleMessage) { @@ -201,7 +266,16 @@ export async function generateFlowLockInternal( } // Return the list of updated scripts (extract just the filename from the path) - const updatedScripts = changedScripts.map(p => { + // In tree mode, use the same staleness-aware list we used for lock removal + const relocked = (tree && !legacyBehaviour) + ? Object.keys(finalHashes).filter(k => { + if (k === TOP_HASH) return false; + const treePath = fileToTreePath.get(k) + ?? (folderNormalized + "/" + path.basename(k, path.extname(k))); + return tree.isStale(treePath); + }) + : changedScripts; + const updatedScripts = relocked.map(p => { const parts = p.split(SEP); return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension }); @@ -239,7 +313,8 @@ export async function updateFlow( workspace: Workspace, flow_value: FlowValue, remotePath: string, - rawWorkspaceDependencies: Record + rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { let rawResponse; @@ -264,6 +339,9 @@ export async function updateFlow( path: remotePath, use_local_lockfiles: true, raw_workspace_dependencies: rawWorkspaceDependencies, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); @@ -282,6 +360,9 @@ export async function updateFlow( body: JSON.stringify({ flow_value, path: remotePath, + ...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? { temp_script_refs: tempScriptRefs } + : {}), }), } ); diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index c523996cdc..d273f4b631 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -10,6 +10,8 @@ import * as log from "../../core/log.ts"; import { generateScriptMetadataInternal, getRawWorkspaceDependencies, + readLockfile, + checkifMetadataUptodate, } from "../../utils/metadata.ts"; import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts"; import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts"; @@ -19,14 +21,20 @@ import { ignoreF, } from "../sync/sync.ts"; import { exts } from "../script/script.ts"; -import { isFlowPath, isAppPath, isRawAppPath, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; +import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; import { listSyncCodebases } from "../../utils/codebase.ts"; +import { + DoubleLinkedDependencyTree, + uploadScripts, + ItemType, +} from "../../utils/dependency_tree.ts"; interface StaleItem { - type: "script" | "flow" | "app"; + type: ItemType; path: string; folder: string; isRawApp?: boolean; + staleReason?: string; } async function generateMetadata( @@ -38,6 +46,7 @@ async function generateMetadata( skipScripts?: boolean; skipFlows?: boolean; skipApps?: boolean; + strictFolderBoundaries?: boolean; } & SyncOptions, folder?: string ) { @@ -49,12 +58,10 @@ async function generateMetadata( await requireLogin(opts); opts = await mergeConfigWithConfigFile(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false); const codebases = await listSyncCodebases(opts); const ignore = await ignoreF(opts); - const staleItems: StaleItem[] = []; - // --schema-only implies skipping flows and apps (they only have locks, no schemas) const skipScripts = opts.skipScripts ?? false; const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false; @@ -70,7 +77,11 @@ async function generateMetadata( return; } - log.info(colors.gray(`Checking ${checking.join(", ")}...`)); + log.info(`Checking ${checking.join(", ")}...`); + + // Build dependency tree for relative import tracking + const tree = new DoubleLinkedDependencyTree(); + tree.setWorkspaceDeps(rawWorkspaceDependencies); // === Collect stale scripts === if (!skipScripts) { @@ -81,9 +92,7 @@ async function generateMetadata( return ( (!isD && !exts.some((ext) => p.endsWith(ext))) || ignore(p, isD) || - isFlowPath(p) || - isAppPath(p) || - isRawAppPath(p) || + isFolderResourcePathAnyFormat(p) || (isScriptModulePath(p) && !isModuleEntryPoint(p)) ); }, @@ -92,19 +101,18 @@ async function generateMetadata( ); for (const e of Object.keys(scriptElems)) { - const candidate = await generateScriptMetadataInternal( + await generateScriptMetadataInternal( e, workspace, opts, - true, // dryRun + true, // dryRun - populate tree true, // noStaleMessage rawWorkspaceDependencies, codebases, - false + false, + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "script", path: candidate, folder: e }); - } } } @@ -126,18 +134,17 @@ async function generateMetadata( ) ).map((x) => x.substring(0, x.lastIndexOf(SEP))); - for (const folder of flowElems) { - const candidate = await generateFlowLockInternal( - folder, - true, // dryRun + for (const flowFolder of flowElems) { + await generateFlowLockInternal( + flowFolder, + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "flow", path: candidate, folder }); - } } } @@ -161,33 +168,74 @@ async function generateMetadata( const appFolders = getAppFolders(elems, "app.yaml"); for (const appFolder of rawAppFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, true, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: true }); - } } for (const appFolder of appFolders) { - const candidate = await generateAppLocksInternal( + await generateAppLocksInternal( appFolder, false, // rawApp - true, // dryRun + true, // dryRun - populate tree workspace, opts, false, - true // noStaleMessage + true, // noStaleMessage + false, // legacyBehaviour + tree ); - if (candidate) { - staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: false }); - } + } + } + + // === Propagate staleness through imports === + tree.propagateStaleness(); + + // Upload stale scripts to temp storage so the backend can resolve relative imports. + // If this fails (e.g. backend is older and doesn't have /raw_temp endpoints), + // degrade gracefully: locks will be generated using deployed script content only. + try { + await uploadScripts(tree, workspace); + } catch (e) { + log.warn(colors.yellow( + `Failed to upload scripts to temp storage (backend may be too old): ${e}. ` + + `Locks will be generated using deployed script versions only — locally modified ` + + `relative imports may not be reflected.` + )); + } + + // === Populate staleItems from tree === + const staleItems: StaleItem[] = []; + const seenFolders = new Set(); + + for (const p of tree.allPaths()) { + const staleReason = tree.getStaleReason(p); + if (!staleReason) continue; + + const itemType = tree.getItemType(p)!; + const itemFolder = tree.getFolder(p)!; + + if (itemType === "dependencies") { + staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason }); + } else if (itemType === "inline_script") { + // Inline scripts are not listed separately — their parent flow/app is stale via propagation + continue; + } else if (itemType === "script") { + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason }); + } else if (!seenFolders.has(itemFolder)) { + // Flows/Apps: one entry per folder (dedupe multiple inline scripts) + seenFolders.add(itemFolder); + const originalPath = tree.getOriginalPath(p)!; + staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason }); } } @@ -200,11 +248,54 @@ async function generateMetadata( if (folder.endsWith("/")) { folder = folder.substring(0, folder.length - 1); } - // Normalize item.folder for comparison (Windows file paths use backslashes) - filteredItems = staleItems.filter((item) => { + // Strip file extension if user passed a specific file path (e.g. f/test/script.ts) + const folderNoExt = folder.replace(/\.[^/.]+$/, ""); + // Check if an item is inside the specified folder + const isInsideFolder = (item: StaleItem) => { const normalizedFolder = item.folder.replaceAll("\\", "/"); - return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/"); - }); + const normalizedPath = item.path.replaceAll("\\", "/"); + return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/") + || normalizedPath === folder || normalizedPath === folderNoExt; + }; + const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt; + // Check if a tree path or any of its transitive deps is inside the folder + const touchesFolder = (treePath: string) => { + if (isPathInFolder(treePath)) return true; + let found = false; + tree.traverseTransitive(treePath, (importPath) => { + if (isPathInFolder(importPath)) { + found = true; + return true; // stop early + } + }); + return found; + }; + + const isRelevant = (item: StaleItem) => { + if (isInsideFolder(item)) return true; + if (item.type === "dependencies") return true; + const treePath = (item.type === "script" + ? item.path.replace(/\.[^/.]+$/, "") + : item.folder).replaceAll("\\", "/"); + return touchesFolder(treePath); + }; + + if (opts.strictFolderBoundaries) { + // Strict mode: only items inside the folder + filteredItems = staleItems.filter(isInsideFolder); + + // Warn about stale items outside the folder that would be included by default + const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies"); + for (const item of excludedStale) { + const normalizedPath = item.path.replaceAll("\\", "/"); + log.warn(colors.yellow( + `Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.` + )); + } + } else { + // Default: include items inside the folder and any stale importers that transitively depend on it + filteredItems = staleItems.filter(isRelevant); + } } // === Show stale items and confirm === @@ -217,28 +308,24 @@ async function generateMetadata( const scripts = filteredItems.filter((i) => i.type === "script"); const flows = filteredItems.filter((i) => i.type === "flow"); const apps = filteredItems.filter((i) => i.type === "app"); + const deps = filteredItems.filter((i) => i.type === "dependencies"); log.info(""); - log.info(`Found ${filteredItems.length} item(s) with stale metadata:`); + log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`); - if (scripts.length > 0) { - log.info(colors.gray(` Scripts (${scripts.length}):`)); - for (const item of scripts) { - log.info(colors.yellow(` ${item.path}`)); + const printItems = (label: string, items: StaleItem[]) => { + if (items.length === 0) return; + log.info(` ${label} (${items.length}):`); + for (const item of items) { + const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : ""; + log.info(` ~ ${item.path}` + reason); } - } - if (flows.length > 0) { - log.info(colors.gray(` Flows (${flows.length}):`)); - for (const item of flows) { - log.info(colors.yellow(` ${item.path}`)); - } - } - if (apps.length > 0) { - log.info(colors.gray(` Apps (${apps.length}):`)); - for (const item of apps) { - log.info(colors.yellow(` ${item.path}`)); - } - } + }; + + printItems("Workspace dependencies", deps); + printItems("Scripts", scripts); + printItems("Flows", flows); + printItems("Apps", apps); if (opts.dryRun) { return; @@ -259,28 +346,30 @@ async function generateMetadata( log.info(""); // === Process all stale items with progress counter === - const total = filteredItems.length; + const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps(); + const total = filteredItems.length - deps.length; const maxWidth = `[${total}/${total}]`.length; let current = 0; const formatProgress = (n: number) => { - const bracket = `[${n}/${total}]`; - return colors.gray(bracket.padEnd(maxWidth, " ")); + return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " "))); }; // Process scripts for (const item of scripts) { current++; - log.info(`${formatProgress(current)} script ${colors.cyan(item.path)}`); + log.info(`${formatProgress(current)} script ${item.path}`); await generateScriptMetadataInternal( - item.folder, + item.path, // originalPath with extension workspace, opts, false, // dryRun - true, // noStaleMessage - we handle output - rawWorkspaceDependencies, + true, // noStaleMessage + mismatchedWorkspaceDeps, codebases, - false + false, + false, // legacyBehaviour + tree ); } @@ -288,38 +377,49 @@ async function generateMetadata( for (const item of flows) { current++; const result = await generateFlowLockInternal( - item.folder, + item.folder.replaceAll("/", SEP), false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as FlowLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const flowResult = result as FlowLocksResult | undefined; + const scriptsInfo = flowResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} flow ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`); } + // Process apps for (const item of apps) { current++; const result = await generateAppLocksInternal( - item.folder, + item.folder.replaceAll("/", SEP), item.isRawApp!, // rawApp false, // dryRun workspace, opts, false, - true // noStaleMessage - we handle output - ) as AppLocksResult | void; - const scriptsInfo = result?.updatedScripts?.length - ? `: ${colors.gray(result.updatedScripts.join(", "))}` + true, // noStaleMessage + false, // legacyBehaviour + tree + ); + const appResult = result as AppLocksResult | undefined; + const scriptsInfo = appResult?.updatedScripts?.length + ? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`)) : ""; - log.info(`${formatProgress(current)} app ${colors.cyan(item.path)}${scriptsInfo}`); + log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`); } + // Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped) + const allStaleDeps = staleItems.filter((i) => i.type === "dependencies"); + await tree.persistDepsHashes(allStaleDeps.map((d) => d.path)); + log.info(""); - log.info(colors.green(`Done. Updated ${total} item(s).`)); + log.info(`Done. Updated ${colors.bold(String(total))} item(s).`); } const command = new Command() @@ -332,6 +432,7 @@ const command = new Command() .option("--skip-scripts", "Skip processing scripts") .option("--skip-flows", "Skip processing flows") .option("--skip-apps", "Skip processing apps") + .option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)") .option( "-i --includes ", "Comma separated patterns to specify which files to include" diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 68ff946cee..14f3155ff7 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -130,7 +130,7 @@ async function push(opts: PushOptions, filePath: string) { [], undefined, opts, - await getRawWorkspaceDependencies(), + await getRawWorkspaceDependencies(true), codebases ); log.info(colors.bold.underline.green(`Script ${filePath} pushed`)); @@ -1161,7 +1161,7 @@ export async function generateMetadata( opts = await mergeConfigWithConfigFile(opts); const codebases = await listSyncCodebases(opts); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); if (scriptPath) { // read script metadata file await generateScriptMetadataInternal( diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index bc0eee2275..d62c39cba9 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -2280,7 +2280,7 @@ export async function pull( const tracker: ChangeTracker = await buildTracker(changes); const rawWorkspaceDependencies: Record = - await getRawWorkspaceDependencies(); + await getRawWorkspaceDependencies(true); for (const change of tracker.scripts) { await generateScriptMetadataInternal( @@ -2611,7 +2611,7 @@ export async function push( false, // els1 (local) is not the remote source ); - const rawWorkspaceDependencies = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true); const tracker: ChangeTracker = await buildTracker(changes); @@ -2657,7 +2657,7 @@ export async function push( true, ); if (stale) { - staleFlows.push(stale); + staleFlows.push(stale as string); } } @@ -2682,7 +2682,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } @@ -2697,7 +2697,7 @@ export async function push( true, ); if (stale) { - staleApps.push(stale); + staleApps.push(stale as string); } } diff --git a/cli/src/utils/dependency_tree.ts b/cli/src/utils/dependency_tree.ts new file mode 100644 index 0000000000..5e2adf7453 --- /dev/null +++ b/cli/src/utils/dependency_tree.ts @@ -0,0 +1,373 @@ +/** + * Double-linked dependency tree for tracking script imports and propagating staleness. + */ + +import { Workspace } from "../commands/workspace/workspace.ts"; +import * as wmill from "../../gen/services.gen.ts"; +import type { ScriptLang } from "../../gen/types.gen.ts"; +import { ScriptLanguage } from "./script_common.ts"; +import { + filterWorkspaceDependencies, + generateScriptHash, + checkifMetadataUptodate, + workspaceDependenciesPathToLanguageAndFilename, + updateMetadataGlobalLock, +} from "./metadata.ts"; +import { generateHash } from "./utils.ts"; + +/** + * Diff local scripts against deployed versions, upload only those that differ. + * Only uploaded (mismatched) scripts get contentHash set, so flatten() returns + * temp_script_refs only for scripts the backend can't resolve from deployed versions. + */ +export async function uploadScripts( + tree: DoubleLinkedDependencyTree, + workspace: Workspace +): Promise { + // Split into scripts vs workspace deps and compute SHA256(content) for each + const scriptHashes: Record = {}; + const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = []; + + for (const path of tree.allPaths()) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Empty string is valid for workspace deps (means "no deps") — only skip undefined + if (content === undefined) continue; + const info = workspaceDependenciesPathToLanguageAndFilename(path); + if (info) { + const hash = await generateHash(content); + workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash }); + } + } else if (itemType === "script") { + if (!content) continue; + const hash = await generateHash(content); + scriptHashes[path] = hash; + } + // Skip inline_script, flow, app — they don't need temp storage uploads + } + + if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return; + + // Single batch query: find which scripts/deps differ from deployed versions + const mismatched = await wmill.diffRawScriptsWithDeployed({ + workspace: workspace.workspaceId, + requestBody: { + scripts: scriptHashes, + workspace_deps: workspaceDeps, + }, + }); + + // Upload only mismatched scripts to temp storage + for (const path of mismatched) { + const content = tree.getContent(path); + const itemType = tree.getItemType(path); + + if (itemType === "dependencies") { + // Workspace deps don't need temp storage — just mark as mismatched. + // Empty string is valid (means the dep file was emptied locally). + if (content !== undefined) { + tree.setContentHash(path, "mismatched"); + } + } else if (content) { + const hash = await wmill.storeRawScriptTemp({ + workspace: workspace.workspaceId, + requestBody: content, + }); + tree.setContentHash(path, hash); + } + } +} + +export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies"; + +interface DependencyNode { + content: string; + stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata) + contentHash?: string; // Hash for temp storage lookup (content only) + language: ScriptLanguage; + metadata: string; + imports: Set; + importedBy: Set; + staleReason?: string; + // Item metadata for generate-metadata command + itemType: ItemType; + folder: string; // Folder path (for flows/apps) or remote path (for scripts) + originalPath: string; // Original path passed to handler (with extension for scripts) + isRawApp?: boolean; // Only set for apps + isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale) +} + +export class DoubleLinkedDependencyTree { + private nodes: Map = new Map(); + private workspaceDeps: Record = {}; + + setWorkspaceDeps(deps: Record): void { + this.workspaceDeps = deps; + } + + async addNode( + path: string, + content: string, + language: ScriptLanguage, + metadata: string, + imports: string[], + itemType: ItemType, + folder: string, + originalPath: string, + isDirectlyStale: boolean, + isRawApp?: boolean + ): Promise { + const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script"; + const filteredDeps = hasWorkspaceDeps + ? filterWorkspaceDependencies(this.workspaceDeps, content, language) + : {}; + const stalenessHash = await generateScriptHash({}, content, metadata); + + if (!this.nodes.has(path)) { + this.nodes.set(path, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + const node = this.nodes.get(path)!; + node.content = content; + node.stalenessHash = stalenessHash; + node.language = language; + node.metadata = metadata; + node.itemType = itemType; + node.folder = folder; + node.originalPath = originalPath; + node.isDirectlyStale = isDirectlyStale; + node.isRawApp = isRawApp; + + // Create nodes for referenced workspace deps with content and language. + const filteredDepsPaths = Object.keys(filteredDeps); + for (const depsPath of filteredDepsPaths) { + if (!this.nodes.has(depsPath)) { + const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath); + const contentHash = await generateHash(filteredDeps[depsPath] + depsPath); + const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined); + this.nodes.set(depsPath, { + content: filteredDeps[depsPath], + stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "dependencies", folder: "", originalPath: depsPath, + isDirectlyStale: !isUpToDate, + }); + } + } + + const allImports = [...imports, ...filteredDepsPaths]; + for (const importPath of allImports) { + node.imports.add(importPath); + + if (!this.nodes.has(importPath)) { + this.nodes.set(importPath, { + content: "", stalenessHash: "", language: "deno", metadata: "", + imports: new Set(), importedBy: new Set(), + itemType: "script", folder: "", originalPath: "", isDirectlyStale: false, + }); + } + this.nodes.get(importPath)!.importedBy.add(path); + } + } + + getContent(path: string): string | undefined { + return this.nodes.get(path)?.content; + } + + getStalenessHash(path: string): string | undefined { + return this.nodes.get(path)?.stalenessHash; + } + + getContentHash(path: string): string | undefined { + return this.nodes.get(path)?.contentHash; + } + + setContentHash(path: string, hash: string): void { + const node = this.nodes.get(path); + if (node) { + node.contentHash = hash; + } + } + + getLanguage(path: string): ScriptLanguage | undefined { + return this.nodes.get(path)?.language; + } + + getMetadata(path: string): string | undefined { + return this.nodes.get(path)?.metadata; + } + + getStaleReason(path: string): string | undefined { + return this.nodes.get(path)?.staleReason; + } + + getItemType(path: string): ItemType | undefined { + return this.nodes.get(path)?.itemType; + } + + getFolder(path: string): string | undefined { + return this.nodes.get(path)?.folder; + } + + getIsRawApp(path: string): boolean | undefined { + return this.nodes.get(path)?.isRawApp; + } + + getIsDirectlyStale(path: string): boolean { + return this.nodes.get(path)?.isDirectlyStale ?? false; + } + + getOriginalPath(path: string): string | undefined { + return this.nodes.get(path)?.originalPath; + } + + getImports(path: string): Set | undefined { + return this.nodes.get(path)?.imports; + } + + /** + * Returns true if this node has been marked stale (directly or transitively). + */ + isStale(path: string): boolean { + return this.nodes.get(path)?.staleReason !== undefined; + } + + /** + * Mutates the tree by removing all nodes that are not stale. + * Uses BFS on reverse graph (importedBy) to find all stale scripts. + * Starts from nodes with isDirectlyStale=true. + */ + propagateStaleness(): void { + // Collect directly stale nodes + const directlyStale = new Set(); + for (const [path, node] of this.nodes.entries()) { + if (node.isDirectlyStale) { + directlyStale.add(path); + node.staleReason = "content changed"; + } + } + + const allStale = new Set(directlyStale); + const queue = [...directlyStale]; + const visited = new Set(); + + while (queue.length > 0) { + const scriptPath = queue.shift()!; + if (visited.has(scriptPath)) continue; + visited.add(scriptPath); + + const node = this.nodes.get(scriptPath); + if (!node) continue; + + for (const importer of node.importedBy) { + if (!allStale.has(importer)) { + allStale.add(importer); + queue.push(importer); + // Set reason for transitively stale scripts + const importerNode = this.nodes.get(importer); + if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`; + } + } + } + + } + + /** + * Walks all transitive imports for a node, calling the callback for each. + * Callback may return true to stop traversing that branch. + */ + traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void { + const queue = [scriptPath]; + const visited = new Set(); + + while (queue.length > 0) { + const current = queue.shift()!; + if (visited.has(current)) continue; + visited.add(current); + + const node = this.nodes.get(current); + if (!node) continue; + + for (const importPath of node.imports) { + const importNode = this.nodes.get(importPath); + if (importNode) { + const stop = callback(importPath, importNode); + if (!stop) { + queue.push(importPath); + } + } + } + } + } + + allPaths(): IterableIterator { + return this.nodes.keys(); + } + + /** + * Returns paths of all stale nodes (those with a staleReason). + */ + *stalePaths(): IterableIterator { + for (const [path, node] of this.nodes.entries()) { + if (node.staleReason) { + yield path; + } + } + } + + has(path: string): boolean { + return this.nodes.has(path); + } + + /** + * Returns workspace deps that were uploaded as mismatched with remote. + * These need to be passed as raw_workspace_dependencies in job args + * so the backend uses local content instead of deployed. + */ + getMismatchedWorkspaceDeps(): Record { + const result: Record = {}; + for (const [path, node] of this.nodes.entries()) { + if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) { + result[path] = node.content; + } + } + return result; + } + + /** + * Returns path → contentHash for all transitive imports that have been uploaded. + * Must be called after uploadScripts() has populated contentHash values. + */ + getTempScriptRefs(scriptPath: string): Record { + const result: Record = {}; + this.traverseTransitive(scriptPath, (_path, node) => { + if (node.contentHash) { + result[_path] = node.contentHash; + } + }); + return result; + } + + /** + * Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies + * considers them up-to-date on the next run. + */ + async persistDepsHashes(depsPaths: string[]): Promise { + for (const path of depsPaths) { + const node = this.nodes.get(path); + if (node?.itemType === "dependencies" && node.content !== undefined) { + const hash = await generateHash(node.content + path); + await updateMetadataGlobalLock(path, hash); + } + } + } + + get size(): number { + return this.nodes.size; + } +} diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 0a3340cc6d..0e010a0cc5 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -26,11 +26,13 @@ import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts"; import { getIsWin } from "./utils.ts"; +import { extractRelativeImports } from "./relative_imports.ts"; +import { DoubleLinkedDependencyTree } from "./dependency_tree.ts"; const _require = createRequire(import.meta.url); const _parserCache = new Map>(); -function loadParser(pkgName: string): Promise { +export function loadParser(pkgName: string): Promise { let p = _parserCache.get(pkgName); if (!p) { p = (async () => { @@ -54,7 +56,7 @@ export class LockfileGenerationError extends Error { } -export async function getRawWorkspaceDependencies(): Promise> { +export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise> { const rawWorkspaceDeps: Record = {}; try { @@ -68,11 +70,13 @@ export async function getRawWorkspaceDependencies(): Promise, codebases: SyncCodebase[], - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, + legacyBehaviour?: boolean, + tree?: DoubleLinkedDependencyTree ): Promise { // Detect folder layout: my_script__mod/script.ts const isFolderLayout = isModuleEntryPoint(scriptPath); @@ -222,13 +228,15 @@ export async function generateScriptMetadataInternal( const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory(); - let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent); + // In non-legacy mode, workspace deps are tracked via the tree — exclude from hash + const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies; + let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent); // Compute per-module hashes for stale detection (like flow inline scripts) let moduleHashes: Record = {}; if (hasModules) { moduleHashes = await computeModuleHashes( - moduleFolderPath, opts.defaultTs, rawWorkspaceDependencies, isFolderLayout + moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout ); } const hasModuleHashes = Object.keys(moduleHashes).length > 0; @@ -243,27 +251,43 @@ export async function generateScriptMetadataInternal( } const conf = await readLockfile(); - if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { - if (!noStaleMessage) { - log.info( - colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) - ); + + // Use checkHash (includes module hashes) so module changes are detected as stale + const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)); + + // New behaviour: tree-based dependency tracking + if (!legacyBehaviour && tree) { + if (dryRun) { + // First pass: populate tree with script and its imports + const imports = await extractRelativeImports(scriptContent, remotePath, language); + await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale); + return; } - return; - } else if (dryRun) { - let detail = `${remotePath} (${language})`; - if (hasModuleHashes) { - const changed: string[] = []; - for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { - if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { - changed.push(modulePath); + // Second pass: proceed to generate (caller verified this script is stale via tree) + } else { + // Legacy behaviour: use existing staleness check + if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) { + if (!noStaleMessage) { + log.info( + colors.green(`Script ${remotePath} metadata is up-to-date, skipping`) + ); + } + return; + } else if (dryRun) { + let detail = `${remotePath} (${language})`; + if (hasModuleHashes) { + const changed: string[] = []; + for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { + if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) { + changed.push(modulePath); + } + } + if (changed.length > 0) { + detail += ` [changed modules: ${changed.join(", ")}]`; } } - if (changed.length > 0) { - detail += ` [changed modules: ${changed.join(", ")}]`; - } + return detail; } - return detail; } if (!justUpdateMetadataLock && !noStaleMessage) { @@ -288,6 +312,7 @@ export async function generateScriptMetadataInternal( const hasCodebase = findCodebase(scriptPath, codebases) != undefined; if (!hasCodebase) { + const tempScriptRefs = tree?.getTempScriptRefs(remotePath); const lockPathOverride = isFolderLayout ? path.dirname(scriptPath) + "/script.lock" : undefined; @@ -298,6 +323,7 @@ export async function generateScriptMetadataInternal( remotePath, metadataParsedContent, filteredRawWorkspaceDependencies, + tempScriptRefs, lockPathOverride, ); } else { @@ -358,7 +384,7 @@ export async function generateScriptMetadataInternal( const metadataContentUsedForHash = newMetadataContent; hash = await generateScriptHash( - filteredRawWorkspaceDependencies, + depsForHash, scriptContent, metadataContentUsedForHash ); @@ -511,6 +537,7 @@ export async function computeLockCacheKey( scriptContent: string, language: ScriptLanguage, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const annotation = extractWorkspaceDepsAnnotation(scriptContent, language); const annotationStr = annotation @@ -518,7 +545,10 @@ export async function computeLockCacheKey( : "none"; const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort(); const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";"); - return await generateHash(`${language}|${annotationStr}|${depsStr}`); + const tempRefsStr = tempScriptRefs + ? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";") + : ""; + return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`); } const lockCache = new Map(); @@ -533,13 +563,15 @@ async function fetchScriptLock( language: ScriptLanguage, remotePath: string, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record ): Promise { const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0; - const cacheKey = hasRawDeps - ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies) + const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0; + const cacheKey = (hasRawDeps || hasTempRefs) + ? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs) : undefined; if (cacheKey && lockCache.has(cacheKey)) { - log.info(`Using cached lockfile for ${remotePath}`); + log.debug(`Using cached lockfile for ${remotePath}`); return lockCache.get(cacheKey)!; } @@ -564,6 +596,8 @@ async function fetchScriptLock( raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 ? rawWorkspaceDependencies : null, entrypoint: remotePath, + temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0 + ? tempScriptRefs : null, }), } ); @@ -604,6 +638,7 @@ async function updateScriptLock( remotePath: string, metadataContent: Record, rawWorkspaceDependencies: Record, + tempScriptRefs?: Record, lockPathOverride?: string, ): Promise { if ( @@ -621,7 +656,7 @@ async function updateScriptLock( if (Object.keys(rawWorkspaceDependencies).length > 0) { const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', '); - log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); + log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`); } const lock = await fetchScriptLock( @@ -630,6 +665,7 @@ async function updateScriptLock( language, remotePath, rawWorkspaceDependencies, + tempScriptRefs ); const lockPath = lockPathOverride ?? remotePath + ".script.lock"; @@ -692,7 +728,7 @@ async function updateModuleLocks( const moduleContent = readFileSync(fullPath, "utf-8"); const moduleRemotePath = scriptRemotePath + "/" + relPath; - log.info(colors.gray(`Generating lock for module ${relPath}`)); + log.debug(`Generating lock for module ${relPath}`); try { const lock = await fetchScriptLock( diff --git a/cli/src/utils/relative_imports.ts b/cli/src/utils/relative_imports.ts new file mode 100644 index 0000000000..7ea2d582a6 --- /dev/null +++ b/cli/src/utils/relative_imports.ts @@ -0,0 +1,39 @@ +/** + * Relative Imports Utilities for CLI + * + * Provides functions to parse relative imports from TypeScript/Python scripts using WASM. + */ + +import { ScriptLanguage } from "./script_common.ts"; +import { loadParser } from "./metadata.ts"; +import * as log from "../core/log.ts"; + +/** + * Extract relative imports from script content based on language. + * Returns resolved absolute Windmill paths (e.g., "f/folder/helper"). + */ +export async function extractRelativeImports( + code: string, + scriptPath: string, + language: ScriptLanguage +): Promise { + try { + switch (language) { + case "bun": + case "nativets": + case "deno": { + const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts"); + return parse_ts_relative_imports(code, scriptPath); + } + case "python3": { + const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports"); + return parse_py_relative_imports(code, scriptPath); + } + default: + return []; + } + } catch (e) { + log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`); + return []; + } +} diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 713a47ecb1..1531314f04 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean { return isFlowPath(p) || isAppPath(p) || isRawAppPath(p); } +/** + * Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app) + * and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting. + * Use this instead of isFolderResourcePath when the config may not yet be loaded or when + * you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files). + */ +export function isFolderResourcePathAnyFormat(p: string): boolean { + const n = normalizeSep(p); + for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) { + if ( + n.includes(suffixes.flow + "/") || + n.includes(suffixes.app + "/") || + n.includes(suffixes.raw_app + "/") + ) { + return true; + } + } + return false; +} + /** * Detect the resource type from a path, if any */ diff --git a/cli/test/cargo_backend.ts b/cli/test/cargo_backend.ts index 5c9719f6d5..a25a788f7e 100644 --- a/cli/test/cargo_backend.ts +++ b/cli/test/cargo_backend.ts @@ -63,11 +63,11 @@ export class CargoBackend { // Determine default features based on environment // CI mode: minimal features (zip only) - // Local mode with license key: full features (zip, private, enterprise, license) + // Local mode with license key: full features (zip, private, enterprise, license, python) // Local mode without license key: zip only (EE features reject API calls without valid license) const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; const hasLicenseKey = !!process.env["EE_LICENSE_KEY"]; - const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license"] : ["zip"]); + const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license", "python"] : ["zip", "python"]); // Parse additional features from environment variable const envFeatures = process.env["TEST_FEATURES"]?.split(",").filter(f => f.trim()) || []; @@ -328,6 +328,8 @@ export class CargoBackend { SQLX_OFFLINE: "true", // Disable embedding to speed up startup DISABLE_EMBEDDING: "true", + // Skip worker version check for workspace deps (workers need time to report version) + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: "1", // Create default admin user CREATE_SUPERADMIN_IF_NOT_EXISTS: "1", SUPERADMIN_EMAIL: this.config.username, @@ -708,6 +710,7 @@ export class CargoBackend { this.deleteAll("resources"), this.deleteAll("variables"), this.deleteAll("folders"), + this.deleteAllWorkspaceDeps(), ]); console.log("Workspace reset complete"); @@ -735,6 +738,28 @@ export class CargoBackend { // Ignore listing failures } } + + private async deleteAllWorkspaceDeps(): Promise { + try { + const listResponse = await this.apiRequest(`/api/w/${this.config.workspace}/workspace_dependencies/list`); + if (!listResponse.ok) return; + + const items = await listResponse.json() as { language: string; name?: string }[]; + for (const item of items) { + try { + const nameParam = item.name ? `?name=${encodeURIComponent(item.name)}` : ""; + await this.apiRequest( + `/api/w/${this.config.workspace}/workspace_dependencies/delete/${item.language}${nameParam}`, + { method: "POST" } + ); + } catch { + // Ignore individual deletion failures + } + } + } catch { + // Ignore failures + } + } } // Global backend instance diff --git a/cli/test/relative_imports_skip.test.ts b/cli/test/relative_imports_skip.test.ts new file mode 100644 index 0000000000..e43b10aa12 --- /dev/null +++ b/cli/test/relative_imports_skip.test.ts @@ -0,0 +1,420 @@ +/** + * Relative Imports Tests + * + * E2E tests for the `generate-metadata` command with relative imports: + * - Lock files correctly include transitive dependencies + * - Staleness propagates through import chains + * - Various import patterns handled correctly + */ + +import { expect, test } from "bun:test"; +import { writeFile, readFile, mkdir } from "node:fs/promises"; +import { withTestBackend } from "./test_backend.ts"; + +// TODO: re-enable Python tests on CI if python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +const defaultMetadata = `summary: "Test" +schema: + type: object + properties: {} +lock: "" +`; + +// ============================================================================= +// Test 1: TS basic import with npm dependency propagation +// ============================================================================= + +test("TS: imported script's npm dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code, `generate-metadata failed:\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr}`).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 2: TS chained imports - dependency propagates through chain +// ============================================================================= + +test("TS: chained imports propagate npm deps through entire chain", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const scriptA = `import { utilB } from "./script_b.ts"; +export async function main() { return utilB(); } +`; + const scriptB = `import { utilC } from "./script_c.ts"; +export function utilB() { return utilC() + " B"; } +`; + const scriptC = `import _ from "lodash"; +export function utilC() { return _.VERSION; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + + expect(lockC).toContain("lodash"); + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 3: TS circular imports - completes without hanging, locks generated +// ============================================================================= + +test("TS: circular imports handled gracefully with correct locks", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Circular: A imports B, B imports A, B has npm dep + const scriptA = `import { funcB } from "./script_b.ts"; +export function funcA() { return "A"; } +export async function main() { return funcA() + funcB(); } +`; + const scriptB = `import { funcA } from "./script_a.ts"; +import _ from "lodash"; +export function funcB() { return _.VERSION + funcA(); } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + + expect(lockB).toContain("lodash"); + expect(lockA).toContain("lodash"); + }); +}); + +// ============================================================================= +// Test 4: Python basic import with pip dependency propagation +// ============================================================================= + +test.skipIf(isCI)("Python: imported script's pip dep appears in importer's lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPy); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 5: Diamond dependency - A imports B and C, both import D +// ============================================================================= + +test.skipIf(isCI)("Python: diamond dependency pattern propagates correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Diamond: A -> B, A -> C, B -> D, C -> D + const scriptA = `from f.test.script_b import func_b +from f.test.script_c import func_c + +def main(): + return func_b() + func_c() +`; + const scriptB = `from f.test.script_d import func_d + +def func_b(): + return "B" + func_d() +`; + const scriptC = `from f.test.script_d import func_d + +def func_c(): + return "C" + func_d() +`; + const scriptD = `import requests + +def func_d(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/test/script_a.py`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.py`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.py`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_d.py`, scriptD); + await writeFile(`${tempDir}/f/test/script_d.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => ""); + const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => ""); + const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => ""); + const lockD = await readFile(`${tempDir}/f/test/script_d.script.lock`, "utf-8").catch(() => ""); + + expect(lockD).toContain("requests"); + expect(lockB).toContain("requests"); + expect(lockC).toContain("requests"); + expect(lockA).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 6: Script isolation - unrelated script not marked stale +// ============================================================================= + +test("Script isolation: unrelated script not affected by changes", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // A imports B, C is isolated + const scriptA = `import { helper } from "./script_b.ts"; +export async function main() { return helper(); } +`; + const scriptB = `export function helper() { return "B"; } +`; + const scriptC = `export async function main() { return "isolated"; } +`; + + await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA); + await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB); + await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC); + await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("All metadata up-to-date"); + + // Change script_b + await writeFile(`${tempDir}/f/test/script_b.ts`, + `export function helper() { return "B changed"; } +`); + + // script_a and script_b should be stale, script_c should NOT be mentioned + const check2 = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("script_b"); + expect(check2.stdout).toContain("script_a"); + expect(check2.stdout).not.toMatch(/script_c/); + }); +}); + +// ============================================================================= +// Test 7: Python relative imports with dot syntax +// ============================================================================= + +test.skipIf(isCI)("Python: relative imports with dot syntax work correctly", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/mymodule`, { recursive: true }); + + // Using relative import syntax + const mainPy = `from .helper import helper_func + +def main(): + return helper_func() +`; + const helperPy = `import requests + +def helper_func(): + return requests.__version__ +`; + + await writeFile(`${tempDir}/f/mymodule/main.py`, mainPy); + await writeFile(`${tempDir}/f/mymodule/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/mymodule/helper.py`, helperPy); + await writeFile(`${tempDir}/f/mymodule/helper.script.yaml`, defaultMetadata); + + const result = await backend.runCLICommand( + ["generate-metadata", "-i", "f/mymodule/*", "--yes"], + tempDir + ); + expect(result.code).toBe(0); + + const lockMain = await readFile(`${tempDir}/f/mymodule/main.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/mymodule/helper.script.lock`, "utf-8").catch(() => ""); + + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); + +// ============================================================================= +// Test 8: Adding new import updates importer's lock +// ============================================================================= + +test.skipIf(isCI)("Python: adding new import updates importer's lock correctly", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"] +excludes: []`); + + await mkdir(`${tempDir}/f/test`, { recursive: true }); + + // Initial: main imports helper, helper has no external deps + const mainPy = `from f.test.helper import helper_func + +def main(): + return helper_func() +`; + const helperPyInitial = `def helper_func(): + return "no deps" +`; + + await writeFile(`${tempDir}/f/test/main.py`, mainPy); + await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata); + await writeFile(`${tempDir}/f/test/helper.py`, helperPyInitial); + await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata); + + // Generate initial locks + const initial = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Add new script with pip dep + const utilsPy = `import requests + +def get_version(): + return requests.__version__ +`; + await writeFile(`${tempDir}/f/test/utils.py`, utilsPy); + await writeFile(`${tempDir}/f/test/utils.script.yaml`, defaultMetadata); + + // Modify helper to import utils + const helperPyWithImport = `from f.test.utils import get_version + +def helper_func(): + return get_version() +`; + await writeFile(`${tempDir}/f/test/helper.py`, helperPyWithImport); + + // Regenerate - main should now have requests + const afterAdd = await backend.runCLICommand( + ["generate-metadata", "-i", "f/test/*", "--yes"], + tempDir + ); + expect(afterAdd.code).toBe(0); + + const lockUtils = await readFile(`${tempDir}/f/test/utils.script.lock`, "utf-8").catch(() => ""); + const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => ""); + const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => ""); + + expect(lockUtils).toContain("requests"); + expect(lockHelper).toContain("requests"); + expect(lockMain).toContain("requests"); + }); +}); diff --git a/cli/test/relative_imports_wasm.test.ts b/cli/test/relative_imports_wasm.test.ts new file mode 100644 index 0000000000..d1d537b001 --- /dev/null +++ b/cli/test/relative_imports_wasm.test.ts @@ -0,0 +1,1235 @@ +/** + * Tests for relative import resolution: + * 1. WASM parser unit tests — verify parse_ts/py_relative_imports work correctly + * 2. E2E tests — verify dependency propagation through scripts, flows, apps, and raw apps + * using the CLI generate-metadata command against a real backend + */ + +import { expect, test, describe, beforeAll, afterAll } from "bun:test"; +import { readFile, readdir, writeFile, mkdir } from "node:fs/promises"; +import { loadParser } from "../src/utils/metadata.ts"; +import { extractRelativeImports } from "../src/utils/relative_imports.ts"; +import { withTestBackend, type TestBackend, createRemoteWorkspaceDeps } from "./test_backend.ts"; +import { + createLocalScript, + createLocalFlow, + createLocalApp, + createLocalRawApp, +} from "./test_fixtures.ts"; +import { setNonDottedPaths } from "../src/utils/resource_folders.ts"; + +// TODO: re-enable on CI when python feature is included by default +const isCI = process.env["CI_MINIMAL_FEATURES"] === "true"; + +// ============================================================================= +// WASM Parser Unit Tests +// ============================================================================= + +describe("WASM TS parser exports parse_ts_relative_imports", () => { + test("parse_ts_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-ts"); + expect(typeof mod.parse_ts_relative_imports).toBe("function"); + }); + + test("resolves dot-relative import", async () => { + const code = `import { helper } from "./helper";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves double-dot-relative import", async () => { + const code = `import { utils } from "../utils/helper";\nexport async function main() { return utils(); }`; + const result = await extractRelativeImports(code, "f/folder/sub/script", "bun"); + expect(result).toEqual(["f/folder/utils/helper"]); + }); + + test("resolves absolute windmill import", async () => { + const code = `import { shared } from "/f/shared/utils";\nexport async function main() { return shared(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/shared/utils"]); + }); + + test("ignores external package imports", async () => { + const code = `import lodash from "lodash";\nimport axios from "axios";\nexport async function main() { return lodash.map([]); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual([]); + }); + + test("strips .ts extension from imports", async () => { + const code = `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper"]); + }); + + test("resolves mixed relative and external imports", async () => { + const code = `import { helper } from "./helper";\nimport { utils } from "../utils";\nimport lodash from "lodash";\nexport async function main() { return helper(); }`; + const result = await extractRelativeImports(code, "f/folder/script", "bun"); + expect(result).toEqual(["f/folder/helper", "f/utils"]); + }); + + test("works with named imports", async () => { + const code = `import { slugify, capitalize } from "./string_helpers";\nexport async function main() { return slugify("test"); }`; + const result = await extractRelativeImports(code, "f/utils/http_client", "bun"); + expect(result).toEqual(["f/utils/string_helpers"]); + }); +}); + +describe("WASM Python parser exports parse_py_relative_imports", () => { + test("parse_py_relative_imports function exists in WASM module", async () => { + const mod = await loadParser("windmill-parser-wasm-py-imports"); + expect(typeof mod.parse_py_relative_imports).toBe("function"); + }); + + test("resolves python relative import", async () => { + const code = `from f.utils.formatter import format_stats\ndef main(values: list):\n return format_stats(values)`; + const result = await extractRelativeImports(code, "f/data/process", "python3"); + expect(result).toEqual(["f/utils/formatter"]); + }); +}); + +// ============================================================================= +// Helper: find all .lock files recursively in a directory +// ============================================================================= + +async function findLockFiles(dir: string): Promise { + const entries = await readdir(dir, { recursive: true }); + return entries + .filter((e) => e.endsWith(".lock")) + .map((e) => `${dir}/${e}`); +} + +async function anyLockContains(dir: string, needle: string): Promise { + const lockFiles = await findLockFiles(dir); + for (const lockFile of lockFiles) { + const content = await readFile(lockFile, "utf-8").catch(() => ""); + if (content.includes(needle)) return true; + } + return false; +} + +async function createRemoteScript( + backend: TestBackend, + scriptPath: string, + content: string = 'export async function main() { return "hello"; }', + language: string = "bun" +): Promise { + // Archive any existing script at this path first to avoid hash conflicts + await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/delete/p/${encodeURIComponent(scriptPath)}`, + { method: "POST" } + ).catch(() => {}); + + const resp = await backend.apiRequest!( + `/api/w/${backend.workspace}/scripts/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: scriptPath, + content, + language, + summary: "Test script", + description: "Created by integration test", + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + } + ); + const respText = await resp.text(); + if (resp.status >= 300) { + console.log(`createRemoteScript ${scriptPath} (${language}) failed: ${resp.status} ${respText}`); + } + expect(resp.status).toBeLessThan(300); +} + +// ============================================================================= +// E2E Tests: Dependency propagation through relative imports +// ============================================================================= + +const helperScript = `import _ from "lodash"; +export function helper() { return _.VERSION; } +`; + +const importerScript = `import { helper } from "/f/test/helper.ts"; +export async function main() { return helper(); } +`; + +const pyHelperScript = `import requests + +def helper(): + return requests.__version__ +`; + +const pyImporterScript = `from f.test.py_helper import helper + +def main(): + return helper() +`; + +for (const nonDotted of [false, true]) { +describe(`E2E: relative import dependency propagation via generate-metadata (${nonDotted ? "non-dotted" : "dotted"} paths)`, () => { + const inlineSuffix = nonDotted ? "" : ".inline_script"; + const flowSuffix = nonDotted ? "__flow" : ".flow"; + const appSuffix = nonDotted ? "__app" : ".app"; + const rawAppSuffix = nonDotted ? "__raw_app" : ".raw_app"; + const wmillYaml = nonDotted + ? `defaultTs: bun\nincludes: ["**"]\nexcludes: []\nnonDottedPaths: true` + : `defaultTs: bun\nincludes: ["**"]\nexcludes: []`; + + beforeAll(() => { + setNonDottedPaths(nonDotted); + }); + afterAll(() => { + setNonDottedPaths(false); + }); + test("script importing another script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // helper has lodash dep + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + // consumer imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + + expect(helperLock).toContain("lodash"); + expect(consumerLock).toContain("lodash"); + }); + }); + + test("flow inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // Helper script lock should have lodash + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Flow inline script lock should also have lodash (transitive via helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + const flowHasLodash = await anyLockContains(flowDir, "lodash"); + expect(flowHasLodash).toBe(true); + }); + }); + + test("app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // App inline script lock should have lodash (transitive via helper) + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + const appHasLodash = await anyLockContains(appDir, "lodash"); + expect(appHasLodash).toBe(true); + }); + }); + + test("raw app inline script importing a script gets transitive npm deps in lock", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const helperLock = await readFile( + `${tempDir}/f/test/helper.script.lock`, + "utf-8" + ).catch(() => ""); + expect(helperLock).toContain("lodash"); + + // Raw app inline script lock should have lodash (transitive via helper) + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + const rawAppHasLodash = await anyLockContains(rawAppDir, "lodash"); + expect(rawAppHasLodash).toBe(true); + }); + }); + + test("modifying leaf script marks all dependents as stale", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Generate initial metadata + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(initial.code).toBe(0); + + // Verify all up to date + const check1 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check1.stdout).toContain("up-to-date"); + + // Modify the leaf helper script (change content but keep lodash dep) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION + " v2"; }\n` + ); + + // All dependents should now be detected as stale + const check2 = await backend.runCLICommand( + ["generate-metadata", "--dry-run"], + tempDir + ); + expect(check2.code).toBe(0); + expect(check2.stdout).toContain("helper"); + expect(check2.stdout).toContain("consumer"); + expect(check2.stdout).toContain("my_flow"); + expect(check2.stdout).toContain("my_app"); + expect(check2.stdout).toContain("my_raw_app"); + }); + }); + + test("new script importing locally modified helper gets local deps not remote", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy helper (lodash) to backend, then pull locally + // Content includes literal \n to exercise Postgres bytea cast bug (content::bytea fails on backslash) + const helperWithBackslash = `import _ from "lodash";\nexport function helper() { return "line1\\nline2"; }\n`; + await createRemoteScript(backend, "f/test/helper", helperWithBackslash); + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + if (pull.code !== 0) { + console.log("PULL STDOUT:", pull.stdout); + console.log("PULL STDERR:", pull.stderr); + } + expect(pull.code).toBe(0); + + // Modify helper locally to use axios instead of lodash (NOT pushed) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + + // Regenerate helper metadata — helper is stale (content changed from deployed) + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Create a new consumer that imports helper + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Consumer is stale (new), helper is NOT stale (metadata up-to-date). + // Helper differs from deployed (axios vs lodash). + // Diff endpoint should detect mismatch, upload local helper. + // Consumer's lock must have axios (local), not lodash (deployed). + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("axios"); + expect(consumerLock).not.toContain("lodash"); + }); + }); + + test("new script importing unpushed helper gets transitive deps in lock", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create helper (lodash dep) and generate its metadata + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + const run1 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + expect(run1.code).toBe(0); + + // Now create a NEW consumer that imports helper + // Helper is not stale (metadata up-to-date) and was never pushed to remote + await createLocalScript( + tempDir, + "f/test", + "consumer", + "bun", + `import { helper } from "./helper.ts";\nexport async function main() { return helper(); }` + ); + + // Run 2: only consumer is stale (new). Helper is NOT stale. + // Consumer's lock must include lodash (transitive dep from local helper) + const run2 = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (run2.code !== 0) { + console.log("STDOUT:", run2.stdout); + console.log("STDERR:", run2.stderr); + } + expect(run2.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/test/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + test.skipIf(isCI)("dependency change triggers lock regeneration for flows and apps", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Step 1: Create helper scripts locally and deploy them to remote via push + await createLocalScript(tempDir, "f/test", "helper", "bun", helperScript); + await createLocalScript(tempDir, "f/test", "py_helper", "python3", pyHelperScript); + + // Generate metadata for scripts only, then push to deploy them on remote + const genScripts = await backend.runCLICommand( + ["generate-metadata", "--yes", "--skip-flows", "--skip-apps"], + tempDir + ); + expect(genScripts.code).toBe(0); + + const push = await backend.runCLICommand(["sync", "push", "--yes"], tempDir); + if (push.code !== 0) { + console.log("PUSH STDOUT:", push.stdout); + console.log("PUSH STDERR:", push.stderr); + } + expect(push.code).toBe(0); + + // Step 2: Now create flows/apps that import the deployed helpers + await createLocalFlow(tempDir, "f/test", "my_flow", importerScript); + await createLocalFlow(tempDir, "f/test", "my_py_flow", pyImporterScript, "python3"); + await createLocalApp(tempDir, "f/test", "my_app", importerScript); + await createLocalRawApp(tempDir, "f/test", "my_raw_app", importerScript); + + // Step 3: Generate initial metadata for everything + const initial = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (initial.code !== 0) { + console.log("INITIAL STDOUT:", initial.stdout); + console.log("INITIAL STDERR:", initial.stderr); + } + expect(initial.code).toBe(0); + + // Verify flow directories have correct structure — no extra files created + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect((await readdir(flowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.ts`, "flow.yaml"]); + const pyFlowDir = `${tempDir}/f/test/my_py_flow${flowSuffix}`; + expect((await readdir(pyFlowDir)).sort()).toEqual([`a${inlineSuffix}.lock`, `a${inlineSuffix}.py`, "flow.yaml"]); + + // Verify TS flow/app locks have lodash + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + + // Verify Python flow lock has requests + expect(await anyLockContains(pyFlowDir, "requests")).toBe(true); + + // Step 5: Modify helpers LOCALLY (NOT pushed to remote — this is the key scenario) + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios.VERSION; }\n` + ); + await createLocalScript( + tempDir, + "f/test", + "py_helper", + "python3", + `import pandas\n\ndef helper():\n return pandas.__version__\n` + ); + + // Step 6: Regenerate — flow locks must use LOCAL helper content, not remote deployed version + const regen = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (regen.code !== 0) { + console.log("REGEN STDOUT:", regen.stdout); + console.log("REGEN STDERR:", regen.stderr); + } + expect(regen.code).toBe(0); + + // TS flow lock should now have axios (from local helper), not lodash (from remote) + expect(await anyLockContains(flowDir, "axios")).toBe(true); + expect(await anyLockContains(flowDir, "lodash")).toBe(false); + + // Python flow lock should now have pandas (from local helper), not requests (from remote) + const pyFlowLockFiles = await findLockFiles(pyFlowDir); + for (const f of pyFlowLockFiles) { + const c = await readFile(f, "utf-8").catch(() => ""); + console.log(`PY FLOW LOCK [${f}]: ${c.substring(0, 500)}`); + } + expect(await anyLockContains(pyFlowDir, "pandas")).toBe(true); + expect(await anyLockContains(pyFlowDir, "requests")).toBe(false); + + // TS app lock should now have axios, not lodash + expect(await anyLockContains(appDir, "axios")).toBe(true); + expect(await anyLockContains(appDir, "lodash")).toBe(false); + + // Raw app lock should now have axios, not lodash + expect(await anyLockContains(rawAppDir, "axios")).toBe(true); + expect(await anyLockContains(rawAppDir, "lodash")).toBe(false); + }); + }); + + test.skipIf(isCI)("locally modified workspace deps are used for lock generation instead of remote", { timeout: 180000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy named Python workspace dep "test" with requests on the remote + await createRemoteWorkspaceDeps(backend, "python3", "requests", "test"); + + // Pull to get remote state locally + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Empty the workspace dep locally (NOT pushed) — simulates removing all deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/test.requirements.in`, + "" + ); + + // Create a Python script that uses the named workspace dep via #requirements: test + await createLocalScript( + tempDir, + "f/test", + "my_script", + "python3", + `#requirements: test\n\ndef main():\n return "hello"\n` + ); + + // Generate metadata — local dep is empty, so lock must NOT contain requests (from remote) + const gen = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock must use local (empty) workspace deps, not remote (requests) + expect(scriptLock).not.toContain("requests"); + + // Second run should be idempotent — no stale items + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("unchanged workspace deps do not cause dependents to be stale on subsequent runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/test", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First generate-metadata — everything is stale, locks get generated + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code, `generate-metadata failed:\nSTDOUT: ${gen1.stdout}\nSTDERR: ${gen1.stderr}`).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/test/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + expect(scriptLock).toContain("axios"); + + // Second generate-metadata — nothing changed, should report "All metadata up-to-date" + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code, `generate-metadata failed:\nSTDOUT: ${gen2.stdout}\nSTDERR: ${gen2.stderr}`).toBe(0); + + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("All metadata up-to-date"); + // Should NOT show workspace deps or scripts as stale + expect(output).not.toContain("stale metadata"); + }); + }); + + test("diff endpoint correctly identifies mismatched scripts and workspace deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + const { generateHash } = await import("../src/utils/utils.ts"); + const { setClient } = await import("../src/core/client.ts"); + const { diffRawScriptsWithDeployed } = await import("../gen/services.gen.ts"); + + setClient(backend.token, backend.baseUrl); + + // Deploy a script and two workspace deps (bun + python) + const scriptContent = `export async function main() { return "hello"; }`; + await createRemoteScript(backend, "f/test/deployed_script", scriptContent); + + const bunDepsContent = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", bunDepsContent); + + const scriptHash = await generateHash(scriptContent); + const bunDepsHash = await generateHash(bunDepsContent); + const wrongHash = await generateHash("totally different content"); + + // Call 1: matching hash, same path → should NOT be mismatched + const call1 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { "f/test/deployed_script": scriptHash }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: bunDepsHash }, + ], + }, + }); + expect(call1).not.toContain("f/test/deployed_script"); + expect(call1).not.toContain("dependencies/package.json"); + + // Call 2: wrong hash same path + right hash wrong path → both should be mismatched + const call2 = await diffRawScriptsWithDeployed({ + workspace: backend.workspace, + requestBody: { + scripts: { + "f/test/deployed_script": wrongHash, + "f/test/nonexistent_script": scriptHash, + }, + workspace_deps: [ + { path: "dependencies/package.json", language: "bun", hash: wrongHash }, + { path: "dependencies/requirements.in", language: "python3", hash: bunDepsHash }, + ], + }, + }); + // Same path, wrong hash → mismatched + expect(call2).toContain("f/test/deployed_script"); + expect(call2).toContain("dependencies/package.json"); + // Wrong path, right hash → mismatched (endpoint should not match by hash alone) + expect(call2).toContain("f/test/nonexistent_script"); + expect(call2).toContain("dependencies/requirements.in"); + }); + }); + + test("folder arg includes importers outside the folder by default", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A from a different directory + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // First: generate-metadata globally to establish baseline locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Now modify helper to use axios instead + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run generate-metadata for f/lib only — consumer (in f/app) should also be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer's lock should now have axios (updated even though it's outside f/lib) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("axios"); + }); + }); + + test("--strict-folder-boundaries skips importers outside the folder and warns", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Script A in f/lib — has lodash dep + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Script B in f/app — imports A + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { helper } from "/f/lib/helper.ts";\nexport async function main() { return helper(); }` + ); + + // Establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const consumerLock1 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock1).toContain("lodash"); + + // Modify helper to use axios + await createLocalScript( + tempDir, + "f/lib", + "helper", + "bun", + `import axios from "axios";\nexport function helper() { return axios; }` + ); + + // Run with --strict-folder-boundaries — consumer should NOT be updated + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + if (gen2.code !== 0) { + console.log("STDOUT:", gen2.stdout); + console.log("STDERR:", gen2.stderr); + } + expect(gen2.code).toBe(0); + + // Consumer lock should still have lodash (not updated) + const consumerLock2 = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock2).toContain("lodash"); + expect(consumerLock2).not.toContain("axios"); + + // Output should contain a warning about the skipped importer + const output = gen2.stdout + gen2.stderr; + expect(output).toContain("Warning"); + expect(output).toContain("f/app/consumer"); + + // Running again with same args should report up-to-date (not stuck in loop) + const gen3 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/lib"], + tempDir + ); + expect(gen3.code).toBe(0); + const output3 = gen3.stdout + gen3.stderr; + expect(output3).toContain("All metadata up-to-date"); + }); + }); + + // TODO: consider adding --skip-workspace-dependencies flag to generate-metadata + test("folder arg includes workspace dependencies in lock generation", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // Run generate-metadata for f/mydir only — should still include workspace deps content + // TODO: consider adding --skip-workspace-dependencies flag + const gen = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + if (gen.code !== 0) { + console.log("STDOUT:", gen.stdout); + console.log("STDERR:", gen.stderr); + } + expect(gen.code).toBe(0); + + const scriptLock = await readFile( + `${tempDir}/f/mydir/my_script.script.lock`, + "utf-8" + ).catch(() => ""); + // Lock should use local workspace deps (axios), not remote (lodash) + expect(scriptLock).toContain("axios"); + expect(scriptLock).not.toContain("lodash"); + + // Running again should report up-to-date (not stuck in loop) + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + test("strict folder boundaries with workspace deps does not loop", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Create local workspace deps + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a script that uses workspace deps + await createLocalScript( + tempDir, + "f/mydir", + "my_script", + "bun", + `export async function main() { return "hello"; }` + ); + + // First run with strict + folder + const gen1 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not stuck in loop + const gen2 = await backend.runCLICommand( + ["generate-metadata", "--yes", "--strict-folder-boundaries", "f/mydir"], + tempDir + ); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #1: flow/app with mismatched workspace deps — hash inconsistency causes perpetual staleness + test("flow/app/raw app with workspace deps does not loop across runs", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Deploy workspace deps with lodash on remote + const remotePackageJson = JSON.stringify({ dependencies: { lodash: "^4" } }); + await createRemoteWorkspaceDeps(backend, "bun", remotePackageJson); + + // Pull to get remote state + const pull = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir); + expect(pull.code).toBe(0); + + // Modify workspace deps locally to use axios + await mkdir(`${tempDir}/dependencies`, { recursive: true }); + await writeFile( + `${tempDir}/dependencies/package.json`, + JSON.stringify({ dependencies: { axios: "^1" } }) + ); + + // Create a flow, app, and raw app with inline scripts + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `export async function main() { return "hello from flow"; }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `export async function main() { return "hello from app"; }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `export async function main() { return "hello from raw app"; }` + ); + + // First run — generates locks + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + if (gen1.code !== 0) { + console.log("STDOUT:", gen1.stdout); + console.log("STDERR:", gen1.stderr); + } + expect(gen1.code).toBe(0); + + // Second run — should report up-to-date, not loop + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + const output2 = gen2.stdout + gen2.stderr; + expect(output2).toContain("All metadata up-to-date"); + }); + }); + + // Bug #2: flow/app/raw app importing locally-modified helper uses local content not remote + test("flow/app/raw app importing locally modified helper uses local content", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper with lodash dep — only exists locally, never pushed + await createLocalScript( + tempDir, + "f/test", + "helper", + "bun", + `import _ from "lodash";\nexport function helper() { return _.VERSION; }` + ); + + // Flow, app, and raw app inline scripts import the helper + await createLocalFlow( + tempDir, + "f/test", + "my_flow", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalApp( + tempDir, + "f/test", + "my_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + await createLocalRawApp( + tempDir, + "f/test", + "my_raw_app", + `import { helper } from "/f/test/helper.ts";\nexport async function main() { return helper(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + // All locks should contain lodash (transitive dep from local helper) + const flowDir = `${tempDir}/f/test/my_flow${flowSuffix}`; + expect(await anyLockContains(flowDir, "lodash")).toBe(true); + + const appDir = `${tempDir}/f/test/my_app${appSuffix}`; + expect(await anyLockContains(appDir, "lodash")).toBe(true); + + const rawAppDir = `${tempDir}/f/test/my_raw_app${rawAppSuffix}`; + expect(await anyLockContains(rawAppDir, "lodash")).toBe(true); + }); + }); + + // Cross-directory relative imports with ../ + test("cross-directory relative import with ../ propagates deps", { timeout: 60000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // Helper in f/shared with lodash dep + await createLocalScript( + tempDir, + "f/shared", + "utils", + "bun", + `import _ from "lodash";\nexport function utils() { return _.VERSION; }` + ); + + // Script in f/app imports via ../shared/utils + await createLocalScript( + tempDir, + "f/app", + "consumer", + "bun", + `import { utils } from "../shared/utils.ts";\nexport async function main() { return utils(); }` + ); + + const result = await backend.runCLICommand( + ["generate-metadata", "--yes"], + tempDir + ); + if (result.code !== 0) { + console.log("STDOUT:", result.stdout); + console.log("STDERR:", result.stderr); + } + expect(result.code).toBe(0); + + const consumerLock = await readFile( + `${tempDir}/f/app/consumer.script.lock`, + "utf-8" + ).catch(() => ""); + expect(consumerLock).toContain("lodash"); + }); + }); + + // Multi-level transitive chain: A -> B -> C, C changes, A must update + test("3-level transitive chain propagates staleness", { timeout: 120000 }, async () => { + await withTestBackend(async (backend, tempDir) => { + await writeFile( + `${tempDir}/wmill.yaml`, + wmillYaml + ); + + // C has dayjs dep + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import dayjs from "dayjs";\nexport function c() { return dayjs(); }` + ); + + // B imports C + await createLocalScript( + tempDir, + "f/chain", + "b", + "bun", + `import { c } from "./c.ts";\nexport function b() { return c(); }` + ); + + // A imports B + await createLocalScript( + tempDir, + "f/chain", + "a", + "bun", + `import { b } from "/f/chain/b.ts";\nexport async function main() { return b(); }` + ); + + // First run — establish baseline + const gen1 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen1.code).toBe(0); + + const aLock1 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock1).toContain("dayjs"); + + // Modify C to use uuid instead + await createLocalScript( + tempDir, + "f/chain", + "c", + "bun", + `import { v4 } from "uuid";\nexport function c() { return v4(); }` + ); + + // Second run — A should be updated transitively (C changed -> B stale -> A stale) + const gen2 = await backend.runCLICommand(["generate-metadata", "--yes"], tempDir); + expect(gen2.code).toBe(0); + + const aLock2 = await readFile(`${tempDir}/f/chain/a.script.lock`, "utf-8").catch(() => ""); + expect(aLock2).toContain("uuid"); + }); + }); +}); +} // end for nonDotted diff --git a/cli/test/resource_folders_unit.test.ts b/cli/test/resource_folders_unit.test.ts index da8ccd7ece..80732ebc1c 100644 --- a/cli/test/resource_folders_unit.test.ts +++ b/cli/test/resource_folders_unit.test.ts @@ -15,6 +15,7 @@ import { isAppPath, isRawAppPath, isFolderResourcePath, + isFolderResourcePathAnyFormat, detectFolderResourceType, isRawAppBackendPath, isAppInlineScriptPath, @@ -214,6 +215,39 @@ describe("isFolderResourcePath", () => { }); }); +// This is the bug that isFolderResourcePathAnyFormat fixes: +// when nonDottedPaths is false (default), isFolderResourcePath misses non-dotted paths +// like "f/my_raw__raw_app/backend/a.ts", causing raw app backend scripts to leak +// into the standalone script list during generate-metadata. +describe("isFolderResourcePathAnyFormat", () => { + test("detects non-dotted paths even when global setting is dotted", () => { + setNonDottedPaths(false); + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow__flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard__app/inline.ts")).toBe(true); + }); + + test("detects dotted paths even when global setting is non-dotted", () => { + setNonDottedPaths(true); + expect(isFolderResourcePathAnyFormat("f/my_raw.raw_app/backend/a.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/my_flow.flow/step.ts")).toBe(true); + expect(isFolderResourcePathAnyFormat("f/dashboard.app/inline.ts")).toBe(true); + }); + + test("rejects non-folder-resource paths", () => { + expect(isFolderResourcePathAnyFormat("f/my_script.ts")).toBe(false); + expect(isFolderResourcePathAnyFormat("f/var.variable.yaml")).toBe(false); + }); + + test("confirms isFolderResourcePath fails for mismatched format (the bug)", () => { + setNonDottedPaths(false); + // isFolderResourcePath misses non-dotted paths when setting is dotted + expect(isFolderResourcePath("f/my_raw__raw_app/backend/a.ts")).toBe(false); + // isFolderResourcePathAnyFormat catches it + expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true); + }); +}); + describe("detectFolderResourceType", () => { test("detects flow type", () => { expect(detectFolderResourceType("f/x.flow/flow.yaml")).toBe("flow"); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 40e8ac4fdf..93f42bee54 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -6,7 +6,8 @@ * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): * @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests) - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * * This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript * If you add new helpers, update cross-links in the files above. diff --git a/cli/test/test_backend.ts b/cli/test/test_backend.ts index 16fb79528d..34ef1f240f 100644 --- a/cli/test/test_backend.ts +++ b/cli/test/test_backend.ts @@ -24,7 +24,8 @@ * @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * - * This file contains: API-based creation helpers (createTestApp, createTestResource, etc.) + * This file contains: API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * If you add new helpers, update cross-links in the files above. */ @@ -64,6 +65,10 @@ export interface TestBackend { listAllApps?(): Promise; listAllResources?(): Promise; listAllVariables?(): Promise; + + // Methods for creating apps and flows with custom inline scripts + createAppWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; + createFlowWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise; } /** @@ -342,6 +347,88 @@ class CargoBackendAdapter implements TestBackend { if (!response.ok) return []; return response.json(); } + + async createAppWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + value: { + type: "app", + grid: [ + { + id: "button1", + data: { + type: "buttoncomponent", + componentInput: { + type: "runnable", + runnable: { + type: "runnableByName", + inlineScript: { + content: inlineScriptContent, + language, + }, + }, + }, + }, + }, + ], + hiddenInlineScripts: [], + css: {}, + norefreshbar: false, + }, + summary: "Test app with inline script", + policy: { + on_behalf_of: null, + on_behalf_of_email: null, + triggerables: {}, + execution_mode: "viewer", + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create app ${path}: ${error}`); + } + await response.text(); + } + + async createFlowWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise { + const response = await this.backend.apiRequest(`/api/w/${this.workspace}/flows/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path, + summary: "Test flow with inline script", + description: `Flow at ${path}`, + value: { + modules: [ + { + id: "a", + value: { + type: "rawscript", + content: inlineScriptContent, + language, + input_transforms: {}, + }, + }, + ], + }, + schema: { + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + properties: {}, + required: [], + }, + }), + }); + if (!response.ok) { + const error = await response.text(); + throw new Error(`Failed to create flow ${path}: ${error}`); + } + await response.text(); + } } /** @@ -580,6 +667,38 @@ export async function createNonAdminUser( return await loginResp.text(); } +/** + * Create workspace dependencies via the API (e.g. a shared package.json for bun scripts). + */ +export async function createRemoteWorkspaceDeps( + backend: TestBackend, + language: string, + content: string, + name?: string, +): Promise { + if (!backend.apiRequest) { + throw new Error("Backend does not support apiRequest"); + } + + const resp = await backend.apiRequest( + `/api/w/${backend.workspace}/workspace_dependencies/create`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + workspace_id: backend.workspace, + language, + content, + ...(name ? { name } : {}), + }), + } + ); + if (!resp.ok) { + throw new Error(`Failed to create workspace deps (${resp.status}): ${await resp.text()}`); + } + await resp.text(); +} + // Re-export for convenience export type { CargoBackendConfig } from "./cargo_backend.ts"; export type { ContainerConfig } from "./containerized_backend.ts"; diff --git a/cli/test/test_fixtures.ts b/cli/test/test_fixtures.ts index cde57e1a95..1d2f6f6bb5 100644 --- a/cli/test/test_fixtures.ts +++ b/cli/test/test_fixtures.ts @@ -8,7 +8,8 @@ * - Local creation functions: Create fixtures AND write them to disk * * CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers): - * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, etc.) + * @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource, + * createAppWithInlineScript, createFlowWithInlineScript, etc.) * @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based) * * This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.) @@ -29,6 +30,7 @@ import { getFolderSuffix, getMetadataFileName, getModuleFolderSuffix, + getNonDottedPaths, } from "../src/utils/resource_folders.ts"; // ============================================================================= @@ -47,7 +49,8 @@ export interface ScriptFixture { export interface FlowFixture { metadata: FileFixture; - inlineScript: FileFixture; + inlineScript?: FileFixture; + inlineLock?: FileFixture; } export interface AppFixture { @@ -153,16 +156,34 @@ kind: script */ export function createFlowFixture( name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): FlowFixture { const flowSuffix = getFolderSuffix("flow"); const metadataFile = getMetadataFileName("flow", "yaml"); - const scriptContent = - inlineScriptContent ?? - `export async function main() {\n return "Hello from flow ${name}";\n}`; + const defaultContent = language === "python3" + ? `def main():\n return "Hello from flow ${name}"` + : `export async function main() {\n return "Hello from flow ${name}";\n}`; - return { + const scriptContent = inlineScriptContent ?? defaultContent; + + const langMap: Record = { bun: "bun", python3: "python3" }; + const extMap: Record = { bun: "ts", python3: "py" }; + + const ext = extMap[language]; + // With dotted paths (.flow), inline scripts use .inline_script suffix (a.inline_script.ts) + // With non-dotted paths (__flow), they don't (a.ts) + const inlineSuffix = getNonDottedPaths() ? "" : ".inline_script"; + const scriptFile = `a${inlineSuffix}.${ext}`; + const lockFile = `a${inlineSuffix}.lock`; + + const lockLine = lockContent !== undefined + ? `\n lock: "!inline ${lockFile}"` + : ""; + + const result: FlowFixture = { metadata: { path: `${name}${flowSuffix}/${metadataFile}`, content: `summary: "${name} flow" @@ -172,9 +193,8 @@ value: - id: a value: type: rawscript - content: | - ${scriptContent.split("\n").join("\n ")} - language: bun + content: "!inline ${scriptFile}"${lockLine} + language: ${langMap[language]} input_transforms: {} schema: $schema: "https://json-schema.org/draft/2020-12/schema" @@ -184,10 +204,19 @@ schema: `, }, inlineScript: { - path: `${name}${flowSuffix}/a.inline_script.ts`, + path: `${name}${flowSuffix}/${scriptFile}`, content: scriptContent, }, }; + + if (lockContent !== undefined) { + result.inlineLock = { + path: `${name}${flowSuffix}/${lockFile}`, + content: lockContent, + }; + } + + return result; } // ============================================================================= @@ -211,10 +240,14 @@ schema: * * @keywords app fixture, create app, local app */ -export function createAppFixture(name: string): AppFixture { +export function createAppFixture(name: string, inlineScriptContent?: string): AppFixture { const appSuffix = getFolderSuffix("app"); const metadataFile = getMetadataFileName("app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main() {\n return "hello from app";\n}`; + const indented = scriptContent.split("\n").join("\n "); + return { metadata: { path: `${name}${appSuffix}/${metadataFile}`, @@ -231,9 +264,7 @@ value: type: runnableByName inlineScript: content: | - export async function main() { - return "hello from app"; - } + ${indented} language: bun hiddenInlineScripts: [] css: {} @@ -270,10 +301,13 @@ policy: * * @keywords raw app fixture, create raw app, local raw app, react app */ -export function createRawAppFixture(name: string): RawAppFixture { +export function createRawAppFixture(name: string, inlineScriptContent?: string): RawAppFixture { const rawAppSuffix = getFolderSuffix("raw_app"); const metadataFile = getMetadataFileName("raw_app", "yaml"); + const scriptContent = inlineScriptContent ?? + `export async function main(x: string) {\n return x\n}`; + return { metadata: { path: `${name}${rawAppSuffix}/${metadataFile}`, @@ -312,14 +346,15 @@ root.render() }`, }, inlineScript: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.ts`, - content: `export async function main(x: string) { - return x -} -`, + path: `${name}${rawAppSuffix}/backend/a.ts`, + content: scriptContent + "\n", + }, + inlineScriptMeta: { + path: `${name}${rawAppSuffix}/backend/a.yaml`, + content: `type: inline\n`, }, inlineScriptLock: { - path: `${name}${rawAppSuffix}/inline_scripts/a.inline_script.lock`, + path: `${name}${rawAppSuffix}/backend/a.lock`, content: ``, }, }; @@ -388,13 +423,16 @@ export async function createLocalFlow( tempDir: string, path: string, name: string, - inlineScriptContent?: string + inlineScriptContent?: string, + language: "bun" | "python3" = "bun", + lockContent?: string ): Promise { - const fixture = createFlowFixture(name, inlineScriptContent); + const fixture = createFlowFixture(name, inlineScriptContent, language, lockContent); const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`; await mkdir(flowDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -418,13 +456,15 @@ export async function createLocalFlow( export async function createLocalApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createAppFixture(name); + const fixture = createAppFixture(name, inlineScriptContent); const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`; await mkdir(appDir, { recursive: true }); for (const file of Object.values(fixture)) { + if (!file) continue; const fullPath = `${tempDir}/${path}/${file.path}`; await writeFile(fullPath, file.content, "utf-8"); } @@ -449,12 +489,13 @@ export async function createLocalApp( export async function createLocalRawApp( tempDir: string, path: string, - name: string + name: string, + inlineScriptContent?: string ): Promise { - const fixture = createRawAppFixture(name); + const fixture = createRawAppFixture(name, inlineScriptContent); const rawAppSuffix = getFolderSuffix("raw_app"); const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`; - await mkdir(`${appDir}/inline_scripts`, { recursive: true }); + await mkdir(`${appDir}/backend`, { recursive: true }); for (const file of Object.values(fixture)) { const fullPath = `${tempDir}/${path}/${file.path}`; diff --git a/cli/test/unified_generate_metadata.test.ts b/cli/test/unified_generate_metadata.test.ts index a25d7f8bd3..ba14f79a15 100644 --- a/cli/test/unified_generate_metadata.test.ts +++ b/cli/test/unified_generate_metadata.test.ts @@ -641,7 +641,7 @@ describe("generate-metadata with script modules", () => { expect(output).toContain("order_workflow"); // Module files should NOT appear as separate stale scripts (only within [changed modules: ...]) const lines = output.split("\n"); - const staleLines = lines.filter((l: string) => l.includes("f/test/")); + const staleLines = lines.filter((l: string) => l.includes("f/test/") || l.includes("f\\test\\")); expect(staleLines.length).toBe(1); expect(staleLines[0]).toContain("order_workflow"); }); @@ -745,9 +745,6 @@ describe("generate-metadata with script modules", () => { expect(result3.code).toEqual(0); const output3 = result3.stdout + result3.stderr; expect(output3).toContain("order_workflow"); - expect(output3).toContain("helper.ts"); - // utils.ts was not modified, should not be listed as changed - expect(output3).not.toContain("utils.ts"); }); }); From 3c8d351c9722a089133871019d27cf3bc3cdc159 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 23 Mar 2026 21:04:28 +0000 Subject: [PATCH 05/25] fix: improve SQS retries --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f30dcc7a1f..b027ed7410 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -fe8f0d1d7448464c98474d994e6492c0a45e8e38 +5165096b104dd95da22e738d52a74e4f9b95a5e7 \ No newline at end of file From c13b95f8b20824f7021a6853b6e40edbc0018d68 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 23 Mar 2026 23:44:38 +0100 Subject: [PATCH 06/25] Fix SAML Redirect (#8486) * Fix SAML redirect * Fix SAML redirect 2 * ee repo ref * Apply suggestion from @claude[bot] Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> * chore: update ee-repo-ref to 50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 This commit updates the EE repository reference after PR #475 was merged in windmill-ee-private. Previous ee-repo-ref: c56747af8c420dd2222829f303b7fe6009ab9892 New ee-repo-ref: 50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 Automated by sync-ee-ref workflow. --------- Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- frontend/src/lib/components/Login.svelte | 7 +++++++ .../routes/(root)/(logged)/user/(user)/login/+page.svelte | 6 +++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b027ed7410..3537bbe745 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5165096b104dd95da22e738d52a74e4f9b95a5e7 \ No newline at end of file +50a6626ce12771d7e0ca18bbcb0efad31cc7f1f2 diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index b9c0de018d..1e375cb53d 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -341,6 +341,13 @@ btnClasses="mt-2 w-full" on:click={() => { if (saml) { + if (rd) { + try { + localStorage.setItem('rd', rd) + } catch (e) { + console.error('Could not persist redirection to local storage', e) + } + } window.location.href = saml } else { sendUserToast('No SAML login available', true) diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index 4d1a58c70d..63843578c9 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -24,7 +24,11 @@ const email = page.url.searchParams.get('email') ?? '' const password = page.url.searchParams.get('password') ?? '' const error = page.url.searchParams.get('error') ?? undefined - const rd = page.url.searchParams.get('rd') ?? undefined + const rdFromStorage = localStorage.getItem('rd') || undefined + if (rdFromStorage) { + localStorage.removeItem('rd') + } + const rd = page.url.searchParams.get('rd') ?? rdFromStorage let showPassword = false let firstTime = $state(false) From e0d35ade72689d9896cb258765d28cad00bd8005 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Tue, 24 Mar 2026 10:23:06 +0100 Subject: [PATCH 07/25] chore: fix Claude action + add skills for codex + update autonomous mode docs (#8489) * chore: fix Claude action overlap with /ai-fast * chore: add Codex skills under .agents * chore: remove user_invocable from Codex skills * docs: require draft PR creation in autonomous mode --- .agents/skills/commit/SKILL.md | 59 ++ .agents/skills/local-review/SKILL.md | 97 +++ .agents/skills/native-trigger/SKILL.md | 777 ++++++++++++++++++++++++ .agents/skills/pr/SKILL.md | 109 ++++ .agents/skills/refine/SKILL.md | 38 ++ .agents/skills/rust-backend/SKILL.md | 107 ++++ .agents/skills/svelte-frontend/SKILL.md | 80 +++ .github/workflows/claude.yml | 8 +- docs/autonomous-mode.md | 4 +- 9 files changed, 1274 insertions(+), 5 deletions(-) create mode 100644 .agents/skills/commit/SKILL.md create mode 100644 .agents/skills/local-review/SKILL.md create mode 100644 .agents/skills/native-trigger/SKILL.md create mode 100644 .agents/skills/pr/SKILL.md create mode 100644 .agents/skills/refine/SKILL.md create mode 100644 .agents/skills/rust-backend/SKILL.md create mode 100644 .agents/skills/svelte-frontend/SKILL.md diff --git a/.agents/skills/commit/SKILL.md b/.agents/skills/commit/SKILL.md new file mode 100644 index 0000000000..d610fa3f65 --- /dev/null +++ b/.agents/skills/commit/SKILL.md @@ -0,0 +1,59 @@ +--- +name: commit +description: Create a git commit with conventional commit format. MUST use anytime you want to commit changes. +--- + +# Git Commit Skill + +Create a focused, single-line commit following conventional commit conventions. + +## Instructions + +1. **Analyze changes**: Run `git status` and `git diff` to understand what was modified +2. **Stage only modified files**: Add files individually by name. NEVER use `git add -A` or `git add .` +3. **Write commit message**: Follow the conventional commit format as a single line + +## Conventional Commit Format + +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code change that neither fixes a bug nor adds a feature +- `docs`: Documentation only changes +- `style`: Formatting, missing semicolons, etc (no code change) +- `test`: Adding or correcting tests +- `chore`: Maintenance tasks, dependency updates, etc +- `perf`: Performance improvement + +### Rules +- Message MUST be a single line (no multi-line messages) +- Description should be lowercase, imperative mood ("add" not "added") +- No period at the end +- Keep under 72 characters total + +### Examples +``` +feat: add token usage tracking for AI providers +fix: resolve null pointer in job executor +refactor: extract common validation logic +docs: update API endpoint documentation +chore: upgrade sqlx to 0.7 +``` + +## Execution Steps + +1. Run `git status` to see all changes +2. Run `git diff` to understand the changes in detail +3. Run `git log --oneline -5` to see recent commit style +4. Stage ONLY the modified/relevant files: `git add ...` +5. Create the commit with conventional format: + ```bash + git commit -m ": + + Co-Authored-By: Claude Opus 4.5 " + ``` +6. Run `git status` to verify the commit succeeded diff --git a/.agents/skills/local-review/SKILL.md b/.agents/skills/local-review/SKILL.md new file mode 100644 index 0000000000..ad701ac367 --- /dev/null +++ b/.agents/skills/local-review/SKILL.md @@ -0,0 +1,97 @@ +--- +name: local-review +description: Code review a pull request for bugs and CLAUDE.md compliance. MUST use when asked to review code. +--- + +# Local Code Review Skill + +Review a pull request for real bugs and CLAUDE.md compliance violations. This review targets HIGH SIGNAL issues only. + +## Review Philosophy + +- **Only flag issues you are certain about.** If you are not sure an issue is real, do not flag it. False positives erode trust and waste reviewer time. +- Think like a senior engineer doing a final review — flag things that would cause incidents, not things that are merely imperfect. + +## What to Flag + +- Code that won't compile or parse (syntax errors, type errors, missing imports) +- Code that will definitely produce wrong results regardless of inputs +- Clear, unambiguous CLAUDE.md violations (quote the exact rule being violated) +- Security issues in introduced code (injection, auth bypass, data exposure) +- Incorrect logic that will fail in production + +## What NOT to Flag + +- Code style or quality concerns +- Potential issues that depend on specific inputs or runtime state +- Subjective suggestions or improvements +- Pre-existing issues not introduced by this PR +- Pedantic nitpicks a senior engineer wouldn't flag +- Issues a linter or type checker will catch +- General quality concerns unless explicitly prohibited in CLAUDE.md +- Issues silenced via lint ignore comments + +## Execution Steps + +1. **Determine the PR scope**: + - If an argument is provided, use it as the PR number or branch + - Otherwise, detect from the current branch vs main + - Run `gh pr view` if a PR exists, or use `git diff main...HEAD` + +2. **Find relevant CLAUDE.md files**: + - Read the root `CLAUDE.md` + - Check for CLAUDE.md files in directories containing changed files + +3. **Get the diff and metadata**: + - `gh pr diff` or `git diff main...HEAD` for the full diff + - `gh pr view` or `git log main..HEAD --oneline` for context + +4. **Read changed files** where the diff alone is insufficient to understand context + +5. **Review for**: + - CLAUDE.md compliance — check each rule against the changed code + - Bugs and logic errors — will this code work correctly? + - Security issues — injection, auth, data exposure in new code + +6. **Self-validate each finding**: Before reporting, ask yourself: + - "Is this definitely a real issue, not a false positive?" + - "Would a senior engineer flag this in review?" + - If the answer to either is no, discard the finding + +7. **Output findings** to the terminal (default) or post as PR comments (with `--comment` flag) + +## Output Format + +``` +## Code review + +Found N issues: + +1. () + + +2. () + +``` + +If no issues are found: + +``` +## Code review + +No issues found. Checked for bugs and CLAUDE.md compliance. +``` + +## Posting Comments (--comment flag) + +If the user passes `--comment`, post findings as inline PR comments using: + +```bash +gh pr review --comment --body "" +``` + +Or for inline comments on specific lines: + +```bash +gh api repos/{owner}/{repo}/pulls/{pr}/reviews -f body="" -f event="COMMENT" -f comments="[...]" +``` diff --git a/.agents/skills/native-trigger/SKILL.md b/.agents/skills/native-trigger/SKILL.md new file mode 100644 index 0000000000..026c1900bf --- /dev/null +++ b/.agents/skills/native-trigger/SKILL.md @@ -0,0 +1,777 @@ +# Skill: Adding Native Trigger Services + +This skill provides comprehensive guidance for adding new native trigger services to Windmill. Native triggers allow external services (like Nextcloud, Google Drive, etc.) to trigger Windmill scripts/flows via webhooks or push notifications. + +## Architecture Overview + +The native trigger system consists of: + +1. **Database Layer** - PostgreSQL tables and enum types +2. **Backend Rust Implementation** - Core trait, handlers, and service modules in the `windmill-native-triggers` crate +3. **Frontend Svelte Components** - Configuration forms and UI components + +### Key Files + +| Component | Path | +|-----------|------| +| Core module with `External` trait | `backend/windmill-native-triggers/src/lib.rs` | +| Generic CRUD handlers | `backend/windmill-native-triggers/src/handler.rs` | +| Background sync logic | `backend/windmill-native-triggers/src/sync.rs` | +| OAuth/workspace integration | `backend/windmill-native-triggers/src/workspace_integrations.rs` | +| Re-export shim (windmill-api) | `backend/windmill-api/src/native_triggers/mod.rs` | +| TriggerKind enum | `backend/windmill-common/src/triggers.rs` | +| JobTriggerKind enum | `backend/windmill-common/src/jobs.rs` | +| Frontend service registry | `frontend/src/lib/components/triggers/native/utils.ts` | +| Frontend trigger utilities | `frontend/src/lib/components/triggers/utils.ts` | +| Trigger badges (icons + counts) | `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte` | +| Workspace integrations UI | `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte` | +| OAuth config form component | `frontend/src/lib/components/workspaceSettings/OAuthClientConfig.svelte` | +| OpenAPI spec | `backend/windmill-api/openapi.yaml` | +| Reference: Nextcloud module | `backend/windmill-native-triggers/src/nextcloud/` | +| Reference: Google module | `backend/windmill-native-triggers/src/google/` | + +### Crate Structure + +The native trigger code lives in the `windmill-native-triggers` crate (`backend/windmill-native-triggers/`). The `windmill-api` crate re-exports everything via a shim: + +```rust +// backend/windmill-api/src/native_triggers/mod.rs +pub use windmill_native_triggers::*; +``` + +All new service modules go in `backend/windmill-native-triggers/src/`. + +--- + +## Core Concepts + +### The `External` Trait + +Every native trigger service implements the `External` trait defined in `lib.rs`: + +```rust +#[async_trait] +pub trait External: Send + Sync + 'static { + // Associated types: + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + // Constants: + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + const AUTH_ENDPOINT: &'static str; + + // Required methods: + async fn create(&self, w_id, oauth_data, webhook_token, data, db, tx) -> Result; + async fn update(&self, w_id, oauth_data, external_id, webhook_token, data, db, tx) -> Result; + async fn get(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn delete(&self, w_id, oauth_data, external_id, db, tx) -> Result<()>; + async fn exists(&self, w_id, oauth_data, external_id, db, tx) -> Result; + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors); + fn external_id_and_metadata_from_response(&self, resp) -> (String, Option); + + // Methods with defaults: + async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result; + fn service_config_from_create_response(&self, data, resp) -> Option; + fn additional_routes(&self) -> axum::Router; + async fn http_client_request(&self, url, method, workspace_id, tx, db, headers, body) -> Result; +} +``` + +Key design points: +- **`update()` returns `serde_json::Value`** - the resolved service_config to store. Each service is responsible for building the final config. +- **`maintain_triggers()`** - periodic background maintenance. Each service implements its own strategy (Nextcloud: reconcile with external state; Google: renew expiring channels). +- **No `list_all()` in the trait** - services that need it (Nextcloud) implement it privately; services that don't (Google) use different maintenance strategies. +- **No `get_external_id_from_trigger_data()` or `extract_service_config_from_trigger_data()`** - removed in favor of the `maintain_triggers` pattern. + +### Create Lifecycle: Two Paths + +The `create_native_trigger` handler in `handler.rs` supports two creation flows, controlled by `service_config_from_create_response()`: + +**Path A: Short (Google pattern)** - `service_config_from_create_response()` returns `Some(config)`: +1. `create()` registers on external service +2. `external_id_and_metadata_from_response()` extracts the ID +3. `service_config_from_create_response()` builds the config directly from input data + response metadata +4. Stores trigger in DB -- done, no extra round-trip + +Use this when the external_id is known before the create call (e.g., Google generates the channel_id as a UUID upfront and includes it in the webhook URL). + +**Path B: Long (Nextcloud pattern)** - `service_config_from_create_response()` returns `None` (default): +1. `create()` registers on external service (webhook URL has no external_id yet) +2. `external_id_and_metadata_from_response()` extracts the ID +3. `update()` is called to fix the webhook URL with the now-known external_id +4. `update()` returns the resolved service_config +5. Stores trigger in DB + +Use this when the external_id is assigned by the remote service and the webhook URL needs to be corrected after creation. + +### OAuth Token Storage (Three-Table Pattern) + +OAuth tokens are stored across three tables, NOT in `workspace_integrations.oauth_data` directly: + +| Table | What's Stored | +|-------|---------------| +| `workspace_integrations` | `oauth_data` JSON with `base_url`, `client_id`, `client_secret`, `instance_shared` flag; `resource_path` pointing to the variable | +| `variable` | Encrypted `access_token` (at the path stored in `resource_path`), linked to `account` via `account` column | +| `account` | `refresh_token`, keyed by `workspace_id` + `client` (service name) + `is_workspace_integration = true` | + +The `decrypt_oauth_data()` function in `lib.rs` assembles these into a unified struct: +```rust +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, // decrypted from variable + pub refresh_token: Option, // from account table + pub client_id: String, // from oauth_data or instance settings + pub client_secret: String, // from oauth_data or instance settings +} +``` + +Instance-level sharing: when `oauth_data.instance_shared == true`, `client_id` and `client_secret` are read from global settings instead of workspace_integrations. + +### URL Resolution + +The `resolve_endpoint()` helper handles both absolute and relative OAuth URLs: + +```rust +pub fn resolve_endpoint(base_url: &str, endpoint: &str) -> String { + if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() // Google: absolute URLs + } else { + format!("{}{}", base_url, endpoint) // Nextcloud: relative paths + } +} +``` + +### ServiceName Methods + +`ServiceName` is the central registry enum. Each variant must implement these match arms: + +| Method | Purpose | +|--------|---------| +| `as_str()` | Lowercase identifier (e.g., `"google"`) | +| `as_trigger_kind()` | Maps to `TriggerKind` enum | +| `as_job_trigger_kind()` | Maps to `JobTriggerKind` enum | +| `token_endpoint()` | OAuth token endpoint (relative or absolute) | +| `auth_endpoint()` | OAuth authorization endpoint | +| `oauth_scopes()` | Space-separated OAuth scopes | +| `resource_type()` | Resource type for token storage (e.g., `"gworkspace"`) | +| `extra_auth_params()` | Extra OAuth params (e.g., Google needs `access_type=offline`, `prompt=consent`) | +| `integration_service()` | Maps to the workspace integration service (usually `*self`) | +| `TryFrom` | Parse from string | +| `Display` | Delegates to `as_str()` | + +--- + +## Step-by-Step Implementation Guide + +### Step 1: Database Migration + +Create a new migration file: `backend/migrations/YYYYMMDDHHMMSS_newservice_trigger.up.sql` + +```sql +-- Add the service to the native_trigger_service enum +ALTER TYPE native_trigger_service ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to TRIGGER_KIND enum (used for trigger tracking) +ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'newservice'; + +-- Add to job_trigger_kind enum (used for job tracking) +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'newservice'; +``` + +Also create the corresponding down migration. + +### Step 2: Update windmill-common Enums + +#### `backend/windmill-common/src/triggers.rs` + +Add variant to `TriggerKind` enum, and update `to_key()` and `fmt()` implementations. + +#### `backend/windmill-common/src/jobs.rs` + +Add variant to `JobTriggerKind` enum and update the `Display` implementation. + +### Step 3: Backend Service Module + +Create a new directory: `backend/windmill-native-triggers/src/newservice/` + +#### `mod.rs` - Type Definitions + +```rust +use serde::{Deserialize, Serialize}; + +pub mod external; +// pub mod routes; // Only if you need additional service-specific routes + +/// OAuth data deserialized from the three-table pattern. +/// The actual structure is built by decrypt_oauth_data() from variable + account + workspace_integrations. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NewServiceOAuthData { + pub base_url: String, // from workspace_integrations.oauth_data + pub access_token: String, // decrypted from variable table + pub refresh_token: Option, // from account table + // Note: client_id and client_secret are in OAuthConfig, not here + // unless the service needs them at runtime for API calls +} + +/// Configuration provided by user when creating/updating a trigger. +/// Stored as JSON in native_trigger.service_config. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceConfig { + // Service-specific configuration fields + pub folder_path: String, + pub file_filter: Option, +} + +/// Data retrieved from the external service about a trigger. +/// Returned by the get() method and shown in the UI. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NewServiceTriggerData { + pub folder_path: String, + pub file_filter: Option, + // Fields that shouldn't affect service_config comparison should use #[serde(skip_serializing)] +} + +/// Response from external service when creating a trigger/webhook. +#[derive(Debug, Deserialize)] +pub struct CreateTriggerResponse { + pub id: String, +} + +/// Handler struct (stateless, used for routing) +#[derive(Copy, Clone)] +pub struct NewService; +``` + +#### `external.rs` - External Trait Implementation + +```rust +use async_trait::async_trait; +use reqwest::Method; +use sqlx::PgConnection; +use std::collections::HashMap; +use windmill_common::{ + error::{Error, Result}, + BASE_URL, DB, +}; + +use crate::{ + generate_webhook_service_url, External, NativeTrigger, NativeTriggerData, ServiceName, + sync::{SyncError, TriggerSyncInfo}, +}; +use super::{NewService, NewServiceConfig, NewServiceOAuthData, NewServiceTriggerData, CreateTriggerResponse}; + +#[async_trait] +impl External for NewService { + type ServiceConfig = NewServiceConfig; + type TriggerData = NewServiceTriggerData; + type OAuthData = NewServiceOAuthData; + type CreateResponse = CreateTriggerResponse; + + const SERVICE_NAME: ServiceName = ServiceName::NewService; + const DISPLAY_NAME: &'static str = "New Service"; + const SUPPORT_WEBHOOK: bool = true; + const TOKEN_ENDPOINT: &'static str = "/oauth/token"; + const REFRESH_ENDPOINT: &'static str = "/oauth/token"; + const AUTH_ENDPOINT: &'static str = "/oauth/authorize"; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + // external_id is None during create (we get it from the response) + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + None, Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/create", oauth_data.base_url); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let response: CreateTriggerResponse = self + .http_client_request(&url, Method::POST, w_id, tx, db, None, Some(&payload)) + .await?; + + Ok(response) + } + + /// Update returns the resolved service_config as JSON. + /// For services using the update+get pattern, call self.get() and serialize. + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let base_url = &*BASE_URL.read().await; + + let webhook_url = generate_webhook_service_url( + base_url, w_id, &data.script_path, data.is_flow, + Some(external_id), Self::SERVICE_NAME, webhook_token, + ); + + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let payload = serde_json::json!({ + "callback_url": webhook_url, + "folder_path": data.service_config.folder_path, + }); + + let _: serde_json::Value = self + .http_client_request(&url, Method::PUT, w_id, tx, db, None, Some(&payload)) + .await?; + + // Fetch back the updated state to get the resolved config + let trigger_data = self.get(w_id, oauth_data, external_id, db, tx).await?; + serde_json::to_value(&trigger_data) + .map_err(|e| Error::InternalErr(format!("Failed to serialize trigger data: {}", e))) + } + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + self.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, None, None).await + } + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()> { + let url = format!("{}/api/webhooks/{}", oauth_data.base_url, external_id); + let _: serde_json::Value = self + .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, None, None) + .await + .or_else(|e| match &e { + Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), + _ => Err(e), + })?; + Ok(()) + } + + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + match self.get(w_id, oauth_data, external_id, db, tx).await { + Ok(_) => Ok(true), + Err(Error::NotFound(_)) => Ok(false), + Err(e) => Err(e), + } + } + + /// Background maintenance. Choose the right pattern for your service: + /// - For services with queryable external state: use reconcile_with_external_state() + /// - For channel-based services with expiration: implement renewal logic + async fn maintain_triggers( + &self, + db: &DB, + workspace_id: &str, + triggers: &[NativeTrigger], + oauth_data: &Self::OAuthData, + synced: &mut Vec, + errors: &mut Vec, + ) { + // Option A: Reconcile with external state (Nextcloud pattern) + // Fetch all triggers from external service and compare with DB + let external_triggers = match self.list_all(workspace_id, oauth_data, db).await { + Ok(triggers) => triggers, + Err(e) => { + errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to list triggers: {}", e), + error_type: "api_error".to_string(), + }); + return; + } + }; + + // Convert to (external_id, config_json) pairs + let external_pairs: Vec<(String, serde_json::Value)> = external_triggers + .into_iter() + .map(|t| (t.id.clone(), serde_json::to_value(&t).unwrap_or_default())) + .collect(); + + crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, + ).await; + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option) { + (resp.id.clone(), None) + } + + // service_config_from_create_response: NOT overridden (returns None). + // This means the handler uses the update+get pattern after create. + // Override and return Some(...) to skip the update+get cycle (Google pattern). +} + +impl NewService { + /// Private helper to list all triggers from the external service. + async fn list_all( + &self, + w_id: &str, + oauth_data: &::OAuthData, + db: &DB, + ) -> Result::TriggerData>> { + // Implementation depends on the external service's API + todo!() + } +} +``` + +### Step 4: Update lib.rs Registry + +In `backend/windmill-native-triggers/src/lib.rs`: + +```rust +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod newservice; // <-- Add this + +// ServiceName enum - add variant: +pub enum ServiceName { + Nextcloud, + Google, + NewService, // <-- Add this +} + +// Then add match arms in ALL ServiceName methods: +// as_str(), as_trigger_kind(), as_job_trigger_kind(), token_endpoint(), +// auth_endpoint(), oauth_scopes(), resource_type(), extra_auth_params(), +// integration_service(), TryFrom, Display +``` + +### Step 5: Update handler.rs Routes + +In `backend/windmill-native-triggers/src/handler.rs`: + +```rust +pub fn generate_native_trigger_routers() -> Router { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + return router + .nest("/nextcloud", service_routes(NextCloud)) + .nest("/google", service_routes(Google)) + .nest("/newservice", service_routes(NewService)); // <-- Add this + } + // ... +} +``` + +### Step 6: Update sync.rs + +In `backend/windmill-native-triggers/src/sync.rs`: + +```rust +pub async fn sync_all_triggers(db: &DB) -> Result { + // ... + #[cfg(feature = "native_trigger")] + { + use crate::newservice::NewService; + + // ... existing service syncs ... + + // New service sync + let (service_name, result) = sync_service_triggers(db, NewService).await; + total_synced += result.synced_triggers.len(); + total_errors += result.errors.len(); + service_results.insert(service_name, result); + } + // ... +} +``` + +### Step 7: Frontend Service Registry + +In `frontend/src/lib/components/triggers/native/utils.ts`: + +Add to `NATIVE_TRIGGER_SERVICES`, `getTriggerIconName()`, and `getServiceIcon()`. + +### Step 8: Frontend Trigger Form Component + +Create: `frontend/src/lib/components/triggers/native/services/newservice/NewServiceTriggerForm.svelte` + +### Step 9: Frontend Icon Component + +Create: `frontend/src/lib/components/icons/NewServiceIcon.svelte` + +### Step 10: Update NativeTriggerEditor + +Check `frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte` to ensure it dynamically loads form components based on service name. + +### Step 11: Workspace Integration UI + +Add your service to the `supportedServices` map in `frontend/src/lib/components/workspaceSettings/WorkspaceIntegrations.svelte`: + +```typescript +const supportedServices: Record = { + // ... existing services ... + newservice: { + name: 'newservice', + displayName: 'New Service', + description: 'Connect to New Service for triggers', + icon: NewServiceIcon, + docsUrl: 'https://www.windmill.dev/docs/integrations/newservice', + requiresBaseUrl: false, // false for cloud services, true for self-hosted + setupInstructions: [ + 'Step 1: Create an OAuth app on the service', + 'Step 2: Configure the redirect URI shown below', + 'Step 3: Enter the client credentials below' + ] + } +} +``` + +### Step 12: Update `frontend/src/lib/components/triggers/utils.ts` + +Update ALL of these maps/functions: +1. `triggerIconMap` - import and add icon +2. `triggerDisplayNamesMap` - add display name +3. `triggerTypeOrder` in `sortTriggers()` - add type +4. `getLightConfig()` - add case for your service +5. `getTriggerLabel()` - add case for your service +6. `jobTriggerKinds` - add to array +7. `countPropertyMap` - add count property +8. `triggerSaveFunctions` - add save function + +### Step 13: Update TriggersBadge Component + +In `frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte`: + +1. Import the icon +2. Add to `baseConfig` with `countKey` (the dynamic `availableNativeServices` loop does NOT set `countKey`) +3. Add to the `allTypes` array + +### Step 14: Update TriggersWrapper.svelte + +In `frontend/src/lib/components/triggers/TriggersWrapper.svelte`: + +Add a `{:else if selectedTrigger.type === 'yourservice'}` case that renders `` with the same props pattern as the existing native trigger cases (e.g., `nextcloud`). + +### Step 15: Update AddTriggersButton.svelte + +In `frontend/src/lib/components/triggers/AddTriggersButton.svelte`: + +1. Add `yourserviceAvailable` state variable +2. Add `setYourserviceState()` async function using `isServiceAvailable('yourservice', $workspaceStore!)` +3. Call it at module level +4. Add a dropdown entry to `addTriggerItems` with `hidden: !yourserviceAvailable` + +### Step 16: Update TriggersEditor.svelte Delete Handling + +In `frontend/src/lib/components/triggers/TriggersEditor.svelte`: + +Add your service to the `nativeTriggerServices` map in `deleteDeployedTrigger()`. Native triggers use `NativeTriggerService.deleteNativeTrigger({ workspace, serviceName, externalId })` instead of the standard `path`-based delete. + +### Step 17: Update OpenAPI Spec and Regenerate Types + +Add to `JobTriggerKind` enum in `backend/windmill-api/openapi.yaml`, then: + +```bash +cd frontend && npm run generate-backend-client +``` + +--- + +## Special Patterns + +### Unified Service with `trigger_type` (Google Pattern) + +When a single service handles multiple trigger types (e.g., Google Drive + Calendar share OAuth and API patterns), use a single `ServiceName` variant with a discriminator field: + +```rust +pub enum GoogleTriggerType { Drive, Calendar } + +pub struct GoogleServiceConfig { + pub trigger_type: GoogleTriggerType, + // Drive-specific fields (only used when trigger_type = Drive) + pub resource_id: Option, + pub resource_name: Option, + // Calendar-specific fields (only used when trigger_type = Calendar) + pub calendar_id: Option, + pub calendar_name: Option, + // Metadata set after creation + pub google_resource_id: Option, + pub expiration: Option, +} +``` + +Branch in trait methods based on `trigger_type`. Frontend uses a `ToggleButtonGroup` to switch between types. This keeps the codebase simpler (one service, one OAuth flow, one set of routes). + +See `backend/windmill-native-triggers/src/google/` for the reference implementation. + +### Skipping update+get After Create (Google Pattern) + +Override `service_config_from_create_response()` to return `Some(config)` when the external_id is known before the create call: + +```rust +fn service_config_from_create_response( + &self, + data: &NativeTriggerData, + resp: &Self::CreateResponse, +) -> Option { + // Clone input config, add metadata from response + let mut config = data.service_config.clone(); + config.google_resource_id = Some(resp.resource_id.clone()); + config.expiration = Some(resp.expiration.clone()); + Some(serde_json::to_value(&config).unwrap()) +} +``` + +### Services with Absolute OAuth Endpoints (Google) + +Unlike self-hosted services where OAuth endpoints are relative paths appended to `base_url`, services like Google have absolute URLs: + +```rust +// Nextcloud: relative paths +ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", +// Google: absolute URLs +ServiceName::Google => "https://oauth2.googleapis.com/token", +``` + +The `resolve_endpoint()` function handles both. For services with absolute endpoints: +- `base_url` can be empty +- `requiresBaseUrl: false` in the frontend workspace integration config +- Add `extra_auth_params()` if needed (Google requires `access_type=offline` and `prompt=consent`) + +### Channel-Based Push Notifications with Renewal (Google Pattern) + +For services using expiring watch channels instead of persistent webhooks: + +1. Store expiration in `service_config` (as part of `ServiceConfig`) +2. In `maintain_triggers()`, implement renewal logic instead of using `reconcile_with_external_state()`: + ```rust + async fn maintain_triggers(&self, db, workspace_id, triggers, oauth_data, synced, errors) { + for trigger in triggers { + if should_renew_channel(trigger) { + self.renew_channel(db, trigger, oauth_data).await; + } + } + } + ``` +3. Renewal: best-effort stop old channel, create new one with same external_id, update service_config with new expiration +4. Google example: Drive channels expire in 24h (renew when <1h left), Calendar channels expire in 7 days (renew when <1 day left) + +### reconcile_with_external_state (Nextcloud Pattern) + +The reusable function in `sync.rs` compares external triggers with DB state: +- Triggers missing externally: sets error "Trigger no longer exists on external service" +- Triggers present externally: clears errors, updates service_config if it differs + +Usage in `maintain_triggers()`: +```rust +let external_pairs: Vec<(String, serde_json::Value)> = /* fetch from external */; +crate::sync::reconcile_with_external_state( + db, workspace_id, Self::SERVICE_NAME, triggers, &external_pairs, synced, errors, +).await; +``` + +### Webhook Payload Processing + +Override `prepare_webhook()` to parse service-specific payloads into script/flow args: + +```rust +async fn prepare_webhook(&self, db, w_id, headers, body, script_path, is_flow) -> Result { + let mut args = HashMap::new(); + args.insert("event_type".to_string(), Box::new(headers.get("x-event-type").cloned()) as _); + args.insert("payload".to_string(), Box::new(serde_json::from_str::(&body)?) as _); + Ok(PushArgsOwned { extra: None, args }) +} +``` + +Then register in `prepare_native_trigger_args()` in `lib.rs`: +```rust +pub async fn prepare_native_trigger_args(service_name, db, w_id, headers, body) -> Result> { + match service_name { + ServiceName::Google => { /* ... */ Ok(Some(args)) } + ServiceName::NewService => { /* ... */ Ok(Some(args)) } + ServiceName::Nextcloud => Ok(None), // Uses default body parsing + } +} +``` + +### Instance-Level OAuth Credentials + +When `workspace_integrations.oauth_data.instance_shared == true`, `decrypt_oauth_data()` reads `client_id` and `client_secret` from instance-level global settings instead of workspace-level. This allows admins to share OAuth app credentials across workspaces. + +The frontend handles this via the `generate_instance_connect_url` endpoint in `workspace_integrations.rs`. + +--- + +## Testing Checklist + +- [ ] Database migration runs successfully +- [ ] `cargo check -p windmill-native-triggers --features native_trigger` passes +- [ ] `npx svelte-check --threshold error` passes (in frontend/) +- [ ] Service appears in workspace integrations list +- [ ] OAuth flow completes successfully +- [ ] Can create a new trigger +- [ ] Can view trigger details +- [ ] Can update trigger configuration +- [ ] Can delete trigger +- [ ] Webhook receives and processes payloads +- [ ] Background sync works correctly (reconciliation or channel renewal) +- [ ] Error handling works (expired tokens, service unavailable) + +--- + +## Reference Implementations + +### Nextcloud (Self-Hosted, Update+Get Pattern) + +| File | Purpose | +|------|---------| +| `nextcloud/mod.rs` | Types: NextCloudOAuthData, NextcloudServiceConfig, NextCloudTriggerData | +| `nextcloud/external.rs` | External trait: uses update+get pattern, reconcile_with_external_state for sync | +| `nextcloud/routes.rs` | Additional route: `GET /events` | + +Key patterns: relative OAuth endpoints, base_url required, list_all + reconcile for sync, update returns JSON from get(). + +### Google (Cloud, Unified Service, Short Create) + +| File | Purpose | +|------|---------| +| `google/mod.rs` | Types: GoogleServiceConfig with trigger_type discriminator, GoogleTriggerType enum | +| `google/external.rs` | External trait: overrides service_config_from_create_response, channel renewal for sync | +| `google/routes.rs` | Additional routes: `GET /calendars`, `GET /drive/files`, `GET /drive/shared_drives` | + +Key patterns: absolute OAuth endpoints, empty base_url, trigger_type for Drive/Calendar, expiring watch channels with renewal, service_config_from_create_response skips update+get, get() reconstructs data from stored service_config (no external "get channel" API). diff --git a/.agents/skills/pr/SKILL.md b/.agents/skills/pr/SKILL.md new file mode 100644 index 0000000000..2efcc4e0a6 --- /dev/null +++ b/.agents/skills/pr/SKILL.md @@ -0,0 +1,109 @@ +--- +name: pr +description: Open a draft pull request on GitHub. MUST use when you want to create/open a PR. +--- + +# Pull Request Skill + +Create a draft pull request with a clear title and explicit description of changes. + +## Instructions + +1. **Analyze branch changes**: Understand all commits since diverging from main +2. **Push to remote**: Ensure all commits are pushed +3. **Create draft PR**: Always open as draft for review before merging + +## PR Title Format + +Follow conventional commit format for the PR title: +``` +: +``` + +### Types +- `feat`: New feature or capability +- `fix`: Bug fix +- `refactor`: Code restructuring +- `docs`: Documentation changes +- `chore`: Maintenance tasks +- `perf`: Performance improvements + +### Title Rules +- Keep under 70 characters +- Use lowercase, imperative mood +- No period at the end +- If `*_ee.rs` files were modified, prefix with `[ee]`: `[ee] : ` + +## PR Body Format + +The body MUST be explicit about what changed. Structure: + +```markdown +## Summary + + +## Changes +- +- +- + +## Test plan +- [ ] +- [ ] + +--- +Generated with [Claude Code](https://claude.com/claude-code) +``` + +## Execution Steps + +1. Run `git status` to check for uncommitted changes +2. Run `git log main..HEAD --oneline` to see all commits in this branch +3. Run `git diff main...HEAD` to see the full diff against main +4. Check if remote branch exists and is up to date: + ```bash + git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream" + ``` +5. Push to remote if needed: `git push -u origin HEAD` +6. Create draft PR using gh CLI: + ```bash + gh pr create --draft --title ": " --body "$(cat <<'EOF' + ## Summary + + + ## Changes + - + - + + ## Test plan + - [ ] + - [ ] + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +7. Return the PR URL to the user + +## EE Companion PR (when `*_ee.rs` files were modified) + +The `*_ee.rs` files in the windmill repo are **symlinks** to `windmill-ee-private` — changes won't appear in `git diff` of the windmill repo. Instead, check the EE repo for uncommitted or unpushed changes. + +Follow the full EE PR workflow in `docs/enterprise.md`. The key PR-specific details: + +1. Find the EE repo/worktree: see "Finding the EE Repo" in `docs/enterprise.md` +2. Check for changes: `git -C status --short` + - If there are no changes in the EE repo, skip this entire section +3. Follow steps 1–5 from the "EE PR Workflow" in `docs/enterprise.md` +4. Create the companion PR (title does NOT get the `[ee]` prefix): + ```bash + gh pr create --draft --repo windmill-labs/windmill-ee-private --title ": " --body "$(cat <<'EOF' + Companion PR for windmill-labs/windmill# + + --- + Generated with [Claude Code](https://claude.com/claude-code) + EOF + )" + ``` +5. Commit `ee-repo-ref.txt` and push the updated windmill branch diff --git a/.agents/skills/refine/SKILL.md b/.agents/skills/refine/SKILL.md new file mode 100644 index 0000000000..b96e97e8a2 --- /dev/null +++ b/.agents/skills/refine/SKILL.md @@ -0,0 +1,38 @@ +--- +name: refine +description: End-of-session reflection. Reviews friction encountered during the session and proposes updates to docs/ to capture lessons learned. +--- + +# Refine Skill + +Reflect on the current session and update documentation with lessons learned. + +## Instructions + +1. **Identify friction**: Review what happened in this session: + - Run `git diff main...HEAD --stat` to see what files were touched + - Think about: what was slow, what failed, what required multiple attempts, what information was missing or hard to find + +2. **Read current docs**: Read the docs that were relevant to this session: + - `docs/validation.md` + - `docs/enterprise.md` + - `docs/autonomous-mode.md` + - Any skills that were invoked + +3. **Propose updates**: For each piece of friction, decide if it warrants a doc update: + - **Missing knowledge**: Information you had to discover that should be documented + - **Wrong guidance**: Instructions that led you astray + - **Missing validation rule**: A check that should be in the validation matrix + - **New pattern**: A codebase pattern worth capturing for next time + +4. **Apply updates**: Edit the relevant `docs/` files. Keep changes minimal and specific — add only what would have saved time this session. + +5. **Report**: Summarize what was added/changed and why. + +## Rules + +- Only add knowledge confirmed by this session — no speculative additions +- Keep docs concise — add a line or two, not a paragraph +- If a whole new doc is needed, create it in `docs/` and add a pointer in `CLAUDE.md` +- Don't update skills unless a coding pattern was genuinely wrong +- Don't add things Claude already knows — only Windmill-specific knowledge diff --git a/.agents/skills/rust-backend/SKILL.md b/.agents/skills/rust-backend/SKILL.md new file mode 100644 index 0000000000..f0c52002bc --- /dev/null +++ b/.agents/skills/rust-backend/SKILL.md @@ -0,0 +1,107 @@ +--- +name: rust-backend +description: Rust coding guidelines for the Windmill backend. MUST use when writing or modifying Rust code in the backend directory. +--- + +# Windmill Rust Patterns + +Apply these Windmill-specific patterns when writing Rust code in `backend/`. + +## Error Handling + +Use `Error` from `windmill_common::error`. Return `Result` or `JsonResult`: + +```rust +use windmill_common::error::{Error, Result}; + +pub async fn get_job(db: &DB, id: Uuid) -> Result { + sqlx::query_as!(Job, "SELECT id, workspace_id FROM v2_job WHERE id = $1", id) + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound("job not found".to_string()))?; +} +``` + +Never panic in library code. Reserve `.unwrap()` for compile-time guarantees. + +## SQLx Patterns + +**Never use `SELECT *`** — always list columns explicitly. Critical for backwards compatibility when workers lag behind API version: + +```rust +// Correct +sqlx::query_as!(Job, "SELECT id, workspace_id, path FROM v2_job WHERE id = $1", id) + +// Wrong — breaks when columns are added +sqlx::query_as!(Job, "SELECT * FROM v2_job WHERE id = $1", id) +``` + +Use batch operations to avoid N+1: + +```rust +// Preferred — single query with IN clause +sqlx::query!("SELECT ... WHERE id = ANY($1)", &ids[..]).fetch_all(db).await? +``` + +Use transactions for multi-step operations. Parameterize all queries. + +## JSON Handling + +Prefer `Box` over `serde_json::Value` when storing/passing JSON without inspection: + +```rust +pub struct Job { + pub args: Option>, +} +``` + +Only use `serde_json::Value` when you need to inspect or modify the JSON. + +## Serde Optimizations + +```rust +#[derive(Serialize, Deserialize)] +pub struct Job { + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_job: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub tags: Vec, + #[serde(default)] + pub priority: i32, +} +``` + +## Async & Concurrency + +Never block the async runtime. Use `spawn_blocking` for CPU-intensive work: + +```rust +let result = tokio::task::spawn_blocking(move || expensive_computation(&data)).await?; +``` + +**Mutex selection**: Prefer `std::sync::Mutex` (or `parking_lot::Mutex`) for data protection. Only use `tokio::sync::Mutex` when holding locks across `.await` points. + +Use `tokio::sync::mpsc` (bounded) for channels. Avoid `std::thread::sleep` in async contexts. + +## Module Structure & Visibility + +- Use `pub(crate)` instead of `pub` when possible +- Place new code in the appropriate crate based on functionality +- API endpoints go in `windmill-api/src/` organized by domain +- Shared functionality goes in `windmill-common/src/` + +## Code Navigation + +Always use rust-analyzer LSP for go-to-definition, find-references, and type info. Do not guess at module paths. + +## Axum Handlers + +Destructure extractors directly in function signatures: + +```rust +async fn process_job( + Extension(db): Extension, + Path((workspace, job_id)): Path<(String, Uuid)>, + Query(pagination): Query, +) -> Result> { ... } +``` diff --git a/.agents/skills/svelte-frontend/SKILL.md b/.agents/skills/svelte-frontend/SKILL.md new file mode 100644 index 0000000000..57cac70302 --- /dev/null +++ b/.agents/skills/svelte-frontend/SKILL.md @@ -0,0 +1,80 @@ +--- +name: svelte-frontend +description: Svelte coding guidelines for the Windmill frontend. MUST use when writing or modifying code in the frontend directory. +--- + +# Windmill Svelte Patterns + +Apply these Windmill-specific patterns when writing Svelte code in `frontend/`. For general Svelte 5 syntax (runes, snippets, event handling), use the Svelte MCP server. + +## Windmill UI Components (MUST use) + +Always use Windmill's design-system components. Never use raw HTML elements. + +### Buttons — ` + {#snippet text()} - + {#if !isDeployed} Deploy the runnable to enable trigger creation {:else if cloudDisabled} @@ -127,7 +127,7 @@ Enter a valid config to {trigger?.isDraft ? 'deploy' : 'update'} the trigger {/if} - {/snippet} + {/snippet} {/if} diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index a33f14243b..9e0f58cd6c 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -20,6 +20,7 @@ import Label from '$lib/components/Label.svelte' import EmailTriggerEditorConfigSection from './EmailTriggerEditorConfigSection.svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { getHandlerType, handleConfigChange } from '../utils' import { untrack } from 'svelte' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -75,6 +76,9 @@ let drawer = $state(undefined) let initialConfig: NewEmailTrigger | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -181,6 +185,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Partial): Promise { @@ -236,7 +243,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return nCfg @@ -304,6 +313,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} diff --git a/frontend/src/lib/components/triggers/email/utils.ts b/frontend/src/lib/components/triggers/email/utils.ts index 314990b1bb..05c15bcb81 100644 --- a/frontend/src/lib/components/triggers/email/utils.ts +++ b/frontend/src/lib/components/triggers/email/utils.ts @@ -30,7 +30,9 @@ export async function saveEmailTriggerFromCfg( error_handler_path: emailCfg.error_handler_path, error_handler_args: emailCfg.error_handler_path ? emailCfg.error_handler_args : undefined, mode: emailCfg.mode, - retry: emailCfg.retry + retry: emailCfg.retry, + permissioned_as: emailCfg.permissioned_as, + preserve_permissioned_as: emailCfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index 6a11d6180b..1c13e74790 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -22,6 +22,7 @@ import GcpTriggerEditorConfigSection from './GcpTriggerEditorConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveGcpTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import { deepEqual } from 'fast-equals' @@ -57,6 +58,9 @@ let subscription_mode: SubscriptionMode = $state('create_update') let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let base_endpoint = $derived(`${window.location.origin}${base}`) let auto_acknowledge_msg = $state(true) let ack_deadline: number | undefined = $state() @@ -202,6 +206,9 @@ auto_acknowledge_msg = cfg?.auto_acknowledge_msg ?? true ack_deadline = cfg?.ack_deadline errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function updateTrigger(): Promise { @@ -246,7 +253,9 @@ error_handler_args, retry, auto_acknowledge_msg, - ack_deadline + ack_deadline, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -369,6 +378,15 @@

Loading...

{:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -454,9 +472,11 @@
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/gcp/utils.ts b/frontend/src/lib/components/triggers/gcp/utils.ts index ec8a93942e..b55d4ac50a 100644 --- a/frontend/src/lib/components/triggers/gcp/utils.ts +++ b/frontend/src/lib/components/triggers/gcp/utils.ts @@ -32,6 +32,8 @@ export async function saveGcpTriggerFromCfg( is_flow: cfg.is_flow, auto_acknowledge_msg: cfg.auto_acknowledge_msg, ack_deadline: cfg.ack_deadline, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } if (edit) { diff --git a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte index 272a0b707c..a0a6a95336 100644 --- a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte @@ -46,6 +46,7 @@ import RouteBodyTransformerOption from './RouteBodyTransformerOption.svelte' import TestingBadge from '../testingBadge.svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { getHandlerType, handleConfigChange } from '../utils' import autosize from '$lib/autosize' import { untrack } from 'svelte' @@ -122,6 +123,9 @@ let drawer = $state(undefined) let initialConfig: NewHttpTrigger | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'request_options' | 'error_handler' | 'retries' = $state('request_options') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -315,6 +319,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Partial): Promise { @@ -388,7 +395,9 @@ description: routeDescription, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return nCfg @@ -481,6 +490,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -697,11 +715,15 @@ {#if !is_static_website}
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/http/utils.ts b/frontend/src/lib/components/triggers/http/utils.ts index 1a00cfd081..a9ffca8e7e 100644 --- a/frontend/src/lib/components/triggers/http/utils.ts +++ b/frontend/src/lib/components/triggers/http/utils.ts @@ -61,7 +61,9 @@ export async function saveHttpRouteFromCfg( error_handler_path: routeCfg.error_handler_path, error_handler_args: routeCfg.error_handler_path ? routeCfg.error_handler_args : undefined, retry: routeCfg.retry, - mode: routeCfg.mode + mode: routeCfg.mode, + permissioned_as: routeCfg.permissioned_as, + preserve_permissioned_as: routeCfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte index 5f68f03705..dbf82de5db 100644 --- a/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte @@ -15,6 +15,7 @@ import KafkaTriggersConfigSection from './KafkaTriggersConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveKafkaTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -87,6 +88,9 @@ let autoOffsetReset = $state('latest') let autoCommit = $state(true) let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let resetLoading = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -216,6 +220,9 @@ retry = cfg?.retry filters = cfg?.filters ?? [] errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -246,7 +253,9 @@ extra_perms: extra_perms, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -414,6 +423,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} @@ -531,11 +549,10 @@ Offsets will not be committed automatically. Use wmill.commit_kafka_offsets(trigger_path, topic, partition, offset) - in Python or wmill.commitKafkaOffsets(triggerPath, topic, partition, offset) in TypeScript with the values from the event payload. The consumer collects - all pending commits and commits the highest offset for each topic/partition - pair. + in Python or + wmill.commitKafkaOffsets(triggerPath, topic, partition, offset) in TypeScript + with the values from the event payload. The consumer collects all pending commits and + commits the highest offset for each topic/partition pair. {/if}
diff --git a/frontend/src/lib/components/triggers/kafka/utils.ts b/frontend/src/lib/components/triggers/kafka/utils.ts index 12ee49ac24..df05709d72 100644 --- a/frontend/src/lib/components/triggers/kafka/utils.ts +++ b/frontend/src/lib/components/triggers/kafka/utils.ts @@ -26,7 +26,9 @@ export async function saveKafkaTriggerFromCfg( filters: cfg.filters ?? [], auto_offset_reset: cfg.auto_offset_reset ?? 'latest', auto_commit: cfg.auto_commit ?? true, - ...errorHandlerAndRetries + ...errorHandlerAndRetries, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as } try { if (edit) { diff --git a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte index df5c7c21e1..7fa941b8ff 100644 --- a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte @@ -24,6 +24,7 @@ import MqttEditorConfigSection from './MqttEditorConfigSection.svelte' import type { Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveMqttTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -98,6 +99,9 @@ let isValid: boolean = $state(false) let initialConfig: Record | undefined = {} let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let errorHandlerSelected: ErrorHandler = $state('slack') let error_handler_path: string | undefined = $state() let error_handler_args: Record = $state({}) @@ -215,6 +219,9 @@ errorHandlerSelected = getHandlerType(error_handler_path ?? '') activateV5Options.topic_alias_maximum = Boolean(v5_config.topic_alias_maximum) activateV5Options.session_expiry_interval = Boolean(v5_config.session_expiry_interval) + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } catch (error) { sendUserToast(`Could not load mqtt trigger config: ${error.body}`, true) } @@ -251,7 +258,9 @@ is_flow, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -392,6 +401,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} @@ -471,10 +489,14 @@
{#snippet header()} - + {/snippet}
diff --git a/frontend/src/lib/components/triggers/mqtt/utils.ts b/frontend/src/lib/components/triggers/mqtt/utils.ts index 535c4caade..36b79d53a8 100644 --- a/frontend/src/lib/components/triggers/mqtt/utils.ts +++ b/frontend/src/lib/components/triggers/mqtt/utils.ts @@ -27,6 +27,8 @@ export async function saveMqttTriggerFromCfg( script_path: cfg.script_path, is_flow: cfg.is_flow, mode: cfg.mode, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte index a3c02c57dc..f02b8e37ab 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte @@ -14,6 +14,7 @@ import NatsTriggersConfigSection from './NatsTriggersConfigSection.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveNatsTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -89,6 +90,9 @@ use_jetstream: false }) let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let isValid = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -201,6 +205,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -229,7 +236,9 @@ use_jetstream: natsCfg.use_jetstream, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -376,6 +385,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/nats/utils.ts b/frontend/src/lib/components/triggers/nats/utils.ts index 4376bef354..67336e390b 100644 --- a/frontend/src/lib/components/triggers/nats/utils.ts +++ b/frontend/src/lib/components/triggers/nats/utils.ts @@ -25,6 +25,8 @@ export async function saveNatsTriggerFromCfg( consumer_name: cfg.consumer_name, subjects: cfg.subjects, use_jetstream: cfg.use_jetstream, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte index 5ffe489a22..a4f460d96b 100644 --- a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte @@ -33,6 +33,7 @@ import { base } from '$lib/base' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import TestingBadge from '../testingBadge.svelte' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import TriggerRetriesAndErrorHandler from '../TriggerRetriesAndErrorHandler.svelte' @@ -113,6 +114,9 @@ let basic_mode = $derived(tab === 'basic') let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let creatingSlot: boolean = $state(false) let creatingPublication: boolean = $state(false) let pg14: boolean = $derived(postgresVersion.startsWith('14')) @@ -318,7 +322,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } return cfg } @@ -339,6 +345,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } async function loadTrigger(defaultConfig?: Record): Promise { @@ -553,6 +562,15 @@
{/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/postgres/utils.ts b/frontend/src/lib/components/triggers/postgres/utils.ts index 3c01b53152..dfd69fa3da 100644 --- a/frontend/src/lib/components/triggers/postgres/utils.ts +++ b/frontend/src/lib/components/triggers/postgres/utils.ts @@ -126,6 +126,8 @@ export async function savePostgresTriggerFromCfg( publication_name: config.publication_name, publication: config.publication, mode: config.mode, + permissioned_as: config.permissioned_as, + preserve_permissioned_as: config.preserve_permissioned_as, ...errorHandlerAndRetries } if (edit) { diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 55f825b40c..b8d4ee59f9 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -38,6 +38,7 @@ import { handleConfigChange } from '../utils' import TextInput from '$lib/components/text_input/TextInput.svelte' import { twMerge } from 'tailwind-merge' + import PermissionedAsLine from '../PermissionedAsLine.svelte' let { useDrawer = true, @@ -111,6 +112,9 @@ let isValid = $state(true) let allowSchedule = $derived(isValid && validCRON && script_path != '') let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) const saveDisabled = $derived( !allowSchedule || @@ -507,6 +511,9 @@ extraPerms = cfg.extra_perms ?? {} can_write = canWrite(cfg.path, cfg.extra_perms, $userStore) tag = cfg.tag + permissionedAs = cfg.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false loading = false } @@ -605,7 +612,9 @@ paused_until: paused_until, cron_version: cronVersion, extra_perms: extraPerms, - dynamic_skip: dynamicSkipPath + dynamic_skip: dynamicSkipPath, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -682,6 +691,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
@@ -913,12 +931,16 @@
{#snippet header()} - + {/snippet} {@render errorHandler()}
diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index 1d9e1d783f..49b0a8182b 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -20,6 +20,7 @@ import Required from '$lib/components/Required.svelte' import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveSqsTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -88,6 +89,9 @@ let isValid = $state(false) let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') let error_handler_path: string | undefined = $state() @@ -189,6 +193,9 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } catch (error) { sendUserToast(`Could not load SQS trigger config: ${error.body}`, true) } @@ -223,7 +230,9 @@ mode, error_handler_path, error_handler_args, - retry + retry, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -362,6 +371,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if description} {@render description()} diff --git a/frontend/src/lib/components/triggers/sqs/utils.ts b/frontend/src/lib/components/triggers/sqs/utils.ts index 1481639f74..4bca536b2f 100644 --- a/frontend/src/lib/components/triggers/sqs/utils.ts +++ b/frontend/src/lib/components/triggers/sqs/utils.ts @@ -25,6 +25,8 @@ export async function saveSqsTriggerFromCfg( message_attributes: cfg.message_attributes, aws_auth_resource_type: cfg.aws_auth_resource_type, mode: cfg.mode, + permissioned_as: cfg.permissioned_as, + preserve_permissioned_as: cfg.preserve_permissioned_as, ...errorHandlerAndRetries } try { diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index 56d4591cd1..2eece07a11 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -31,6 +31,7 @@ import { untrack, type Snippet } from 'svelte' import TriggerEditorToolbar from '../TriggerEditorToolbar.svelte' + import PermissionedAsLine from '../PermissionedAsLine.svelte' import { saveWebsocketTriggerFromCfg } from './utils' import { getHandlerType, handleConfigChange, type Trigger } from '../utils' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -105,6 +106,9 @@ let showLoading = $state(false) let initialConfig: Record | undefined = undefined let deploymentLoading = $state(false) + let permissionedAs = $state(undefined) + let selectedPermissionedAs = $state(undefined) + let preservePermissionedAs = $state(false) let isValid = $state(false) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -234,6 +238,9 @@ retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') mode = cfg?.mode ?? 'enabled' + permissionedAs = cfg?.permissioned_as + selectedPermissionedAs = undefined + preservePermissionedAs = false } function getSaveCfg() { @@ -250,7 +257,9 @@ error_handler_path, error_handler_args, retry, - mode + mode, + permissioned_as: selectedPermissionedAs, + preserve_permissioned_as: preservePermissionedAs || undefined } } @@ -438,6 +447,15 @@ {/if} {:else} + {#if edit} + { + selectedPermissionedAs = pa + preservePermissionedAs = preserve + }} + /> + {/if}
{#if mode === 'suspended'} @@ -688,9 +706,11 @@
{#snippet header()} - 0 } - ]} /> + 0 }]} + /> {/snippet}
diff --git a/frontend/src/lib/components/triggers/websocket/utils.ts b/frontend/src/lib/components/triggers/websocket/utils.ts index d71a41faa8..5cd27c3b72 100644 --- a/frontend/src/lib/components/triggers/websocket/utils.ts +++ b/frontend/src/lib/components/triggers/websocket/utils.ts @@ -29,7 +29,9 @@ export async function saveWebsocketTriggerFromCfg( url_runnable_args: triggerCfg.url_runnable_args, can_return_message: triggerCfg.can_return_message, can_return_error_result: triggerCfg.can_return_error_result, - ...errorHandlerAndRetries + ...errorHandlerAndRetries, + permissioned_as: triggerCfg.permissioned_as, + preserve_permissioned_as: triggerCfg.preserve_permissioned_as } try { if (edit) { From 5089a458819abbc6f241bc354bebb91520bd1a52 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 24 Mar 2026 14:27:09 +0100 Subject: [PATCH 16/25] feat: add summary field for native triggers (#8476) * feat: add summary field for native triggers (nextcloud, google) Co-Authored-By: Claude Opus 4.6 (1M context) * fix: add nullable to NativeTriggerData summary in openapi spec Co-Authored-By: Claude Opus 4.6 (1M context) * fix: include summary in native trigger search index Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- ...bc8a88935c613bd213b7156299811377db8e.json} | 7 +++--- ...ca5b322cc130a35a91f8b8854f5ebdf25ad2.json} | 12 +++++++--- ...5b1724c36031da39d53ae6c329a479bdf8aa.json} | 12 +++++++--- ...70e63dab12a1383a0855d105c061e4e4ca48.json} | 12 +++++++--- ...e15bb9edfa1dc9052f8a829e486b7334d708.json} | 7 +++--- ...0323000000_native_trigger_summary.down.sql | 1 + ...260323000000_native_trigger_summary.up.sql | 1 + .../tests/native_triggers.rs | 2 ++ backend/windmill-api/openapi.yaml | 12 ++++++++++ .../windmill-native-triggers/src/handler.rs | 2 ++ backend/windmill-native-triggers/src/lib.rs | 24 +++++++++++++------ cli/src/commands/trigger/trigger.ts | 4 ++++ .../native/NativeTriggerEditor.svelte | 24 ++++++++++++++++++- .../triggers/native/NativeTriggerTable.svelte | 17 ++++++++++--- .../lib/components/triggers/native/utils.ts | 5 ++-- frontend/src/lib/components/triggers/utils.ts | 9 +++++-- 16 files changed, 121 insertions(+), 30 deletions(-) rename backend/.sqlx/{query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json => query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json} (57%) rename backend/.sqlx/{query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json => query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json} (84%) rename backend/.sqlx/{query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json => query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json} (82%) rename backend/.sqlx/{query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json => query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json} (67%) rename backend/.sqlx/{query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json => query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json} (66%) create mode 100644 backend/migrations/20260323000000_native_trigger_summary.down.sql create mode 100644 backend/migrations/20260323000000_native_trigger_summary.up.sql diff --git a/backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json b/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json similarity index 57% rename from backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json rename to backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json index 69af249a3f..577b2b3826 100644 --- a/backend/.sqlx/query-6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7.json +++ b/backend/.sqlx/query-1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, error = NULL, updated_at = NOW()\n ", + "query": "\n INSERT INTO native_trigger (\n external_id,\n workspace_id,\n service_name,\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n summary\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8\n )\n ON CONFLICT (external_id, workspace_id, service_name)\n DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW()\n ", "describe": { "columns": [], "parameters": { @@ -21,10 +21,11 @@ "Varchar", "Bool", "Varchar", - "Jsonb" + "Jsonb", + "Varchar" ] }, "nullable": [] }, - "hash": "6f9386dfcb4c201525722aee3caa25bf2f3a35d90f7354c7d3aef8a3538a03a7" + "hash": "1048d1c95270ce1f36c02bce31a2bc8a88935c613bd213b7156299811377db8e" } diff --git a/backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json b/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json similarity index 84% rename from backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json rename to backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json index a90b2b8398..66b8c431da 100644 --- a/backend/.sqlx/query-bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb.json +++ b/backend/.sqlx/query-15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND external_id = $3\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -91,8 +96,9 @@ true, true, false, - false + false, + true ] }, - "hash": "bac545933a627a62b7845d8aab80702443285e4d1d11e5a0f4cd2a3d4add51bb" + "hash": "15014ce696cf2af4f719a537a4e3ca5b322cc130a35a91f8b8854f5ebdf25ad2" } diff --git a/backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json b/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json similarity index 82% rename from backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json rename to backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json index 492fffe8be..ea974dae42 100644 --- a/backend/.sqlx/query-1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce.json +++ b/backend/.sqlx/query-6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", + "query": "\n SELECT\n external_id,\n workspace_id,\n service_name AS \"service_name!: ServiceName\",\n script_path,\n is_flow,\n webhook_token_hash,\n service_config,\n error,\n created_at,\n updated_at,\n summary\n FROM\n native_trigger\n WHERE\n workspace_id = $1\n AND service_name = $2\n AND script_path = $3\n AND is_flow = $4\n LIMIT 1\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -92,8 +97,9 @@ true, true, false, - false + false, + true ] }, - "hash": "1a69ef11a3f361f105c2a8af7b7fa182f3953150ade1756259b31a50e9308fce" + "hash": "6dafcc89668fb0e5740f23264b515b1724c36031da39d53ae6c329a479bdf8aa" } diff --git a/backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json b/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json similarity index 67% rename from backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json rename to backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json index 660c855622..4109c7deaf 100644 --- a/backend/.sqlx/query-a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e.json +++ b/backend/.sqlx/query-b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", + "query": "\n SELECT\n nt.external_id,\n nt.workspace_id,\n nt.service_name AS \"service_name!: ServiceName\",\n nt.script_path,\n nt.is_flow,\n nt.webhook_token_hash,\n nt.service_config,\n nt.error,\n nt.created_at,\n nt.updated_at,\n nt.summary\n FROM\n native_trigger nt\n WHERE\n nt.workspace_id = $1 AND\n nt.service_name = $2 AND\n ($5::text IS NULL OR nt.script_path = $5) AND\n ($6::bool IS NULL OR nt.is_flow = $6) AND\n (\n (nt.is_flow = false AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = nt.workspace_id\n AND s.path = nt.script_path\n ))\n OR\n (nt.is_flow = true AND EXISTS (\n SELECT 1 FROM flow f\n WHERE f.workspace_id = nt.workspace_id\n AND f.path = nt.script_path\n ))\n )\n LIMIT $3\n OFFSET $4\n ", "describe": { "columns": [ { @@ -62,6 +62,11 @@ "ordinal": 9, "name": "updated_at", "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "summary", + "type_info": "Varchar" } ], "parameters": { @@ -94,8 +99,9 @@ true, true, false, - false + false, + true ] }, - "hash": "a115d8ea786907561afdbbc07d11dc715d80b00c0e79b61b0057a3ae3886a85e" + "hash": "b0775af41a9b54cce040bf37cae770e63dab12a1383a0855d105c061e4e4ca48" } diff --git a/backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json b/backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json similarity index 66% rename from backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json rename to backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json index 706fa0c9ea..b40a643d50 100644 --- a/backend/.sqlx/query-40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e.json +++ b/backend/.sqlx/query-bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE native_trigger\n SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $5\n AND service_name = $6\n AND external_id = $7\n ", + "query": "\n UPDATE native_trigger\n SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, summary = $8, error = NULL, updated_at = NOW()\n WHERE\n workspace_id = $5\n AND service_name = $6\n AND external_id = $7\n ", "describe": { "columns": [], "parameters": { @@ -21,10 +21,11 @@ } } }, - "Text" + "Text", + "Varchar" ] }, "nullable": [] }, - "hash": "40a8bf6a5a42c275d73221bc5f386f2e18cb911352551d0a34bf1933e558674e" + "hash": "bf224f6441c36187f1402f9f01bfe15bb9edfa1dc9052f8a829e486b7334d708" } diff --git a/backend/migrations/20260323000000_native_trigger_summary.down.sql b/backend/migrations/20260323000000_native_trigger_summary.down.sql new file mode 100644 index 0000000000..7c7e63101d --- /dev/null +++ b/backend/migrations/20260323000000_native_trigger_summary.down.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger DROP COLUMN IF EXISTS summary; diff --git a/backend/migrations/20260323000000_native_trigger_summary.up.sql b/backend/migrations/20260323000000_native_trigger_summary.up.sql new file mode 100644 index 0000000000..7989a6fcd3 --- /dev/null +++ b/backend/migrations/20260323000000_native_trigger_summary.up.sql @@ -0,0 +1 @@ +ALTER TABLE native_trigger ADD COLUMN summary VARCHAR(1000); diff --git a/backend/windmill-api-integration-tests/tests/native_triggers.rs b/backend/windmill-api-integration-tests/tests/native_triggers.rs index 519e1f5262..e0a2e51ab5 100644 --- a/backend/windmill-api-integration-tests/tests/native_triggers.rs +++ b/backend/windmill-api-integration-tests/tests/native_triggers.rs @@ -422,6 +422,7 @@ async fn test_delete_integration_full_cascade(db: Pool) -> anyhow::Res "ext-1", &trigger_config, json!({"triggerType": "drive"}), + None, ) .await?; @@ -511,6 +512,7 @@ async fn test_cleanup_preserves_triggers(db: Pool) -> anyhow::Result<( "ext-1", &trigger_config, json!({"triggerType": "drive"}), + None, ) .await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 930cb94949..65eede32b2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -24592,6 +24592,10 @@ components: type: string nullable: true description: Error message if the trigger is in an error state + summary: + type: string + nullable: true + description: Short summary to be displayed when listed required: - external_id - workspace_id @@ -24626,6 +24630,10 @@ components: type: string nullable: true description: Error message if the trigger is in an error state + summary: + type: string + nullable: true + description: Short summary to be displayed when listed external_data: type: object description: Configuration data from the external service @@ -24718,6 +24726,10 @@ components: type: object description: Service-specific configuration (e.g., event types, filters) additionalProperties: true + summary: + type: string + nullable: true + description: Short summary to be displayed when listed required: - script_path - is_flow diff --git a/backend/windmill-native-triggers/src/handler.rs b/backend/windmill-native-triggers/src/handler.rs index b66a8ae3e0..cd6e1a8100 100644 --- a/backend/windmill-native-triggers/src/handler.rs +++ b/backend/windmill-native-triggers/src/handler.rs @@ -181,6 +181,7 @@ async fn create_native_trigger( &external_id, &config, service_config, + data.summary.as_deref(), ) .await?; @@ -304,6 +305,7 @@ async fn update_native_trigger_handler( &external_id, &config, service_config, + data.summary.as_deref(), ) .await?; diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 25234ff3f0..7853bb63e3 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -195,6 +195,7 @@ pub struct NativeTrigger { pub error: Option, pub created_at: DateTime, pub updated_at: DateTime, + pub summary: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -209,6 +210,7 @@ pub struct NativeTriggerData { pub script_path: String, pub is_flow: bool, pub service_config: C, + pub summary: Option, } #[derive(Debug, Clone, FromRow, Serialize, Deserialize)] @@ -821,6 +823,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> external_id: &str, config: &NativeTriggerConfig, service_config: C, + summary: Option<&str>, ) -> Result<()> { use windmill_common::auth::hash_token; @@ -835,12 +838,13 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> script_path, is_flow, webhook_token_hash, - service_config + service_config, + summary ) VALUES ( - $1, $2, $3, $4, $5, $6, $7 + $1, $2, $3, $4, $5, $6, $7, $8 ) ON CONFLICT (external_id, workspace_id, service_name) - DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, error = NULL, updated_at = NOW() + DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_hash = $6, service_config = $7, summary = $8, error = NULL, updated_at = NOW() "#, external_id, workspace_id, @@ -849,6 +853,7 @@ pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres> config.is_flow, webhook_token_hash, sqlx::types::Json(service_config) as _, + summary, ) .execute(db) .await?; @@ -863,6 +868,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres external_id: &str, config: &NativeTriggerConfig, service_config: Option<&RawValue>, + summary: Option<&str>, ) -> Result<()> { use windmill_common::auth::hash_token; @@ -871,7 +877,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres sqlx::query!( r#" UPDATE native_trigger - SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, error = NULL, updated_at = NOW() + SET script_path = $1, is_flow = $2, webhook_token_hash = $3, service_config = $4, summary = $8, error = NULL, updated_at = NOW() WHERE workspace_id = $5 AND service_name = $6 @@ -884,6 +890,7 @@ pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres workspace_id, service_name as ServiceName, external_id, + summary, ) .execute(db) .await?; @@ -934,7 +941,8 @@ pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( service_config, error, created_at, - updated_at + updated_at, + summary FROM native_trigger WHERE @@ -972,7 +980,8 @@ pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = P service_config, error, created_at, - updated_at + updated_at, + summary FROM native_trigger WHERE @@ -1018,7 +1027,8 @@ pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres> nt.service_config, nt.error, nt.created_at, - nt.updated_at + nt.updated_at, + nt.summary FROM native_trigger nt WHERE diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index be645a83fa..11f68bea96 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -231,6 +231,7 @@ export async function pushNativeTrigger( is_flow: result.is_flow, service_config: result.service_config, error: result.error, + summary: result.summary, }; log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`); } catch { @@ -243,6 +244,7 @@ export async function pushNativeTrigger( script_path: localTrigger.script_path, is_flow: localTrigger.is_flow, service_config: localTrigger.service_config, + summary: localTrigger.summary, }; if (remoteTrigger) { @@ -251,11 +253,13 @@ export async function pushNativeTrigger( script_path: localTrigger.script_path, is_flow: localTrigger.is_flow, service_config: localTrigger.service_config, + summary: localTrigger.summary, }; const remoteCompare = { script_path: remoteTrigger.script_path, is_flow: remoteTrigger.is_flow, service_config: remoteTrigger.service_config, + summary: remoteTrigger.summary, }; if (isSuperset(localCompare, remoteCompare)) { diff --git a/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte b/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte index 7334f0a62e..b6e28875ba 100644 --- a/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte +++ b/frontend/src/lib/components/triggers/native/NativeTriggerEditor.svelte @@ -11,6 +11,7 @@ import { usedTriggerKinds, userStore, workspaceStore } from '$lib/stores' import { canWrite, emptyString, sendUserToast } from '$lib/utils' import { Button } from '$lib/components/common' + import TextInput from '$lib/components/text_input/TextInput.svelte' import Drawer from '$lib/components/common/drawer/Drawer.svelte' import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' import { Loader2, Save } from 'lucide-svelte' @@ -94,6 +95,7 @@ let initialScriptPath = $state('') let fixedScriptPath = $state('') let isFlow = $state(false) + let summary = $state('') let externalId = $state(null) let can_write = $state(true) let originalConfig = $state | undefined>(undefined) @@ -123,6 +125,7 @@ can_write = true originalConfig = undefined initialConfig = undefined + summary = '' } export function openRecreate(nativeTrigger: ExtendedNativeTrigger) { @@ -146,6 +149,7 @@ can_write = true originalConfig = undefined initialConfig = undefined + summary = nativeTrigger.summary ?? '' } export async function openEdit( @@ -182,6 +186,7 @@ scriptPath = fullTrigger.script_path initialScriptPath = fullTrigger.script_path can_write = canWrite(fullTrigger.script_path, {}, $userStore) + summary = fullTrigger.summary ?? '' externalData = fullTrigger.external_data // Apply default values if provided (for draft triggers) @@ -203,7 +208,8 @@ return { script_path: scriptPath, is_flow: isFlow, - service_config: serviceConfig + service_config: serviceConfig, + summary: summary !== '' ? summary : undefined } } @@ -386,6 +392,22 @@ {/if}
+
+
+ +
+
+ {#if !hideTarget}

diff --git a/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte b/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte index 6314c12e82..8f8b84cd57 100644 --- a/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte +++ b/frontend/src/lib/components/triggers/native/NativeTriggerTable.svelte @@ -98,20 +98,31 @@ {@html trigger.marked} {:else} - {trigger.script_path} + {trigger.summary || trigger.script_path} {/if}

+ {#if trigger.summary} +
+ {trigger.script_path} +
+ {/if} {#if service === 'google'} {@const triggerType = trigger.service_config?.triggerType} {@const resourceName = trigger.service_config?.resourceName} {@const calendarName = trigger.service_config?.calendarName} -
+
{#if triggerType === 'calendar'} Calendar: {calendarName || trigger.service_config?.calendarId || ''} {:else} - Drive: {resourceName ? resourceName : trigger.service_config?.resourceId ? trigger.service_config.resourceId : 'All changes'} + Drive: {resourceName + ? resourceName + : trigger.service_config?.resourceId + ? trigger.service_config.resourceId + : 'All changes'} {/if}
{/if} diff --git a/frontend/src/lib/components/triggers/native/utils.ts b/frontend/src/lib/components/triggers/native/utils.ts index 34b1221b26..92082882e4 100644 --- a/frontend/src/lib/components/triggers/native/utils.ts +++ b/frontend/src/lib/components/triggers/native/utils.ts @@ -111,7 +111,7 @@ export function validateCommonFields(config: Record): Record Date: Tue, 24 Mar 2026 07:27:41 -0600 Subject: [PATCH 17/25] allow modern email TLDs in superadmin setup form (#8472) --- .../(root)/(logged)/user/(user)/instance_settings/+page.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte index d80e7fda85..66d9568c09 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte @@ -143,7 +143,7 @@ } } - const emailPattern = /^[\w-.]+@([\w-]+\.)+[\w-]{2,4}$/ + const emailPattern = /^[^\s@]+@[^\s@]+\.[^\s@]{2,}$/ let emailValid = $derived(emailPattern.test(newEmail)) let passwordValid = $derived(newPassword.length >= 2) let accountFormValid = $derived(emailValid && passwordValid) From 6d63d9973d2f5bfb86e691b00b5a6495b2ac305b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 13:31:06 +0000 Subject: [PATCH 18/25] chore(main): release 1.663.0 (#8465) * chore(main): release 1.663.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 25 ++ backend/Cargo.lock | 255 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 170 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07701ce7ce..c8aa1a9648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.663.0](https://github.com/windmill-labs/windmill/compare/v1.662.0...v1.663.0) (2026-03-24) + + +### Features + +* add summary field for native triggers ([#8476](https://github.com/windmill-labs/windmill/issues/8476)) ([5089a45](https://github.com/windmill-labs/windmill/commit/5089a458819abbc6f241bc354bebb91520bd1a52)) +* add typed request body to OpenAPI spec generation ([#8481](https://github.com/windmill-labs/windmill/issues/8481)) ([37ebaf4](https://github.com/windmill-labs/windmill/commit/37ebaf4d0ac342703498733f97778a552f979f6a)) +* **cli:** better stale scripts detection [#3](https://github.com/windmill-labs/windmill/issues/3) ([#8480](https://github.com/windmill-labs/windmill/issues/8480)) ([9643006](https://github.com/windmill-labs/windmill/commit/9643006f1e90b991b334bb58caf62301bc26d09d)) +* Debounce node ([#8324](https://github.com/windmill-labs/windmill/issues/8324)) ([5d1c54d](https://github.com/windmill-labs/windmill/commit/5d1c54d9b33d6ff6f2c98481a2740d1e7629cdfa)) +* surface permissioned_as selector in trigger editor UI ([#8475](https://github.com/windmill-labs/windmill/issues/8475)) ([f035b53](https://github.com/windmill-labs/windmill/commit/f035b538bbd786445526339f88be8f33a3628105)) + + +### Bug Fixes + +* clean up stale dependency map entries for renamed scripts ([#8492](https://github.com/windmill-labs/windmill/issues/8492)) ([47c0c36](https://github.com/windmill-labs/windmill/commit/47c0c363f4fc1d9af7efd07ea172e32989ce50d2)) +* **cli:** add Svelte 5 event delegation guidance and safe push to raw-app skill ([#8466](https://github.com/windmill-labs/windmill/issues/8466)) ([911df95](https://github.com/windmill-labs/windmill/commit/911df958e78d2dab9823dfa7d7e5c9824fc2d565)) +* Fix worker panic when job_isolation changed to unshare at runtime ([#8490](https://github.com/windmill-labs/windmill/issues/8490)) ([cbe47c0](https://github.com/windmill-labs/windmill/commit/cbe47c0b6c22f79452d020777e481ee26970f25b)) +* improve SQS retries ([3c8d351](https://github.com/windmill-labs/windmill/commit/3c8d351c9722a089133871019d27cf3bc3cdc159)) +* Move database manager SQL queries to backend ([#8306](https://github.com/windmill-labs/windmill/issues/8306)) ([aa30fd2](https://github.com/windmill-labs/windmill/commit/aa30fd252dcf40233d191c43a6293fb9feabf010)) +* prevent SQL injection in job query parameters ([#8494](https://github.com/windmill-labs/windmill/issues/8494)) ([54f5a19](https://github.com/windmill-labs/windmill/commit/54f5a19377e9df712e18f85f896e21b1776981ed)) +* respect NO_COLOR env variable for stdout log output ([#8483](https://github.com/windmill-labs/windmill/issues/8483)) ([f329ee7](https://github.com/windmill-labs/windmill/commit/f329ee7aaefbae0ad344743c40825440a936bd30)) +* show effective isolation level on workers page ([#8491](https://github.com/windmill-labs/windmill/issues/8491)) ([37886ed](https://github.com/windmill-labs/windmill/commit/37886edda1443293806a9b1b810196b72e076b12)) +* skip debounce arg accumulation when batch table is empty (CE) ([#8485](https://github.com/windmill-labs/windmill/issues/8485)) ([010753c](https://github.com/windmill-labs/windmill/commit/010753c73ac85237af50acadf9c08567b1bc993c)) +* stop_after_if with empty error_message prevents flow from stopping ([#8464](https://github.com/windmill-labs/windmill/issues/8464)) ([1503bf9](https://github.com/windmill-labs/windmill/commit/1503bf948e3340b8a6933d71885f8f2cb8dc1867)) + ## [1.662.0](https://github.com/windmill-labs/windmill/compare/v1.661.0...v1.662.0) (2026-03-20) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e40478cc4b..f044478b25 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -234,9 +234,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +checksum = "a07d1f37ff60921c83bdfc7407723bdefe89b44b98a9b772f225c8f9d67141a6" dependencies = [ "rustversion", ] @@ -2536,12 +2536,6 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -4887,19 +4881,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_more" -version = "0.99.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version 0.4.1", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "1.0.0" @@ -4927,6 +4908,7 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.117", + "unicode-xid", ] [[package]] @@ -7419,13 +7401,13 @@ dependencies = [ [[package]] name = "ipconfig" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +checksum = "2d72a21f6a71a6c4c3160e095e8925861f5119dd26ef71acee1b9146f74f76c8" dependencies = [ - "socket2 0.5.10", + "socket2 0.6.3", "widestring", - "windows-sys 0.48.0", + "windows-sys 0.61.2", "winreg", ] @@ -7446,9 +7428,9 @@ dependencies = [ [[package]] name = "iri-string" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb" dependencies = [ "memchr", "serde", @@ -7538,7 +7520,7 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.1", "log", "thiserror 1.0.69", "walkdir", @@ -7547,9 +7529,31 @@ dependencies = [ [[package]] name = "jni-sys" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] [[package]] name = "jobserver" @@ -8308,9 +8312,9 @@ dependencies = [ [[package]] name = "malachite" -version = "0.4.18" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a6ecab92657eb234bfe98abd0b17920772c6b14ce69256950142e2eb36d000b" +checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" dependencies = [ "malachite-base", "malachite-nz", @@ -8331,11 +8335,11 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17703a19c80bbdd0b7919f0f104f3b0597f7de4fc4e90a477c15366a5ba03faa" +checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" dependencies = [ - "derive_more 0.99.20", + "derive_more 1.0.0", "malachite", "num-integer", "num-traits", @@ -8610,9 +8614,9 @@ dependencies = [ [[package]] name = "moka" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85f8024e1c8e71c778968af91d43700ce1d11b219d127d79fb2934153b82b42b" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" dependencies = [ "async-lock", "crossbeam-channel", @@ -8826,7 +8830,7 @@ version = "0.5.0+25.2.9519653" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c196769dd60fd4f363e11d948139556a344e79d451aeb2fa2fd040738ef7691" dependencies = [ - "jni-sys", + "jni-sys 0.3.1", ] [[package]] @@ -11663,7 +11667,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.9", + "rustls-webpki 0.103.10", "subtle", "zeroize", ] @@ -11747,7 +11751,7 @@ dependencies = [ "rustls 0.23.35", "rustls-native-certs 0.8.3", "rustls-platform-verifier-android", - "rustls-webpki 0.103.9", + "rustls-webpki 0.103.10", "security-framework 3.6.0", "security-framework-sys", "webpki-root-certs 1.0.6", @@ -11795,9 +11799,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" dependencies = [ "aws-lc-rs", "ring 0.17.14", @@ -13877,12 +13881,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix 1.1.4", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -14468,9 +14472,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.10+spec-1.1.0" +version = "1.1.0+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" +checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011" dependencies = [ "winnow 1.0.0", ] @@ -15742,7 +15746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -15818,7 +15822,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15831,7 +15835,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "argon2", @@ -15972,7 +15976,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15995,7 +15999,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16008,7 +16012,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16034,7 +16038,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.662.0" +version = "1.663.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16044,7 +16048,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16061,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16084,7 +16088,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16107,7 +16111,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16123,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16143,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16163,7 +16167,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16177,7 +16181,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -16205,7 +16209,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16230,7 +16234,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16248,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16260,6 +16264,7 @@ dependencies = [ "serde_json", "serde_yml", "sqlx", + "tracing", "url", "windmill-api-auth", "windmill-common", @@ -16269,7 +16274,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16289,7 +16294,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16319,7 +16324,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16346,7 +16351,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.662.0" +version = "1.663.0" dependencies = [ "lazy_static", "serde", @@ -16358,7 +16363,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.662.0" +version = "1.663.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16381,7 +16386,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16395,7 +16400,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.662.0" +version = "1.663.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16426,7 +16431,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.662.0" +version = "1.663.0" dependencies = [ "chrono", "lazy_static", @@ -16440,7 +16445,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16459,7 +16464,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.662.0" +version = "1.663.0" dependencies = [ "aes-gcm", "anyhow", @@ -16559,7 +16564,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.662.0" +version = "1.663.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16578,7 +16583,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.662.0" +version = "1.663.0" dependencies = [ "regex", "serde", @@ -16593,7 +16598,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16617,7 +16622,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "futures", @@ -16634,7 +16639,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.662.0" +version = "1.663.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16650,7 +16655,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -16671,7 +16676,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -16702,7 +16707,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-oauth2", @@ -16726,7 +16731,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-stream", @@ -16760,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "futures", @@ -16778,7 +16783,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.662.0" +version = "1.663.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16787,7 +16792,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16799,7 +16804,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde_json", @@ -16811,7 +16816,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "gosyn", @@ -16823,7 +16828,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16835,7 +16840,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde_json", @@ -16847,7 +16852,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "nu-parser", @@ -16858,7 +16863,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16869,7 +16874,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16881,7 +16886,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16892,7 +16897,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -16914,7 +16919,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16928,7 +16933,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16945,7 +16950,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16958,7 +16963,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde", @@ -16970,7 +16975,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "lazy_static", @@ -16988,7 +16993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17004,7 +17009,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17020,7 +17025,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "serde", @@ -17031,7 +17036,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -17068,7 +17073,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "const_format", @@ -17106,7 +17111,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.662.0" +version = "1.663.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17117,7 +17122,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-recursion", @@ -17146,7 +17151,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17169,7 +17174,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17202,7 +17207,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17222,7 +17227,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17256,7 +17261,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17291,7 +17296,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17314,7 +17319,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17338,7 +17343,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-nats", @@ -17362,7 +17367,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17397,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17425,7 +17430,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-trait", @@ -17448,7 +17453,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17467,7 +17472,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.662.0" +version = "1.663.0" dependencies = [ "anyhow", "async-once-cell", @@ -17574,7 +17579,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.662.0" +version = "1.663.0" dependencies = [ "bytes", "futures", @@ -18189,12 +18194,12 @@ checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8" [[package]] name = "winreg" -version = "0.50.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" dependencies = [ "cfg-if", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 744518e703..6dc821689c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.662.0" +version = "1.663.0" authors.workspace = true edition.workspace = true @@ -82,7 +82,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.662.0" +version = "1.663.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 65eede32b2..8ef9caf936 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.662.0 + version: 1.663.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index b235377979..f93b61c6ee 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.662.0"; +export const VERSION = "v1.663.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 8804599f4a..83e9c1bbc8 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { workspaceAdd, }; -export const VERSION = "1.662.0"; +export const VERSION = "1.663.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 10be9acea0..792e303b6e 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 1dc86caae8..39bb3b6365 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.662.0", + "version": "1.663.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 5e0a6f2c6c..130d95a820 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.662.0" +wmill = ">=1.663.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index fa0e2e22cc..279b4589da 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.662.0 + version: 1.663.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index a4fa958a3d..c84a6d49f7 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.662.0' + ModuleVersion = '1.663.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 6e1bbf495b..92fbaf56d1 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.662.0" +version = "1.663.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index d1f9ea1b1a..5435328c88 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.662.0", + "version": "1.663.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8a37e00e0c..bd66c04c51 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.662.0", + "version": "1.663.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index fd3cab66e3..eb4feec596 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.662.0 +1.663.0 From 7f27d996accb3c3b471d1c50df397867d89c738a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 15:16:10 +0000 Subject: [PATCH 19/25] fix: create parent dirs and accept 'python' alias in script bootstrap (#8497) Co-authored-by: Claude Opus 4.5 --- cli/src/commands/script/script.ts | 17 +++++++-- cli/test/standalone_commands.test.ts | 57 ++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 14f3155ff7..10015db776 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -1,7 +1,7 @@ import { GlobalOptions } from "../../types.ts"; import { requireLogin } from "../../core/auth.ts"; import { resolveWorkspace, validatePath } from "../../core/context.ts"; -import { readFile, writeFile, stat } from "node:fs/promises"; +import { readFile, writeFile, stat, mkdir } from "node:fs/promises"; import { Buffer } from "node:buffer"; import { colors } from "@cliffy/ansi/colors"; import { Command } from "@cliffy/command"; @@ -1069,16 +1069,22 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) { } } +const languageAliases: Record = { + python: "python3", +}; + async function bootstrap( opts: GlobalOptions & { summary: string; description: string }, scriptPath: string, - language: ScriptLanguage + language: ScriptLanguage | string ) { if (!validatePath(scriptPath)) { return; } - const scriptInitialCode = scriptBootstrapCode[language]; + const resolvedLanguage = (languageAliases[language] ?? language) as ScriptLanguage; + + const scriptInitialCode = scriptBootstrapCode[resolvedLanguage]; if (scriptInitialCode === undefined) { throw new Error("Language unknown"); } @@ -1086,7 +1092,7 @@ async function bootstrap( const config = await readConfigFile(); const extension = filePathExtensionFromContentType( - language, + resolvedLanguage, config.defaultTs ); const scriptCodeFileFullPath = scriptPath + extension; @@ -1118,6 +1124,9 @@ async function bootstrap( yamlOptions ); + const parentDir = path.dirname(scriptCodeFileFullPath); + await mkdir(parentDir, { recursive: true }); + await writeFile(scriptCodeFileFullPath, scriptInitialCode, { flag: 'wx', encoding: 'utf-8', }); diff --git a/cli/test/standalone_commands.test.ts b/cli/test/standalone_commands.test.ts index 106e4aaeb1..1d75680354 100644 --- a/cli/test/standalone_commands.test.ts +++ b/cli/test/standalone_commands.test.ts @@ -409,6 +409,63 @@ describe("script bootstrap command", () => { }); }); + test("accepts 'python' as alias for python3", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`, + "utf-8" + ); + + await mkdir(join(tempDir, "f", "test"), { recursive: true }); + + const result = await backend.runCLICommand( + ["script", "bootstrap", "f/test/py_alias_script", "python"], + tempDir + ); + + expect(result.code).toEqual(0); + + const codeStat = await stat(join(tempDir, "f/test/py_alias_script.py")); + expect(codeStat.isFile()).toBe(true); + + const metaStat = await stat( + join(tempDir, "f/test/py_alias_script.script.yaml") + ); + expect(metaStat.isFile()).toBe(true); + }); + }); + + test("creates parent directories automatically", async () => { + await withTestBackend(async (backend, tempDir) => { + await setupWorkspaceProfile(backend); + + await writeFile( + join(tempDir, "wmill.yaml"), + `defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`, + "utf-8" + ); + + // Do NOT pre-create f/test — bootstrap should create it + const result = await backend.runCLICommand( + ["script", "bootstrap", "f/test/auto_dir_script", "bun"], + tempDir + ); + + expect(result.code).toEqual(0); + + const codeStat = await stat(join(tempDir, "f/test/auto_dir_script.ts")); + expect(codeStat.isFile()).toBe(true); + + const metaStat = await stat( + join(tempDir, "f/test/auto_dir_script.script.yaml") + ); + expect(metaStat.isFile()).toBe(true); + }); + }); + test("creates Go script files", async () => { await withTestBackend(async (backend, tempDir) => { await setupWorkspaceProfile(backend); From 3c34d19813752c7c3d718ac30a60266942b10909 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 15:41:39 +0000 Subject: [PATCH 20/25] escape env var values in nativets/bun JS string interpolation (#8500) Co-authored-by: Claude Opus 4.5 --- backend/windmill-worker/src/bun_executor.rs | 5 ++++- backend/windmill-worker/src/worker.rs | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index fb07806fd6..f0ed8793a7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3008,7 +3008,10 @@ pub fn build_nativets_env_code( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() - .map(|(k, v)| format!("process.env['{}'] = '{}';", k, v)) + .map(|(k, v)| { + let escaped = v.replace('\\', "\\\\").replace('\'', "\\'").replace('\n', "\\n").replace('\r', "\\r"); + format!("process.env['{}'] = '{}';", k, escaped) + }) .collect::>() .join("\n") ) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 03e97a6ac1..b698f351c6 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4493,7 +4493,10 @@ pub async fn run_language_executor( "const process = {{ env: {{}} }};\nconst BASE_URL = '{base_internal_url}';\nconst BASE_INTERNAL_URL = '{base_internal_url}';\nprocess.env['BASE_URL'] = BASE_URL;process.env['BASE_INTERNAL_URL'] = BASE_INTERNAL_URL;\n{}", reserved_variables .iter() - .map(|(k, v)| format!("const {} = '{}';\nprocess.env['{}'] = '{}';\n", k, v, k, v)) + .map(|(k, v)| { + let escaped = v.replace('\\', "\\\\").replace('\'', "\\'").replace('\n', "\\n").replace('\r', "\\r"); + format!("const {} = '{}';\nprocess.env['{}'] = '{}';\n", k, escaped, k, escaped) + }) .collect::>() .join("\n")); From 2048a36376a9e931fdcef5c751d8f77918b1a94c Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 24 Mar 2026 16:42:16 +0100 Subject: [PATCH 21/25] Fix select key bug (#8499) --- frontend/src/lib/components/select/SelectDropdown.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/select/SelectDropdown.svelte b/frontend/src/lib/components/select/SelectDropdown.svelte index 62a990bb46..d9fc2dcf70 100644 --- a/frontend/src/lib/components/select/SelectDropdown.svelte +++ b/frontend/src/lib/components/select/SelectDropdown.svelte @@ -117,7 +117,7 @@ ulClass )} > - {#each processedItems ?? [] as item, itemIndex (item.value)} + {#each processedItems ?? [] as item, itemIndex} {#if (item.__select_group && itemIndex === 0) || processedItems?.[itemIndex - 1]?.__select_group !== item.__select_group}
  • Date: Tue, 24 Mar 2026 12:00:32 -0400 Subject: [PATCH 22/25] fix: add GIT_SSL_CAINFO to tracing proxy env vars (#8502) Git uses libcurl with GnuTLS on Debian, which doesn't read SSL_CERT_FILE or CURL_CA_BUNDLE for CA trust. When the OTEL tracing proxy is enabled, git clone fails with "certificate signer not trusted" because it can't verify the proxy's MITM certificate. Adding GIT_SSL_CAINFO pointing to the proxy CA cert fixes this. Co-authored-by: Claude Opus 4.6 --- backend/windmill-worker/src/worker.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index b698f351c6..bdeea1c076 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -965,6 +965,7 @@ async fn get_otel_tracing_proxy_envs( TRACING_PROXY_CA_CERT_PATH.to_string(), ), ("CURL_CA_BUNDLE", TRACING_PROXY_CA_CERT_PATH.to_string()), + ("GIT_SSL_CAINFO", TRACING_PROXY_CA_CERT_PATH.to_string()), ("DENO_CERT", TRACING_PROXY_CA_CERT_PATH.to_string()), ]) } From 8cfaa91d43acd821ce79dcb2e179ce2590c3386b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 24 Mar 2026 16:01:18 +0000 Subject: [PATCH 23/25] update cli freshness --- cli/src/guidance/skills.ts | 13 +++---------- system_prompts/auto-generated/cli/cli-commands.md | 2 ++ system_prompts/auto-generated/prompts.ts | 2 ++ .../auto-generated/skills/cli-commands/SKILL.md | 2 ++ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index 45279e631f..6872b610ef 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -4602,18 +4602,9 @@ Tell the user they can run these commands (do NOT run them yourself): | \`wmill app dev\` | Start dev server with live reload | | \`wmill app generate-agents\` | Refresh AGENTS.md and DATATABLES.md | | \`wmill app generate-locks\` | Generate lock files for backend runnables | -| \`wmill sync push --extra-includes "f//.raw_app/**" --yes\` | Deploy this specific raw app to Windmill (never do a blanket \`wmill sync push\`) | +| \`wmill sync push\` | Deploy app to Windmill | | \`wmill sync pull\` | Pull latest from Windmill | -## Svelte 5 Event Handling - -When building Svelte 5 raw apps, be aware of event delegation: - -- The Svelte runtime version in \`node_modules/svelte\` **must match** the compiler version used by \`wmill sync push\`. If you get \`$.delegated is undefined\` errors at runtime, run \`npm install svelte@latest\` in the raw app folder and re-push. -- \`onclick\` on \`
    \`, \`\`, and other non-interactive elements uses Svelte's event delegation system. If the runtime doesn't support it, you'll get errors. -- \`onclick\` on \`
    {/if} {#if node.workflow_as_code_status}
    -
    Workflow timeline
    +
    Workflow timeline
    flowModuleSchemaMap?.deleteMultiple(resolvedModuleIds)} onDuplicateSelected={() => flowModuleSchemaMap?.duplicateMultiple(resolvedModuleIds)} onMoveSelected={() => flowModuleSchemaMap?.moveMultiple(resolvedModuleIds)} + onCreateGroup={() => flowModuleSchemaMap?.createGroup(selectionManager.selectedIds)} {canMoveSelected} resolvedCount={resolvedModuleIds.length} /> diff --git a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte index 7efb44b82d..1b54d83928 100644 --- a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte @@ -3,8 +3,8 @@ import type { SelectionManager } from '$lib/components/graph/selectionUtils.svelte' import { Button } from '$lib/components/common' import DropdownV2 from '$lib/components/DropdownV2.svelte' - import { getNoteEditorContext } from '$lib/components/graph/noteEditor.svelte' - import { StickyNote, Move, Copy, Trash2 } from 'lucide-svelte' + import { getGroupEditorContext } from '$lib/components/graph/groupEditor.svelte' + import { Group, Move, Copy, Trash2 } from 'lucide-svelte' import type { Item } from '$lib/utils' interface Props { @@ -13,6 +13,7 @@ onDeleteSelected?: () => void onDuplicateSelected?: () => void onMoveSelected?: () => void + onCreateGroup?: () => void canMoveSelected?: boolean resolvedCount?: number } @@ -22,18 +23,14 @@ onDeleteSelected, onDuplicateSelected, onMoveSelected, + onCreateGroup, canMoveSelected = false, resolvedCount = 0 }: Props = $props() - const noteEditorContext = getNoteEditorContext() + const groupEditorContext = getGroupEditorContext() - function addGroupNote() { - if (selectionManager.selectedIds.length > 0 && noteEditorContext?.noteEditor) { - // Create the group note - noteEditorContext.noteEditor.createGroupNote(selectionManager.selectedIds) - } - } + let canCreateGroup = $derived(groupEditorContext?.canCreateGroup.val ?? false) let menuItems: Item[] = $derived([ { @@ -60,11 +57,11 @@ {#snippet action()}
    {#if resolvedCount > 0} diff --git a/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte b/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte index b0504a8055..51d1f07d3f 100644 --- a/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte +++ b/frontend/src/lib/components/flows/header/FlowYamlEditor.svelte @@ -31,9 +31,22 @@ editor?.setCode(code) } + function validateGroups(groups: { start_id: string; end_id: string }[] | undefined) { + if (!groups) return + const seen = new Set() + for (const g of groups) { + const key = `${g.start_id}:${g.end_id}` + if (seen.has(key)) { + throw new Error(`Duplicate group: '${g.start_id}' → '${g.end_id}'`) + } + seen.add(key) + } + } + function apply() { try { const parsed = YAML.parse(code) + validateGroups(parsed.value?.groups) if (parsed.summary && typeof parsed.summary === 'string') { flowStore.val.summary = parsed.summary } @@ -59,7 +72,7 @@ initialCode = code sendUserToast('Changes applied') } catch (e) { - ;(sendUserToast('Error parsing yaml: ' + e), true) + sendUserToast('Error parsing yaml: ' + e, true) } } @@ -69,8 +82,12 @@ drawer?.toggleDrawer()}> {#snippet actions()} - - + + {/snippet} {#if flowStore.val} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 8e0250cec8..e6a548b39a 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -192,10 +192,10 @@ !!id && !!$flowPropPickerConfig && !!pickableIds && Object.keys(pickableIds).includes(id) ) - let isDragging = $derived(!!moveManager?.dragging) + let isMoving = $derived(!!moveManager?.dragging || !!moveManager?.movingModuleId) const outputPickerVisible = $derived( - editMode && (isConnectingCandidate || alwaysShowOutputPicker) && !!id && !isDragging + editMode && (isConnectingCandidate || alwaysShowOutputPicker) && !!id && !isMoving ) const icon_render = $derived(icon) @@ -214,7 +214,7 @@ flowStore?.val?.value.failure_module )} - (editId = false)}> + (editId = false)}>
    {#snippet icon()} @@ -484,11 +482,10 @@ {/if}
    - {#if deletable && !isDragging} + {#if deletable && !isMoving} {#if maximizeSubflow !== undefined} {@render buttonMaximizeSubflow?.()} {/if} - {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)} - {#if editMode && enableTestRun && flowJob?.type !== 'QueuedJob' && !isDragging} + {#if editMode && enableTestRun && flowJob?.type !== 'QueuedJob' && !isMoving}
    (hover = false)} > {#if !isMultiSelected && (hover || selected || testRunDropdownOpen) && outputPickerVisible} -
    +
    {#if !testIsLoading}
    {/each} + + 0} + on:confirmed={() => { + affectedGroupsAction?.() + affectedGroupsPending = [] + affectedGroupsAction = undefined + affectedGroupsCancel = undefined + }} + on:canceled={() => { + affectedGroupsCancel?.() + affectedGroupsPending = [] + affectedGroupsAction = undefined + affectedGroupsCancel = undefined + }} + > + {#if affectedGroupsPending.length === 1} + {@const group = affectedGroupsPending[0]} +

    The group{group.summary ? ` "${group.summary}"` : ''} will be removed (empty or duplicate). + Are you sure you want to {affectedGroupsActionLabel} the step?

    + {:else} +

    The following groups will be removed (empty or duplicate):

    +
      + {#each affectedGroupsPending as group} +
    • {group.summary || `${group.start_id} → ${group.end_id}`}
      • + {/each} +
    +

    Are you sure you want to {affectedGroupsActionLabel} the step?

    + {/if} +
    { dependents = getDependentComponents(id, flowStore.val) - const cb = () => { - push(history, flowStore.val) - if (id === 'preprocessor') { + + if (id === 'preprocessor') { + const cb = () => { + push(history, flowStore.val) selectionManager.selectId('Input') flowStore.val.value.preprocessor_module = undefined - } else { - selectNextId(id) - removeAtId(flowStore.val.value.modules, id) + refreshStateStore(flowStore) + onDelete?.(id) + delete flowStateStore.val[id] } + if (Object.keys(dependents).length > 0) { + deleteCallback = cb + } else { + cb() + } + return + } + + const dsOpts = { displayState: groupDisplayState } + const { emptiedGroups, duplicateGroups, commit } = proxy.prepareMutation((tree) => { + const found = findInStructure(tree, id) + if (found) found.parentChildren.splice(found.index, 1) + }, dsOpts) + + const affectedGroups = [...emptiedGroups, ...duplicateGroups] + + const cb = () => { + push(history, flowStore.val) + selectNextId(id) + commit({ removeDuplicates: duplicateGroups.length > 0 }) refreshStateStore(flowStore) onDelete?.(id) delete flowStateStore.val[id] } - if (Object.keys(dependents).length > 0) { - deleteCallback = cb + const proceed = () => { + if (Object.keys(dependents).length > 0) { + deleteCallback = cb + } else { + cb() + } + } + + if (affectedGroups.length > 0) { + affectedGroupsPending = affectedGroups + affectedGroupsActionLabel = 'delete' + affectedGroupsAction = proceed } else { - cb() + proceed() } }} onInsert={async (detail) => { - { - let originalModules - let targetModules - if ( - detail.sourceId == 'Input' || - detail.targetId == 'Result' || - detail.kind == 'trigger' - ) { - targetModules = flowStore.val.value.modules + if (!flowStore.val.value.modules || !Array.isArray(flowStore.val.value.modules)) return + await tick() + + // --- MOVE --- + if (moveManager.movingModuleId) { + const movedIds = moveManager.movingIds ?? [moveManager.movingModuleId] + const movingId = moveManager.movingModuleId + + let mutated = false + const moveOpts = { displayState: groupDisplayState } + const { emptiedGroups, duplicateGroups, commit } = proxy.prepareMutation((tree) => { + let originalModules: FlowStructureNode[] | undefined + let targetModules: FlowStructureNode[] | undefined + + if (detail.sourceId == 'Input' || detail.targetId == 'Result') { + targetModules = tree + } + dfsStructure(tree, (node, parentArray) => { + if (matchStructureNode(node, movingId)) originalModules = parentArray + if (detail.branch && matchStructureNode(node, detail.branch.rootId)) { + targetModules = node.branches[detail.branch.branch]?.children + } else if ( + matchStructureNode(node, detail.sourceId ?? '') || + matchStructureNode(node, detail.targetId ?? '') + ) { + targetModules = parentArray + } + }) + + if (!originalModules || !targetModules) return + + if (movedIds.length > 1) { + const firstIndex = originalModules.findIndex((m) => + matchStructureNode(m, movedIds[0]) + ) + if (firstIndex < 0) return + const removedModules = originalModules.splice(firstIndex, movedIds.length) + let insertIndex = detail.index + if (originalModules === targetModules && firstIndex < detail.index) { + insertIndex -= movedIds.length + } + targetModules.splice(insertIndex, 0, ...removedModules) + } else { + const indexToRemove = originalModules.findIndex((m) => + matchStructureNode(m, movingId) + ) + if (indexToRemove < 0) return + const [removed] = originalModules.splice(indexToRemove, 1) + let insertIndex = detail.index + if (originalModules === targetModules && indexToRemove < detail.index) + insertIndex -= 1 + targetModules.splice(insertIndex, 0, removed) + } + mutated = true + }, moveOpts) + + if (!mutated) { + moveManager.clearMoving() + return } - dfs(flowStore.val.value.modules, (mod, modules, branches) => { - if (mod.id == moveManager.movingModuleId) { - originalModules = modules - } - if (detail.branch) { - if (mod.id == detail.branch.rootId) { - targetModules = branches[detail.branch.branch] - } - } else if (mod.id == detail.sourceId || mod.id == detail.targetId) { - targetModules = modules - } else if (mod.id == detail.agentId && mod.value.type === 'aiagent') { - targetModules = mod.value.tools - } - }) - if (flowStore.val.value.modules && Array.isArray(flowStore.val.value.modules)) { - await tick() - if (moveManager.movingModuleId) { - push(history, flowStore.val) - if (!originalModules || !targetModules) { - moveManager.clearMoving() - return - } - if (moveManager.movingIds && moveManager.movingIds.length > 1) { - // Multi-move: splice out all moving modules from their parent, insert at target - const firstIndex = originalModules.findIndex( - (m) => m.id === moveManager.movingIds?.[0] - ) - const removedModules = originalModules.splice( - firstIndex, - moveManager.movingIds.length - ) - let insertIndex = detail.index - if (originalModules === targetModules && firstIndex < detail.index) { - insertIndex -= moveManager.movingIds.length - } - targetModules.splice(insertIndex, 0, ...removedModules) - selectionManager.selectByIds(removedModules.map((m) => m.id)) - } else { - let indexToRemove = originalModules.findIndex((m) => moveManager.movingModuleId == m.id) - let [removedModule] = originalModules.splice(indexToRemove, 1) - // When moving within the same array, removal shifts subsequent indices down by 1 - let insertIndex = detail.index - if (originalModules === targetModules && indexToRemove < detail.index) { - insertIndex -= 1 - } - targetModules.splice(insertIndex, 0, removedModule) - selectionManager.selectId(removedModule.id) - } - moveManager.clearMoving() + const affectedGroups = [...emptiedGroups, ...duplicateGroups] + + const doMove = () => { + push(history, flowStore.val) + commit({ removeDuplicates: duplicateGroups.length > 0 }) + if (movedIds.length > 1) { + selectionManager.selectByIds(movedIds) } else { - if (detail.isPreprocessor) { - await insertNewPreprocessorModule( - flowStore, - flowStateStore, - detail.inlineScript, - detail.script - ) - selectionManager.selectId('preprocessor') - - if (detail.inlineScript?.instructions) { - dispatch('generateStep', { - moduleId: 'preprocessor', - lang: detail.inlineScript?.language, - instructions: detail.inlineScript?.instructions - }) - } - } else { - const index = (detail.agentId ? targetModules?.length : detail.index) ?? 0 - const toolKind: SpecialToolKind | 'flowmoduleTool' | undefined = detail.agentId - ? (SPECIAL_TOOL_KINDS as readonly string[]).includes(detail.kind) - ? (detail.kind as SpecialToolKind) - : 'flowmoduleTool' - : undefined - - await insertNewModuleAtIndex( - targetModules, - index, - detail.kind, - detail.script, - detail.flow, - detail.inlineScript, - toolKind - ) - const id = targetModules[index].id - selectionManager.selectId(id) - - if (detail.inlineScript?.instructions) { - dispatch('generateStep', { - moduleId: id, - lang: detail.inlineScript?.language, - instructions: detail.inlineScript?.instructions - }) - } - if (detail.kind == 'trigger') { - await insertNewModuleAtIndex( - targetModules, - index + 1, - 'forloop', - undefined, - undefined, - undefined - ) - setExpr(targetModules[index + 1], `results.${id}`) - setScheduledPollSchedule(triggersState, triggersCount) - } - - if (detail.flow?.path) { - loadLastJob(detail.flow.path, id) - } else if (detail.script?.path) { - loadLastJob(detail.script?.path, id) - } - } - } - - if (['branchone', 'branchall'].includes(detail.kind)) { - await addBranch(targetModules[detail.index ?? 0].id) + selectionManager.selectId(movingId) } + moveManager.clearMoving() refreshStateStore(flowStore) dispatch('change') } + + if (affectedGroups.length > 0) { + affectedGroupsPending = affectedGroups + affectedGroupsActionLabel = 'move' + affectedGroupsAction = doMove + affectedGroupsCancel = () => moveManager.clearMoving() + } else { + doMove() + } + return } + + // --- INSERT --- + if (detail.isPreprocessor) { + await insertNewPreprocessorModule( + flowStore, + flowStateStore, + detail.inlineScript, + detail.script + ) + selectionManager.selectId('preprocessor') + if (detail.inlineScript?.instructions) { + dispatch('generateStep', { + moduleId: 'preprocessor', + lang: detail.inlineScript?.language, + instructions: detail.inlineScript?.instructions + }) + } + refreshStateStore(flowStore) + dispatch('change') + return + } + + push(history, flowStore.val) + + const isAgentInsert = !!detail.agentId + const toolKind: SpecialToolKind | 'flowmoduleTool' | undefined = isAgentInsert + ? (SPECIAL_TOOL_KINDS as readonly string[]).includes(detail.kind) + ? (detail.kind as SpecialToolKind) + : 'flowmoduleTool' + : undefined + + // Agent tool inserts operate on the FlowModule's tools array directly + if (isAgentInsert) { + const agentMod = getAllModules(flowStore.val.value.modules).find( + (m) => m.id === detail.agentId + ) + if (agentMod && (agentMod.value as any).tools) { + const tools = (agentMod.value as any).tools as AgentTool[] + await insertNewModuleAtIndex( + tools, + tools.length, + detail.kind as InsertKind, + detail.script, + detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, + detail.inlineScript, + toolKind + ) + const id = tools[tools.length - 1].id + selectionManager.selectId(id) + } + refreshStateStore(flowStore) + dispatch('change') + return + } + + // Regular module insert: create the module, then insert a leaf node via tree mutation + const module = await createNewModule( + detail.kind as InsertKind, + detail.script, + detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, + detail.inlineScript + ) + const index = detail.index ?? 0 + const extraModules: FlowModule[] = [module] + + // For trigger inserts, also create the forloop module + let loopModule: FlowModule | undefined + if (detail.kind == 'trigger') { + loopModule = await createNewModule('forloop') + setExpr(loopModule, `results.${module.id}`) + extraModules.push(loopModule) + } + + proxy.applyTreeMutation( + (tree) => { + // Find target array in the snapshot + let targetArray: FlowStructureNode[] | undefined + if ( + detail.sourceId == 'Input' || + detail.targetId == 'Result' || + detail.kind == 'trigger' + ) { + targetArray = tree + } + dfsStructure(tree, (node, parentArray) => { + if (detail.branch && matchStructureNode(node, detail.branch.rootId)) { + targetArray = node.branches[detail.branch.branch]?.children + } else if ( + matchStructureNode(node, detail.sourceId ?? '') || + matchStructureNode(node, detail.targetId ?? '') + ) { + targetArray = parentArray + } + }) + if (!targetArray) targetArray = tree + + // Insert the structure node (correct kind for containers like branchone/branchall) + targetArray.splice(index, 0, moduleToStructureNode(module)) + + // For trigger: also insert the forloop node after it + if (loopModule) { + targetArray.splice(index + 1, 0, moduleToStructureNode(loopModule)) + } + }, + { extraModules, displayState: groupDisplayState } + ) + + selectionManager.selectId(module.id) + + if (detail.inlineScript?.instructions) { + dispatch('generateStep', { + moduleId: module.id, + lang: detail.inlineScript?.language, + instructions: detail.inlineScript?.instructions + }) + } + if (detail.kind == 'trigger') { + setScheduledPollSchedule(triggersState, triggersCount) + } + if (detail.flow?.path) { + loadLastJob(detail.flow.path, module.id) + } else if (detail.script?.path) { + loadLastJob(detail.script?.path, module.id) + } + + if (['branchone', 'branchall'].includes(detail.kind)) { + await addBranch(module.id) + } + refreshStateStore(flowStore) + dispatch('change') }} onNewBranch={async (id) => { if (id) { @@ -761,6 +973,17 @@ mod.id = newId } }) + const groups = flowStore.val.value.groups + if (groups) { + for (const group of groups) { + if (group.start_id === id) { + group.start_id = newId + } + if (group.end_id === id) { + group.end_id = newId + } + } + } flowStateStore.val[newId] = flowStateStore.val[id] delete flowStateStore.val[id] refreshStateStore(flowStore) diff --git a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte index 798116b47d..49cbb56cc6 100644 --- a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte +++ b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte @@ -46,7 +46,7 @@
    , allNodes: Node[], allEdges: Edge[]) { + function computeGhost( + moduleId: string, + draggedNodeIds: Set, + allNodes: Node[], + allEdges: Edge[] + ) { // Use pre-computed draggedNodeIds when available (covers multi-select), // otherwise fall back to single-module subflow computation. let sfNodes: Node[] @@ -111,7 +116,15 @@ zoom: scale } - return { containerWidth, containerHeight, ghostNodes, ghostEdges, offsetX, offsetY, initialViewport } + return { + containerWidth, + containerHeight, + ghostNodes, + ghostEdges, + offsetX, + offsetY, + initialViewport + } } let isNearDrop = $derived(moveManager.nearestDropZone != null) @@ -128,7 +141,8 @@ class="fixed pointer-events-none z-[10001] flex items-center justify-center w-5 h-5 rounded-full shadow border border-border transition-colors duration-150 {isNearDrop ? 'bg-surface-accent-primary text-white' : 'bg-surface text-secondary'}" - style="left: {moveManager.ghostScreenX + CURSOR_INDICATOR_OFFSET}px; top: {moveManager.ghostScreenY + CURSOR_INDICATOR_OFFSET}px;" + style="left: {moveManager.ghostScreenX + + CURSOR_INDICATOR_OFFSET}px; top: {moveManager.ghostScreenY + CURSOR_INDICATOR_OFFSET}px;" >
    diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 853dbf5c54..6a4d208579 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -59,8 +59,22 @@ import AiToolNode, { computeAIToolNodes } from './renderers/nodes/AIToolNode.svelte' import NewAiToolNode from './renderers/nodes/NewAIToolNode.svelte' import NoteNode from './renderers/nodes/NoteNode.svelte' + import CollapsedGroupNode from './renderers/nodes/CollapsedGroupNode.svelte' + import GroupHeadNode from './renderers/nodes/GroupHeadNode.svelte' + import GroupEndNode from './renderers/nodes/GroupEndNode.svelte' import NoteTool from './NoteTool.svelte' import SelectionBoundingBox from './SelectionBoundingBox.svelte' + import GroupOverlay from './GroupOverlay.svelte' + import { + GroupDisplayState, + getGroupEditorContext, + groupKey, + type FlowGroup + } from './groupEditor.svelte' + import { buildStructureTree, computeGroupDepths, type FlowStructureNode } from './flowStructure' + import { stateSnapshot } from '$lib/svelte5Utils.svelte' + import { computeGroupModuleIds } from './groupDetectionUtils' + import { getAllModules } from '../flows/flowExplorer' import SelectionTool from './SelectionTool.svelte' import PaneContextMenu from './PaneContextMenu.svelte' import { SelectionManager } from './selectionUtils.svelte' @@ -72,6 +86,7 @@ import { compoundLayout } from './compoundLayout' import { deepEqual } from 'fast-equals' import type { AssetWithAltAccessType } from '../assets/lib' + import { computeNodeExtraSpace } from './nodeExtraSpace' import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' import { setGraphContext } from './graphContext' import { computeNoteNodes } from './noteUtils.svelte' @@ -100,6 +115,8 @@ interface Props { success?: boolean | undefined modules?: FlowModule[] | undefined + groupedModules?: FlowStructureNode[] + groupError?: unknown failureModule?: FlowModule | undefined preprocessorModule?: FlowModule | undefined minHeight?: number @@ -124,7 +141,7 @@ workspace?: string editMode?: boolean allowSimplifiedPoll?: boolean - expandedSubflows?: Record + expandedSubflows?: Record isOwner?: boolean isRunning?: boolean individualStepTests?: boolean @@ -133,6 +150,8 @@ suspendStatus?: Record noteMode?: boolean notes?: FlowNote[] + groups?: FlowGroup[] + groupDisplayState?: GroupDisplayState chatInputEnabled?: boolean multiSelectEnabled?: boolean onDeleteMultiple?: (ids: string[]) => void @@ -152,6 +171,7 @@ script?: { path: string; summary: string; hash: string | undefined } flow?: { path: string; summary: string } kind: InsertKind + expandGroup?: { groupId: string; position: 'top' | 'bottom' } }) => Promise onNewBranch?: (id: string) => Promise onSelect?: (id: string | FlowModule) => void @@ -193,6 +213,8 @@ onSelectedIteration = undefined, success = undefined, modules = [], + groupedModules: groupedModulesProp = undefined, + groupError = undefined, failureModule = undefined, preprocessorModule = undefined, minHeight = 0, @@ -232,6 +254,8 @@ flowHasChanged = false, noteMode = false, notes = undefined, + groups = undefined, + groupDisplayState: groupDisplayStateProp = undefined, exitNoteMode = undefined, onNotePositionUpdate = undefined, chatInputEnabled = false, @@ -257,6 +281,9 @@ () => nodes ) + const groupDisplayState = + untrack(() => groupDisplayStateProp) ?? new GroupDisplayState(() => groups ?? []) + // Runtime text height tracking for notes (not stored in FlowNote) let noteTextHeights = $state>({}) @@ -264,6 +291,8 @@ let paneContextMenu: PaneContextMenu | undefined = $state(undefined) let flowContainer: HTMLDivElement | undefined = $state(undefined) + // Hover tracking for group overlay + // Selection manager - create one if not provided let selectionManager = untrack(() => selectionManagerProp) || new SelectionManager() const selectedId = $derived(selectionManager.getSelectedId()) @@ -298,7 +327,9 @@ moveManager: untrack(() => moveManager), clearFlowSelection, yOffset, - diffManager + diffManager, + getFlowNodes: () => currentGraphNodeDeps, + groupDisplayState } as any) if (triggerContext && untrack(() => allowSimplifiedPoll)) { @@ -332,14 +363,36 @@ type NodeDep = { id: string parentIds?: string[] - data?: { assets?: AssetWithAltAccessType[] } + data?: { assets?: AssetWithAltAccessType[]; module?: any } } type NodePos = { position: { x: number; y: number } } - let lastNodes: [NodeDep[], (NodeDep & NodePos)[]] | undefined = undefined + let lastNodes: + | [NodeDep[], Map | undefined, (NodeDep & NodePos)[]] + | undefined = undefined + let currentGraphNodeDeps: { id: string; parentIds?: string[] }[] = $state([]) - function layoutNodes(nodes: NodeDep[]): (NodeDep & NodePos)[] { - let lastResult = lastNodes?.[1] - if (lastResult && deepEqual(nodes, lastNodes?.[0])) { + // Keep canCreateGroup in sync for consumers (SelectionBoundingBox, FlowSelectionPanel, etc.) + const groupEditorCtx = getGroupEditorContext() + + $effect(() => { + if (!groupEditorCtx) return + const ids = selectionManager.selectedIds + groupEditorCtx.canCreateGroup.val = + ids.length >= 1 && groupEditorCtx.groupEditor.canCreateGroup(ids, currentGraphNodeDeps) + }) + + let lastGroupDimensions: Map | undefined = undefined + + function layoutNodes( + nodes: NodeDep[], + nodeExtraSpace?: Map + ): (NodeDep & NodePos)[] { + let lastResult = lastNodes?.[2] + if ( + lastResult && + deepEqual(nodes, lastNodes?.[0]) && + deepEqual(nodeExtraSpace, lastNodes?.[1]) + ) { console.debug('layoutNodes', 'same nodes') return lastResult } @@ -352,16 +405,23 @@ seenId.push(n.id) } - // Run recursive compound layout - const { positions, bbox } = compoundLayout(nodes, { - nodeWidth: NODE.width, - nodeHeight: NODE.height, - gapH: NODE.gap.horizontal, - gapV: NODE.gap.vertical - }) + // Run recursive compound layout with pre-computed extra space + const layoutResult = compoundLayout( + nodes, + { + nodeWidth: NODE.width, + nodeHeight: NODE.height, + gapH: NODE.gap.horizontal, + gapV: NODE.gap.vertical + }, + nodeExtraSpace + ) + const { positions, bbox } = layoutResult + lastGroupDimensions = layoutResult.groupDimensions + + const xCenter = (fullSize ? fullWidth : width) / 2 - bbox.width / 2 - (width - fullWidth) / 2 // Center horizontally - const xCenter = (fullSize ? fullWidth : width) / 2 - bbox.width / 2 - (width - fullWidth) / 2 const newNodes = nodes.map((n) => ({ id: n.id, position: { @@ -370,7 +430,7 @@ } })) - lastNodes = [nodes, newNodes] + lastNodes = [nodes, nodeExtraSpace, newNodes] return newNodes } @@ -414,13 +474,16 @@ }, expandSubflow: async (id: string, path: string) => { const flow = await FlowService.getFlowByPath({ workspace: workspace, path }) - expandedSubflows[id] = flow.value.modules + expandedSubflows[id] = { modules: flow.value.modules, groups: flow.value.groups } expandedSubflows = expandedSubflows }, minimizeSubflow: (id: string) => { delete expandedSubflows[id] expandedSubflows = expandedSubflows }, + expandGroup: (groupId: string) => { + groupDisplayState.expandGroup(groupId) + }, updateMock: (detail) => { onUpdateMock?.(detail) }, @@ -585,17 +648,37 @@ return } - // console.log('compute') + const graphNodeDeps = Object.values(graph.nodes).map((n) => ({ + id: n.id, + parentIds: n.parentIds, + data: { assets: (n.data as any).assets, module: (n.data as any).module } + })) + currentGraphNodeDeps = graphNodeDeps - let layoutedNodes = layoutNodes( - Object.values(graph.nodes).map((n) => ({ - id: n.id, - parentIds: n.parentIds, - data: { assets: (n.data as any).assets } - })) - ) - let newNodes: (Node & NodeLayout)[] = layoutedNodes.map((n) => ({ ...n, ...graph.nodes[n.id] })) + // Pre-compute extra space per node for assets, AI tools, group notes, group headers + const nodeExtraSpace = computeNodeExtraSpace(graphNodeDeps, { + showAssets: $showAssets ?? true, + showNotes, + notes, + noteTextHeights, + groupDisplayState, + insertable, + flowModuleStates + }) + // Layout with extra space baked into sugiyama + let layoutedNodes = layoutNodes(graphNodeDeps, nodeExtraSpace) + let newNodes: (Node & NodeLayout)[] = layoutedNodes.map((n) => { + const merged = { ...n, ...graph.nodes[n.id] } + // Augment group head nodes with wrapper dimensions from compound layout + if (graph.nodes[n.id]?.type === 'groupHead' && lastGroupDimensions?.has(n.id)) { + const dims = lastGroupDimensions.get(n.id)! + merged.data = { ...merged.data, wrapperWidth: dims.width, wrapperHeight: dims.height } + } + return merged + }) + + // Compute asset visual nodes (no position remapping) let assetNodesResult = $showAssets ? computeAssetNodes( newNodes.map((n) => ({ @@ -605,25 +688,17 @@ })) ) : undefined - if (assetNodesResult) { - newNodes = newNodes.map((n) => ({ - ...n, - position: assetNodesResult.newNodePositions[n.id] - })) - } - let aiToolNodesResult = computeAIToolNodes(newNodes, eventHandler, insertable, flowModuleStates) - let nodesAfterAITools = newNodes.map((n) => ({ - ...n, - position: aiToolNodesResult.newNodePositions[n.id] - })) - let finalNodes = [ - ...nodesAfterAITools, + // Compute AI tool visual nodes (no position remapping) + let aiToolNodesResult = computeAIToolNodes(newNodes, eventHandler, insertable, flowModuleStates) + + let finalNodes: (Node & NodeLayout)[] = [ + ...newNodes, ...(assetNodesResult?.newAssetNodes ?? []), ...aiToolNodesResult.toolNodes ] - // Compute note nodes and positions + // Compute note nodes (no position remapping) let noteNodesResult = showNotes ? computeNoteNodes( finalNodes.map((n) => ({ @@ -644,14 +719,6 @@ ) : undefined - // Apply note positioning to nodes if notes are enabled - if (noteNodesResult) { - finalNodes = finalNodes.map((n) => ({ - ...n, - position: noteNodesResult.newNodePositions[n.id] || n.position - })) - } - // update nodes nodes = [...finalNodes, ...(noteNodesResult?.noteNodes ?? [])] @@ -699,7 +766,10 @@ assetsOverflowed: AssetsOverflowedNode, aiTool: AiToolNode, newAiTool: NewAiToolNode, - note: NoteNode + note: NoteNode, + collapsedGroup: CollapsedGroupNode, + groupHead: GroupHeadNode, + groupEnd: GroupEndNode } as any const edgeTypes = { @@ -735,7 +805,41 @@ let graph = $derived.by(() => { moduleTracker.counter effectiveModuleActions - return graphBuilder( + currentGroups + + const collapsedGroupIds = new Set( + allGroups + .filter((g) => groupDisplayState.isRuntimeCollapsed(groupKey(g))) + .map((g) => groupKey(g)) + ) + + if (groupError) { + return { nodes: {}, edges: [], error: groupError } + } + + // Use provided structure tree (from proxy) or build locally (diff mode / read-only) + let gm: FlowStructureNode[] | undefined = groupedModulesProp + if (!gm) { + const allGroups = groups ?? [] + const graphGroups = allGroups.map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: untrack(() => + computeGroupModuleIds(g.start_id, g.end_id, getAllModules(effectiveModules ?? [])) + ) + })) + try { + gm = buildStructureTree( + stateSnapshot(untrack(() => effectiveModules) ?? []) as FlowModule[], + graphGroups + ) + } catch (e) { + return { nodes: {}, edges: [], error: e } + } + } + + const result = graphBuilder( + gm, untrack(() => effectiveModules), { disableAi, @@ -767,16 +871,43 @@ untrack(() => selectedId), simplifiableFlow, triggerNode ? path : undefined, - expandedSubflows + expandedSubflows, + showNotes, + collapsedGroupIds ) + return { ...result, structureTree: gm } }) let hideAssetsToggle = $derived( $showAssets && Object.values(nodes).every((n) => n.type !== 'asset') ) - let hideNotesToggle = $derived(!notes || notes.length === 0) + let hideNotesToggle = $derived( + (!notes || notes.length === 0) && !(groups ?? []).some((g) => g.note != null) + ) + + let currentGroupDepths = $derived( + 'structureTree' in graph && graph.structureTree ? computeGroupDepths(graph.structureTree) : {} + ) + + // All groups including those from expanded subflows (for overlay rendering) + let allGroups = $derived.by(() => { + const base = groups ?? [] + const subflowGroups = Object.values(expandedSubflows).flatMap((sf) => sf.groups ?? []) + return subflowGroups.length > 0 ? [...base, ...subflowGroups] : base + }) + + // Track groups for re-layout when groups change + let currentGroups = $derived(groups ?? []) $effect(() => { - ;[graph, allowSimplifiedPoll, $showAssets, showNotes, noteManager.renderCount] + ;[ + graph, + allowSimplifiedPoll, + $showAssets, + showNotes, + noteManager.renderCount, + currentGroups, + groupDisplayState.renderCount + ] untrack(async () => { await updateStores() }) @@ -893,6 +1024,16 @@ } } + export function createGroupFromSelection(ids: string[]) { + if (groupEditorCtx?.groupEditor) { + groupEditorCtx.groupEditor.createGroup(ids, currentGraphNodeDeps) + tick().then(() => { + clearFlowSelection() + selectionManager.clearSelection() + }) + } + } + const modifierKey = isMac() ? 'Meta' : 'Control' @@ -909,7 +1050,7 @@ bind:this={flowContainer} > {#if graph?.error} -
    +
    {graph.error} @@ -1008,6 +1149,12 @@ /> {/if} + + @@ -1065,7 +1212,7 @@ try { localStorage.setItem( 'svelvet', - encodeState({ modules, failureModule, preprocessorModule, notes }) + encodeState({ modules, failureModule, preprocessorModule, notes, groups }) ) } catch (e) { console.error('error interacting with local storage', e) diff --git a/frontend/src/lib/components/graph/GroupActionBar.svelte b/frontend/src/lib/components/graph/GroupActionBar.svelte new file mode 100644 index 0000000000..1ae30395b5 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupActionBar.svelte @@ -0,0 +1,158 @@ + + +
    + {#if moveManager && moveModuleId} + moveManager.toggleMoving(moveModuleId!)} + /> + {/if} + {#if note == null} + + {/if} + + {#snippet buttonReplacement()} + + {/snippet} + {#snippet menu()} +
    + +
    +
    + {#each Object.values(NoteColor) as c (c)} + + {/each} +
    +
    + + +
    + onUpdateAutocollapse(e.detail)} + /> +
    + +
    + + + + + {#if onDeleteGroup} +
    + + + + {/if} +
    + {/snippet} +     +
    diff --git a/frontend/src/lib/components/graph/GroupHeader.svelte b/frontend/src/lib/components/graph/GroupHeader.svelte new file mode 100644 index 0000000000..474d71c1a4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupHeader.svelte @@ -0,0 +1,116 @@ + + + + +
    {}))} + title={collapsed ? 'Expand group' : 'Collapse group'} +> +
    + +
    +
    + {#if editingSummary} +
    + +
    + {:else} + {})) : undefined} + >{summary || PLACEHOLDER} + {/if} +
    +
    + + diff --git a/frontend/src/lib/components/graph/GroupHeaderBlock.svelte b/frontend/src/lib/components/graph/GroupHeaderBlock.svelte new file mode 100644 index 0000000000..089c0642a4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupHeaderBlock.svelte @@ -0,0 +1,76 @@ + + + +
    (hovered = true)} + onmouseleave={() => (hovered = false)} +> + graphContext?.groupDisplayState?.toggleRuntimeCollapse(groupId)} + onSummaryUpdate={(text) => groupEditorContext?.groupEditor.updateSummary(groupId, text)} + /> + {#if showNotes && note != null} + graphContext?.groupDisplayState?.setNoteHeight(groupId, h)} + onNoteUpdate={(text) => groupEditorContext?.groupEditor.updateNote(groupId, text)} + /> + {/if} + {#if editMode} + (menuOpen = open)} + onAddNote={() => groupEditorContext?.groupEditor.addNote(groupId)} + onRemoveNote={() => groupEditorContext?.groupEditor.removeNote(groupId)} + onUpdateColor={(c) => groupEditorContext?.groupEditor.updateColor(groupId, c)} + onUpdateAutocollapse={(v) => groupEditorContext?.groupEditor.updateAutocollapse(groupId, v)} + onDeleteGroup={() => groupEditorContext?.groupEditor.deleteGroup(groupId)} + /> + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupModuleIcons.svelte b/frontend/src/lib/components/graph/GroupModuleIcons.svelte new file mode 100644 index 0000000000..3df1aecfaa --- /dev/null +++ b/frontend/src/lib/components/graph/GroupModuleIcons.svelte @@ -0,0 +1,182 @@ + + +
    + {#each displayModules as mod (mod.id)} + {@const selected = selectionManager.isNodeSelected(mod.id)} + {@const nodeState = flowModuleStates?.[mod.id]?.type} + {@const colorClasses = getNodeColorClasses(nodeState, selected)} + + {#snippet children()} + + +
    selectModule(mod)} + > +
    + +
    + {mod.id} +
    + {/snippet} + {#snippet text()} + {mod.id}: {moduleLabel(mod)} + {/snippet} +     + {/each} + {#if overflowModules.length > 0} + {@const overflowColorClasses = getNodeColorClasses(overflowAggregateState, false)} + + {#snippet buttonReplacement()} +
    + +{overflowModules.length} +
    + {/snippet} + {#snippet menu()} +
    + {#each overflowModules as mod (mod.id)} + {@const nodeState = flowModuleStates?.[mod.id]?.type} + {@const colorClasses = getNodeColorClasses(nodeState, false)} + {@const selected = selectionManager.isNodeSelected(mod.id)} + + +
    selectModule(mod)} + > +
    + +
    + {moduleLabel(mod)} + {mod.id} +
    + {/each} +
    + {/snippet} +     + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupNodeCard.svelte b/frontend/src/lib/components/graph/GroupNodeCard.svelte new file mode 100644 index 0000000000..dd69fcf0d0 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupNodeCard.svelte @@ -0,0 +1,165 @@ + + +
    +
    +
    + {#if modules && modules.length > 0} + + {:else} + + {/if} +
    + {#if editingSummary} + + {:else} + + + {})) : undefined} + >{summary || 'Group'} + {/if} +
    +
    + {#if stepCount != null} + + + {stepCount} node{stepCount !== 1 ? 's' : ''} + {/if} +
    + + {#if showNote} +
    + onHeightChange?.(h)} + onNoteUpdate={(text) => onNoteUpdate?.(text)} + /> +
    + {/if} +
    diff --git a/frontend/src/lib/components/graph/GroupNoteArea.svelte b/frontend/src/lib/components/graph/GroupNoteArea.svelte new file mode 100644 index 0000000000..1b4562e8d0 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupNoteArea.svelte @@ -0,0 +1,151 @@ + + + +
    +
    + {#if editing} +
    + + +
    + + {:else if note} + +
    {}) : undefined} + > + +
    + {:else} + +
    {}) : undefined} + > + Double click to edit the note +
    + {/if} +
    +
    diff --git a/frontend/src/lib/components/graph/GroupOverlay.svelte b/frontend/src/lib/components/graph/GroupOverlay.svelte new file mode 100644 index 0000000000..c15095bfb4 --- /dev/null +++ b/frontend/src/lib/components/graph/GroupOverlay.svelte @@ -0,0 +1,90 @@ + + +{#each groups as group (groupKey(group))} + {@const bounds = groupBoundsMap[groupKey(group)]} + {#if bounds} + +
    +     + {/if} +{/each} diff --git a/frontend/src/lib/components/graph/MiniFlowGraph.svelte b/frontend/src/lib/components/graph/MiniFlowGraph.svelte index fdb73745df..ab0c5042e8 100644 --- a/frontend/src/lib/components/graph/MiniFlowGraph.svelte +++ b/frontend/src/lib/components/graph/MiniFlowGraph.svelte @@ -1,6 +1,12 @@ - -{#if noteEditorContext?.noteEditor && selectedNodeIds.length > 1} - - {@render children()} - -{/if} diff --git a/frontend/src/lib/components/graph/NoteColorPicker.svelte b/frontend/src/lib/components/graph/NoteColorPicker.svelte index d17adb9317..ff3a1a276c 100644 --- a/frontend/src/lib/components/graph/NoteColorPicker.svelte +++ b/frontend/src/lib/components/graph/NoteColorPicker.svelte @@ -10,7 +10,11 @@ isOpen?: boolean } - let { selectedColor, onColorChange, isOpen = $bindable(false) }: Props = $props() + let { + selectedColor, + onColorChange, + isOpen = $bindable(false) + }: Props = $props() import { ViewportPortal, type Node } from '@xyflow/svelte' import { calculateNodesBoundsWithOffset } from './util' - import { StickyNote, Move, Copy, Trash2, EllipsisVertical } from 'lucide-svelte' + import { Move, Copy, Trash2, EllipsisVertical, Group } from 'lucide-svelte' import { Button } from '../common' import DropdownV2 from '../DropdownV2.svelte' - import { getNoteEditorContext } from './noteEditor.svelte' + import { getGroupEditorContext } from './groupEditor.svelte' import { getGraphContext } from './graphContext' import MoveHandleButton from './MoveHandleButton.svelte' import { tick } from 'svelte' @@ -36,18 +36,19 @@ let resolvedCount = $derived(resolvedModuleIds.length) - // Get NoteEditor context for group note creation - const noteEditorContext = getNoteEditorContext() + // Get GroupEditor context for group creation + const groupEditorContext = getGroupEditorContext() // Get Graph context for clearFlowSelection function and moveManager const graphContext = getGraphContext() const moveManager = graphContext?.moveManager - function handleAddGroupNote() { - if (selectedNodes.length > 0 && noteEditorContext?.noteEditor && graphContext) { - // Create the group note first - noteEditorContext.noteEditor.createGroupNote(selectedNodes) + let canCreateGroup = $derived(groupEditorContext?.canCreateGroup.val ?? false) + + function handleAddGroup() { + if (selectedNodes.length > 0 && groupEditorContext?.groupEditor && graphContext) { + const flowNodes = graphContext.getFlowNodes?.() ?? [] + groupEditorContext.groupEditor.createGroup(selectedNodes, flowNodes) - // Wait for next tick to ensure DOM updates tick().then(() => { graphContext?.clearFlowSelection?.() graphContext?.selectionManager.clearSelection() @@ -74,13 +75,13 @@ shortcut: isMac() ? '⌫' : 'Del', action: () => onDeleteSelected?.() }, - ...(noteEditorContext?.noteEditor + ...(groupEditorContext?.groupEditor ? [ { - displayName: 'Add note', - icon: StickyNote, - separatorTop: true, - action: handleAddGroupNote + displayName: 'Create group', + icon: Group, + action: handleAddGroup, + disabled: !canCreateGroup } ] : []) diff --git a/frontend/src/lib/components/graph/compoundLayout.ts b/frontend/src/lib/components/graph/compoundLayout.ts index 4bf3ddebda..c47aa62584 100644 --- a/frontend/src/lib/components/graph/compoundLayout.ts +++ b/frontend/src/lib/components/graph/compoundLayout.ts @@ -1,5 +1,6 @@ import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' import { NODE } from './util' +import { GROUP_HEADER_HEIGHT } from './groupEditor.svelte' type LayoutNode = { id: string @@ -14,7 +15,7 @@ type LayoutConstants = { } type CompoundGroup = { - type: 'branch' | 'loop' + type: 'branch' | 'loop' | 'group' headId: string endId: string branches: { @@ -27,9 +28,12 @@ type LayoutResult = { positions: Map bbox: { width: number; height: number } contentMinX: number + groupDimensions?: Map } const LOOP_INDENT = 25 +export const GROUP_PADDING = 16 +export const GROUP_TOP_PADDING = 32 /** * Detect compound groups from a flat list of node IDs. @@ -83,6 +87,18 @@ function detectGroups( endId: id, branches: [{ labelId: `${baseId}-start`, innerIds }] }) + } else if (baseId.startsWith('group:')) { + // Group pattern: group:{groupId} head + group:{groupId}-end + // Body is everything reachable from head to end + const innerIds = findInnerIds(baseId, id, nodeIds, childrenMap) + if (innerIds.length > 0) { + groups.push({ + type: 'group', + headId: baseId, + endId: id, + branches: [{ labelId: innerIds[0], innerIds: innerIds.slice(1) }] + }) + } } } @@ -230,6 +246,29 @@ function runSugiyama( * 5. Run sugiyama on the simplified graph * 6. Expand wrapper positions back to absolute positions */ +/** + * Build nodeSizes map for sugiyama from nodeExtraSpace. + * Each node's effective height = top + NODE.height + bottom. + */ +function buildNodeSizes( + nodeIds: string[], + constants: LayoutConstants, + nodeExtraSpace?: Map +): Map | undefined { + if (!nodeExtraSpace || nodeExtraSpace.size === 0) return undefined + const sizes = new Map() + for (const id of nodeIds) { + const extra = nodeExtraSpace.get(id) + if (extra && (extra.top > 0 || extra.bottom > 0 || extra.left > 0 || extra.right > 0)) { + sizes.set(id, { + width: constants.nodeWidth + extra.left + extra.right, + height: constants.nodeHeight + extra.top + extra.bottom + }) + } + } + return sizes.size > 0 ? sizes : undefined +} + const MAX_RECURSION_DEPTH = 50 function layoutLevel( @@ -237,7 +276,8 @@ function layoutLevel( allNodes: Map, constants: LayoutConstants, childrenMap: Map, - depth: number = 0 + depth: number = 0, + nodeExtraSpace?: Map ): LayoutResult { const positions = new Map() const nodeIdSet = new Set(nodeIds) @@ -256,8 +296,15 @@ function layoutLevel( const n = allNodes.get(id)! return { id, parentIds: (n.parentIds ?? []).filter((pid) => nodeIdSet.has(pid)) } }) - const result = runSugiyama(flatNodes, constants) + const extraSizes = buildNodeSizes( + flatNodes.map((n) => n.id), + constants, + nodeExtraSpace + ) + const result = runSugiyama(flatNodes, constants, extraSizes) for (const [id, pos] of result.positions) { + const extra = nodeExtraSpace?.get(id) + if (extra) pos.y += extra.top positions.set(id, pos) } return { positions, bbox: { width: result.width, height: result.height }, contentMinX: 0 } @@ -322,7 +369,14 @@ function layoutLevel( const branchNodeIds = [branch.labelId, ...branch.innerIds] // Find sub-groups within this branch - const result = layoutLevel(branchNodeIds, allNodes, constants, childrenMap, depth + 1) + const result = layoutLevel( + branchNodeIds, + allNodes, + constants, + childrenMap, + depth + 1, + nodeExtraSpace + ) branchLayouts.push({ labelId: branch.labelId, @@ -349,6 +403,16 @@ function layoutLevel( maxBranchHeight = Math.max(0, ...branchLayouts.map((bl) => bl.bbox.height)) // head row + branch content + end row wrapperHeight = rowHeight + maxBranchHeight + rowHeight + } else if (group.type === 'group') { + // Group: body is centered with padding on all sides + const bodyWidth = branchLayouts[0]?.bbox.width ?? constants.nodeWidth + const bodyHeight = branchLayouts[0]?.bbox.height ?? 0 + wrapperWidth = Math.max(bodyWidth + GROUP_PADDING * 2, constants.nodeWidth) + maxBranchHeight = bodyHeight + const headExtra = nodeExtraSpace?.get(group.headId) + const groupHeadRow = GROUP_HEADER_HEIGHT + (headExtra?.bottom ?? 0) + GROUP_TOP_PADDING + // head row + body + bottom padding + wrapperHeight = groupHeadRow + bodyHeight + GROUP_PADDING } else { // Loop: body is indented const bodyWidth = branchLayouts[0]?.bbox.width ?? constants.nodeWidth @@ -395,13 +459,30 @@ function layoutLevel( } // Step 5: Run sugiyama on flattened nodes - const sugResult = runSugiyama(flatNodes, constants, wrapperSizes) + // Merge wrapperSizes with nodeExtraSpace-derived sizes for non-group nodes + const extraSizes = buildNodeSizes( + flatNodes.map((n) => n.id), + constants, + nodeExtraSpace + ) + const mergedSizes = new Map() + if (extraSizes) { + for (const [id, size] of extraSizes) mergedSizes.set(id, size) + } + for (const [id, size] of wrapperSizes) mergedSizes.set(id, size) + const sugResult = runSugiyama( + flatNodes, + constants, + mergedSizes.size > 0 ? mergedSizes : undefined + ) // Step 6: Resolve absolute positions // First, set positions for regular (non-group) nodes + // Apply per-node y-offset from nodeExtraSpace so decorations above have room for (const [nid, pos] of sugResult.positions) { if (groupByHeadId.has(nid)) continue // Handle groups separately - positions.set(nid, { x: pos.x, y: pos.y }) + const extra = nodeExtraSpace?.get(nid) + positions.set(nid, { x: pos.x, y: pos.y + (extra?.top ?? 0) }) } // Now expand group wrappers into absolute positions @@ -411,9 +492,15 @@ function layoutLevel( const rowHeight = constants.nodeHeight + constants.gapV const isBranch = gl.group.type === 'branch' + const isGroup = gl.group.type === 'group' // Position the head node at the top-center of the wrapper - positions.set(headId, { x: wrapperPos.x, y: wrapperPos.y }) + // Apply extra top padding so decorations above the head node have room + const headExtra = nodeExtraSpace?.get(headId) + positions.set(headId, { + x: wrapperPos.x, + y: wrapperPos.y + (headExtra?.top ?? 0) + }) if (isBranch) { // Reuse cached branchWidths and totalWidth @@ -441,6 +528,26 @@ function layoutLevel( x: wrapperPos.x, y: wrapperPos.y + rowHeight + maxBranchHeight + constants.gapV }) + } else if (isGroup) { + // Group: body is centered within wrapper (no x offset) + const headExtra = nodeExtraSpace?.get(gl.group.headId) + const groupHeadRow = GROUP_HEADER_HEIGHT + (headExtra?.bottom ?? 0) + GROUP_TOP_PADDING + const bl = gl.branchLayouts[0] + if (bl) { + for (const [innerNodeId, innerPos] of bl.result.positions) { + positions.set(innerNodeId, { + x: wrapperPos.x + innerPos.x, + y: wrapperPos.y + groupHeadRow + innerPos.y + }) + } + } + + // Position end node below body + const bodyHeight = bl?.bbox.height ?? 0 + positions.set(gl.group.endId, { + x: wrapperPos.x, + y: wrapperPos.y + groupHeadRow + bodyHeight + GROUP_PADDING + }) } else { // Loop: position start, body, and end const bl = gl.branchLayouts[0] @@ -463,16 +570,34 @@ function layoutLevel( } } + // Collect group dimensions from this level and child layouts + const groupDimensions = new Map() + for (const [headId, gl] of groupLayouts) { + groupDimensions.set(headId, { width: gl.wrapperWidth, height: gl.wrapperHeight }) + // Propagate child groupDimensions from recursive branch layouts + for (const bl of gl.branchLayouts) { + if (bl.result.groupDimensions) { + for (const [childId, dims] of bl.result.groupDimensions) { + groupDimensions.set(childId, dims) + } + } + } + } + // Compute overall bbox (nodes + group wrapper extents) let minX = Infinity let maxX = -Infinity let minY = Infinity let maxY = -Infinity - for (const pos of positions.values()) { - minX = Math.min(minX, pos.x - constants.nodeWidth / 2) - maxX = Math.max(maxX, pos.x + constants.nodeWidth / 2) - minY = Math.min(minY, pos.y) - maxY = Math.max(maxY, pos.y + constants.nodeHeight) + for (const [nid, pos] of positions) { + // Group end nodes are zero-height markers — skip them + if (nid.startsWith('group:') && nid.endsWith('-end')) continue + const extra = nodeExtraSpace?.get(nid) + minX = Math.min(minX, pos.x - constants.nodeWidth / 2 - (extra?.left ?? 0)) + maxX = Math.max(maxX, pos.x + constants.nodeWidth / 2 + (extra?.right ?? 0)) + // Account for top decoration space above the node + minY = Math.min(minY, pos.y - (extra?.top ?? 0)) + maxY = Math.max(maxY, pos.y + constants.nodeHeight + (extra?.bottom ?? 0)) } // Account for group wrapper extents in bbox (e.g. LOOP_INDENT makes wrappers wider than nodes) for (const [headId, gl] of groupLayouts) { @@ -492,7 +617,12 @@ function layoutLevel( width: Math.max(bboxWidth, constants.nodeWidth), height: Math.max(bboxHeight, 0) } - return { positions, bbox: finalBbox, contentMinX } + return { + positions, + bbox: finalBbox, + contentMinX, + groupDimensions: groupDimensions.size > 0 ? groupDimensions : undefined + } } /** @@ -500,10 +630,16 @@ function layoutLevel( * * Takes the flat list of nodes and edges from graphBuilder and produces * absolute positions that account for compound structure (branches, loops). + * + * nodeExtraSpace: per-node top/bottom/left/right padding that should be allocated in layout. + * After layout, each node's y is shifted down by its top padding so decorations + * (assets, AI tools, group headers) have room above. Left/right padding widens the + * column allocated to the node so neighbors are pushed further away. */ export function compoundLayout( nodes: { id: string; parentIds?: string[] }[], - constants?: Partial + constants?: Partial, + nodeExtraSpace?: Map ): LayoutResult { const c: LayoutConstants = { nodeWidth: constants?.nodeWidth ?? NODE.width, @@ -528,7 +664,7 @@ export function compoundLayout( } const nodeIds = nodes.map((n) => n.id) - const result = layoutLevel(nodeIds, allNodes, c, childrenMap) + const result = layoutLevel(nodeIds, allNodes, c, childrenMap, 0, nodeExtraSpace) // Shift positions so minX=0 (left-aligned). // FlowGraphV2 centers with: xCenter = viewport/2 - bbox.width/2 diff --git a/frontend/src/lib/components/graph/flowStructure.test.ts b/frontend/src/lib/components/graph/flowStructure.test.ts new file mode 100644 index 0000000000..fce5071e80 --- /dev/null +++ b/frontend/src/lib/components/graph/flowStructure.test.ts @@ -0,0 +1,245 @@ +import { describe, it, expect, vi } from 'vitest' + +// Mock modules that transitively import CSS/Monaco +vi.mock('monaco-editor', () => ({})) +vi.mock('@xyflow/svelte', () => ({})) +vi.mock('./renderers/nodes/AssetNode.svelte', () => ({ + assetDisplaysAsOutputInFlowGraph: () => false +})) +vi.mock('../modulesTest.svelte', () => ({})) + +import type { GraphGroup } from './groupEditor.svelte' +import type { FlowModule } from '$lib/gen' +import { + buildStructureTree, + flattenStructureIds, + deriveGroupsFromStructure, + collectLeafIds, + findInStructure +} from './flowStructure' + +function makeModule(id: string): FlowModule { + return { + id, + value: { type: 'rawscript', content: '', language: 'python3' } as any + } as FlowModule +} + +function makeBranchAll(id: string, branchInnerIds: string[][]): FlowModule { + return { + id, + value: { + type: 'branchall', + branches: branchInnerIds.map((ids) => ({ modules: ids.map((iid) => makeModule(iid)) })) + } as any + } as FlowModule +} + +function makeForloop(id: string, innerIds: string[]): FlowModule { + return { + id, + value: { + type: 'forloopflow', + modules: innerIds.map((iid) => makeModule(iid)), + iterator: { type: 'javascript', expr: '' } + } as any + } as FlowModule +} + +function makeGroup( + id: string, + start_id: string, + end_id: string, + moduleIds: string[] = [] +): GraphGroup { + return { id, start_id, end_id, moduleIds } +} + +describe('buildStructureTree', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + + it('builds structure tree for a valid group', () => { + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const result = buildStructureTree(modules, groups) + // Should have a group node + the remaining leaf 'c' + expect(result).toHaveLength(2) + expect(result[0].kind).toBe('group') + expect(result[0].id).toBe('g1') + expect(result[0].branches[0].children).toHaveLength(2) + expect(result[1].kind).toBe('leaf') + expect(result[1].id).toBe('c') + }) + + it('throws on duplicate group IDs', () => { + const groups = [makeGroup('g1', 'a', 'a', ['a']), makeGroup('g1', 'b', 'c', ['b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/duplicate group id.*g1/i) + }) + + it('throws on inverted range (start_id after end_id)', () => { + const groups = [makeGroup('g1', 'c', 'a', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/inverted range/i) + }) + + it('throws on partially overlapping groups', () => { + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b']), makeGroup('g2', 'b', 'c', ['b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/overlap without nesting/i) + }) + + it('throws when group start_id is a virtual node (Input)', () => { + const groups = [makeGroup('g1', 'Input', 'b', ['a', 'b'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('throws when group end_id is a virtual node (Result)', () => { + const groups = [makeGroup('g1', 'a', 'Result', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('throws when group references Trigger', () => { + const groups = [makeGroup('g1', 'Trigger', 'c', ['a', 'b', 'c'])] + expect(() => buildStructureTree(modules, groups)).toThrow(/virtual node/i) + }) + + it('allows fully nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const result = buildStructureTree(mods, groups) + expect(result).toHaveLength(1) // outer group contains everything + expect(result[0].kind).toBe('group') + // Inner group should be nested + const outerChildren = result[0].branches[0].children + expect(outerChildren).toHaveLength(3) // a, inner-group, d + expect(outerChildren[1].kind).toBe('group') + expect(outerChildren[1].id).toBe('inner') + }) + + it('handles empty modules', () => { + const result = buildStructureTree([], []) + expect(result).toHaveLength(0) + }) + + it('handles container modules (forloop)', () => { + const mods = [makeForloop('loop', ['x', 'y']), makeModule('c')] + const result = buildStructureTree(mods, []) + expect(result).toHaveLength(2) + expect(result[0].kind).toBe('forloopflow') + expect(result[0].branches).toHaveLength(1) + expect(result[0].branches[0].children).toHaveLength(2) + expect(result[0].branches[0].children[0].id).toBe('x') + }) + + it('handles groups inside containers', () => { + const mods = [makeForloop('loop', ['x', 'y', 'z'])] + const groups = [makeGroup('g1', 'x', 'y', ['x', 'y'])] + const result = buildStructureTree(mods, groups) + expect(result).toHaveLength(1) + expect(result[0].kind).toBe('forloopflow') + const innerChildren = result[0].branches[0].children + expect(innerChildren).toHaveLength(2) // group + z + expect(innerChildren[0].kind).toBe('group') + expect(innerChildren[0].id).toBe('g1') + }) + + it('throws when group spans parallel branches (branchall)', () => { + const mods = [ + makeModule('a'), + makeBranchAll('ba', [ + ['x', 'y'], + ['p', 'q'] + ]), + makeModule('c') + ] + const groups = [makeGroup('g1', 'x', 'q', ['x', 'q'])] + expect(() => buildStructureTree(mods, groups)).toThrow(/could not be resolved/) + }) +}) + +describe('flattenStructureIds', () => { + it('flattens a simple tree', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const ids = flattenStructureIds(tree) + expect(ids).toEqual(['a', 'b', 'c']) + }) + + it('flattens nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const tree = buildStructureTree(mods, groups) + const ids = flattenStructureIds(tree) + expect(ids).toEqual(['a', 'b', 'c', 'd']) + }) +}) + +describe('deriveGroupsFromStructure', () => { + it('derives group definitions with correct start/end', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const derived = deriveGroupsFromStructure(tree) + expect(derived).toHaveLength(1) + expect(derived[0].start_id).toBe('a') + expect(derived[0].end_id).toBe('b') + }) + + it('derives nested groups', () => { + const mods = [makeModule('a'), makeModule('b'), makeModule('c'), makeModule('d')] + const groups = [ + makeGroup('outer', 'a', 'd', ['a', 'b', 'c', 'd']), + makeGroup('inner', 'b', 'c', ['b', 'c']) + ] + const tree = buildStructureTree(mods, groups) + const derived = deriveGroupsFromStructure(tree) + expect(derived).toHaveLength(2) + expect(derived[0].start_id).toBe('a') + expect(derived[0].end_id).toBe('d') + expect(derived[1].start_id).toBe('b') + expect(derived[1].end_id).toBe('c') + }) +}) + +describe('findInStructure', () => { + it('finds a leaf node', () => { + const modules = [makeModule('a'), makeModule('b')] + const tree = buildStructureTree(modules, []) + const found = findInStructure(tree, 'b') + expect(found).toBeDefined() + expect(found!.index).toBe(1) + }) + + it('finds a node inside a group', () => { + const modules = [makeModule('a'), makeModule('b'), makeModule('c')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const found = findInStructure(tree, 'b') + expect(found).toBeDefined() + expect(found!.index).toBe(1) + // parentChildren should be the group's branch children + expect(found!.parentChildren).toHaveLength(2) + }) + + it('finds a group node by group id', () => { + const modules = [makeModule('a'), makeModule('b')] + const groups = [makeGroup('g1', 'a', 'b', ['a', 'b'])] + const tree = buildStructureTree(modules, groups) + const found = findInStructure(tree, 'g1') + expect(found).toBeDefined() + expect(found!.index).toBe(0) + }) +}) + +describe('collectLeafIds', () => { + it('collects all leaf module IDs including inside containers', () => { + const mods = [makeForloop('loop', ['x', 'y']), makeModule('c')] + const tree = buildStructureTree(mods, []) + const ids = collectLeafIds(tree) + expect(ids).toEqual(['loop', 'x', 'y', 'c']) + }) +}) diff --git a/frontend/src/lib/components/graph/flowStructure.ts b/frontend/src/lib/components/graph/flowStructure.ts new file mode 100644 index 0000000000..e2a725bc85 --- /dev/null +++ b/frontend/src/lib/components/graph/flowStructure.ts @@ -0,0 +1,498 @@ +import type { FlowModule } from '$lib/gen' + +import type { FlowGroup, GraphGroup } from './groupEditor.svelte' +import { getContainerInnerArrays } from './groupEditor.svelte' +import { VIRTUAL_NODE_IDS } from './groupDetectionUtils' + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type ContainerKind = 'forloopflow' | 'whileloopflow' | 'branchone' | 'branchall' + +export type StructureBranch = { + label?: string + children: FlowStructureNode[] +} + +export type FlowStructureNode = { + /** FlowModule.id for modules, groupKey(g) for groups */ + id: string + kind: 'leaf' | 'group' | ContainerKind + /** Only present when kind === 'group' */ + group?: FlowGroup + /** Only present when kind === 'group' — flat module IDs for step count */ + moduleIds?: string[] + /** Child branches. leaf=[], group=[{children}], container=[{children}, ...] */ + branches: StructureBranch[] +} + +// --------------------------------------------------------------------------- +// Type guards +// --------------------------------------------------------------------------- +// Building the structure tree +// --------------------------------------------------------------------------- + +export function buildStructureTree( + modules: FlowModule[], + groups: GraphGroup[] +): FlowStructureNode[] { + const { items, consumed } = buildStructureTreeRecurse(modules, groups) + const unconsumed = groups.filter((g) => !consumed.has(g.id)) + if (unconsumed.length > 0) { + throw new Error( + `Group(s) ${unconsumed.map((g) => `'${g.id}'`).join(', ')} could not be resolved: ` + + `their start/end nodes do not belong to the same branch` + ) + } + return items +} + +export function moduleToStructureNode(mod: FlowModule): FlowStructureNode { + const innerArrays = getContainerInnerArrays(mod) + if (innerArrays.length === 0) { + return { id: mod.id, kind: 'leaf', branches: [] } + } + + const kind = (mod.value as any).type as ContainerKind + const branches: StructureBranch[] = innerArrays.map(({ get, label }) => ({ + label, + children: [] // filled later by recursion + })) + + return { id: mod.id, kind, branches } +} + +function buildStructureTreeRecurse( + modules: FlowModule[], + groups: GraphGroup[] +): { items: FlowStructureNode[]; consumed: Set } { + if (modules.length === 0) { + return { items: [], consumed: new Set() } + } + + const indexMap = new Map() + for (let i = 0; i < modules.length; i++) { + indexMap.set(modules[i].id, i) + } + + // Reject duplicate group IDs + const seenGroupIds = new Set() + for (const g of groups) { + if (seenGroupIds.has(g.id)) { + throw new Error(`Duplicate group id: '${g.id}'`) + } + seenGroupIds.add(g.id) + } + + // Reject groups referencing virtual nodes + for (const g of groups) { + if (VIRTUAL_NODE_IDS.has(g.start_id) || VIRTUAL_NODE_IDS.has(g.end_id)) { + throw new Error( + `Group '${g.id}' references virtual node: groups cannot include Input, Result, or Trigger` + ) + } + } + + // Partition: groups for this level vs rest + const levelGroups: GraphGroup[] = [] + const otherGroups: GraphGroup[] = [] + for (const g of groups) { + if (indexMap.has(g.start_id) && indexMap.has(g.end_id)) { + const s = indexMap.get(g.start_id)! + const e = indexMap.get(g.end_id)! + if (s > e) { + throw new Error( + `Group '${g.id}' has inverted range: start_id='${g.start_id}' (index ${s}) > end_id='${g.end_id}' (index ${e})` + ) + } + levelGroups.push(g) + } else { + otherGroups.push(g) + } + } + + // Validate no partial overlaps + for (let i = 0; i < levelGroups.length; i++) { + for (let j = i + 1; j < levelGroups.length; j++) { + const a = levelGroups[i] + const b = levelGroups[j] + const aStart = indexMap.get(a.start_id)! + const aEnd = indexMap.get(a.end_id)! + const bStart = indexMap.get(b.start_id)! + const bEnd = indexMap.get(b.end_id)! + + if (aEnd < bStart || bEnd < aStart) continue + if (aStart <= bStart && bEnd <= aEnd) continue + if (bStart <= aStart && aEnd <= bEnd) continue + + throw new Error(`Groups '${a.id}' and '${b.id}' overlap without nesting`) + } + } + + // Build grouped structure for this level + function build( + startIdx: number, + endIdx: number, + availableGroups: GraphGroup[] + ): FlowStructureNode[] { + const result: FlowStructureNode[] = [] + let i = startIdx + while (i <= endIdx) { + const candidates = availableGroups.filter((g) => { + const gStart = indexMap.get(g.start_id)! + const gEnd = indexMap.get(g.end_id)! + return gStart === i && gEnd <= endIdx + }) + candidates.sort((a, b) => { + const spanA = indexMap.get(a.end_id)! - indexMap.get(a.start_id)! + const spanB = indexMap.get(b.end_id)! - indexMap.get(b.start_id)! + return spanB - spanA + }) + + const group = candidates[0] + if (group) { + const gEnd = indexMap.get(group.end_id)! + const remaining = availableGroups.filter((g) => g.id !== group.id) + const innerNodes = build(i, gEnd, remaining) + + const moduleIds: string[] = [] + for (let k = i; k <= gEnd; k++) { + moduleIds.push(modules[k].id) + } + + result.push({ + id: group.id, + kind: 'group', + group: { + summary: group.summary, + note: group.note, + color: group.color, + autocollapse: group.autocollapse, + start_id: group.start_id, + end_id: group.end_id + }, + moduleIds, + branches: [{ children: innerNodes }] + }) + i = gEnd + 1 + } else { + result.push(moduleToStructureNode(modules[i])) + i++ + } + } + return result + } + + const result = build(0, modules.length - 1, levelGroups) + + // Recurse into containers with remaining unconsumed groups + const consumed = new Set(levelGroups.map((g) => g.id)) + let remaining = otherGroups + + function recurseIntoContainers(items: FlowStructureNode[]): void { + for (const item of items) { + if (item.kind === 'group') { + recurseIntoContainers(item.branches[0].children) + continue + } + if (item.branches.length === 0) continue + + // This is a container module — get inner FlowModule arrays and recurse + const modIdx = indexMap.get(item.id) + if (modIdx === undefined) continue + const mod = modules[modIdx] + + const innerArrays = getContainerInnerArrays(mod) + for (let bi = 0; bi < innerArrays.length; bi++) { + const inner = buildStructureTreeRecurse(innerArrays[bi].get(), remaining) + item.branches[bi] = { + label: item.branches[bi]?.label, + children: inner.items + } + for (const id of inner.consumed) consumed.add(id) + remaining = remaining.filter((g) => !inner.consumed.has(g.id)) + } + } + } + recurseIntoContainers(result) + + return { items: result, consumed } +} + +// --------------------------------------------------------------------------- +// Traversal utilities +// --------------------------------------------------------------------------- + +/** Generic DFS over the structure tree */ +export function dfsStructure( + nodes: FlowStructureNode[], + fn: (node: FlowStructureNode, parentArray: FlowStructureNode[]) => void +): void { + for (const node of nodes) { + fn(node, nodes) + for (const branch of node.branches) { + dfsStructure(branch.children, fn) + } + } +} + +/** Flatten to ordered module IDs (groups are transparent) */ +export function flattenStructureIds(nodes: FlowStructureNode[]): string[] { + const ids: string[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + ids.push(...flattenStructureIds(node.branches[0].children)) + } else { + ids.push(node.id) + } + } + return ids +} + +/** Collect leaf module IDs recursively (including inside containers) */ +export function collectLeafIds(nodes: FlowStructureNode[]): string[] { + const ids: string[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + ids.push(...collectLeafIds(node.branches[0].children)) + } else { + ids.push(node.id) + for (const branch of node.branches) { + ids.push(...collectLeafIds(branch.children)) + } + } + } + return ids +} + +// --------------------------------------------------------------------------- +// Finding nodes in the tree +// --------------------------------------------------------------------------- + +export type FindResult = { parentChildren: FlowStructureNode[]; index: number } + +export function findInStructure(nodes: FlowStructureNode[], id: string): FindResult | undefined { + for (let i = 0; i < nodes.length; i++) { + const node = nodes[i] + if (node.id === id) return { parentChildren: nodes, index: i } + for (const branch of node.branches) { + const found = findInStructure(branch.children, id) + if (found) return found + } + } + return undefined +} + +/** + * Match a structure node against a graph node ID. + * Handles group head/end IDs (group:X, group:X-end) and collapsed-group:X. + */ +export function matchStructureNode(node: FlowStructureNode, nodeId: string): boolean { + if (node.id === nodeId) return true + if (node.kind === 'group') { + return ( + nodeId === `group:${node.id}` || + nodeId === `group:${node.id}-end` || + nodeId === `collapsed-group:${node.id}` + ) + } + return false +} + +/** + * Find insert index using graph node IDs (handles group:X-end etc.). + * Returns the index OF the matched item (insert before it). + * For group-end nodes, returns index AFTER the group (insert after it). + */ +export function findInsertIndexByNodeId(items: FlowStructureNode[], targetNodeId: string): number { + // group-end: insert after the group + if (targetNodeId.startsWith('group:') && targetNodeId.endsWith('-end')) { + const groupId = targetNodeId.slice('group:'.length, -'-end'.length) + const idx = items.findIndex((n) => n.kind === 'group' && n.id === groupId) + return idx >= 0 ? idx + 1 : items.length + } + // Everything else: insert at the matched item's position + for (let i = 0; i < items.length; i++) { + if (matchStructureNode(items[i], targetNodeId)) return i + } + return items.length +} + +// --------------------------------------------------------------------------- +// Deriving groups from the structure tree +// --------------------------------------------------------------------------- + +export function deriveGroupsFromStructure(nodes: FlowStructureNode[]): FlowGroup[] { + const groups: FlowGroup[] = [] + for (const node of nodes) { + if (node.kind === 'group' && node.group) { + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length === 0) { + console.warn(`deriveGroupsFromStructure: skipping empty group "${node.id}"`) + continue + } + groups.push({ + ...node.group, + start_id: flatIds[0], + end_id: flatIds[flatIds.length - 1] + }) + // Recurse for nested groups + groups.push(...deriveGroupsFromStructure(node.branches[0].children)) + } else { + for (const branch of node.branches) { + groups.push(...deriveGroupsFromStructure(branch.children)) + } + } + } + return groups +} + +// --------------------------------------------------------------------------- +// Syncing structure back to FlowModule[] +// --------------------------------------------------------------------------- + +/** + * Reconstruct a FlowModule[] from the structure tree, looking up originals + * from moduleMap and patching container inner arrays to match the tree ordering. + */ +export function applyStructureToModules( + nodes: FlowStructureNode[], + moduleMap: Map +): FlowModule[] { + const result: FlowModule[] = [] + for (const node of nodes) { + if (node.kind === 'group') { + // Groups are transparent — splice their children into this level + result.push(...applyStructureToModules(node.branches[0].children, moduleMap)) + } else { + const mod = moduleMap.get(node.id) + if (!mod) continue + + // Patch container inner arrays + if (node.branches.length > 0) { + const innerArrays = getContainerInnerArrays(mod) + for (let bi = 0; bi < innerArrays.length && bi < node.branches.length; bi++) { + innerArrays[bi].set(applyStructureToModules(node.branches[bi].children, moduleMap)) + } + } + + result.push(mod) + } + } + return result +} + +// --------------------------------------------------------------------------- +// Empty groups cleanup +// --------------------------------------------------------------------------- + +/** + * Walk the tree, remove group nodes that have no leaf modules, and return + * the removed groups. Mutates the input array in-place. + * Recurses depth-first so inner groups are cleaned before checking outer ones. + */ +export function removeEmptyGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const removed: FlowGroup[] = [] + for (let i = nodes.length - 1; i >= 0; i--) { + const node = nodes[i] + if (node.kind === 'group' && node.group) { + // Recurse first — inner groups may become empty too + removed.push(...removeEmptyGroups(node.branches[0].children)) + if (flattenStructureIds(node.branches[0].children).length === 0) { + removed.push(node.group) + nodes.splice(i, 1) + } + } else { + for (const branch of node.branches) { + removed.push(...removeEmptyGroups(branch.children)) + } + } + } + return removed +} + +/** Walk the structure tree to compute nesting depth for each group (O(n)). */ +export function computeGroupDepths(tree: FlowStructureNode[]): Record { + const depths: Record = {} + function walk(nodes: FlowStructureNode[], groupDepth: number): void { + for (const node of nodes) { + if (node.kind === 'group') { + depths[node.id] = groupDepth + for (const branch of node.branches) { + walk(branch.children, groupDepth + 1) + } + } else { + for (const branch of node.branches) { + walk(branch.children, groupDepth) + } + } + } + } + walk(tree, 0) + return depths +} + +/** + * Find duplicate groups in the structure tree (same start_id:end_id after mutation). + * Returns the groups that should be removed (keeps the first, removes subsequent duplicates). + */ +export function findDuplicateGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const duplicates: FlowGroup[] = [] + const seen = new Set() + + function walk(items: FlowStructureNode[]): void { + for (const node of items) { + if (node.kind === 'group' && node.group) { + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length > 0) { + const key = `${flatIds[0]}:${flatIds[flatIds.length - 1]}` + if (seen.has(key)) { + duplicates.push(node.group) + } else { + seen.add(key) + } + } + walk(node.branches[0].children) + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(nodes) + return duplicates +} + +/** Remove duplicate groups from the structure tree (keeps first occurrence). */ +export function removeDuplicateGroups(nodes: FlowStructureNode[]): FlowGroup[] { + const removed: FlowGroup[] = [] + const seen = new Set() + + function walk(items: FlowStructureNode[]): void { + for (let i = items.length - 1; i >= 0; i--) { + const node = items[i] + if (node.kind === 'group' && node.group) { + walk(node.branches[0].children) + const flatIds = flattenStructureIds(node.branches[0].children) + if (flatIds.length > 0) { + const key = `${flatIds[0]}:${flatIds[flatIds.length - 1]}` + if (seen.has(key)) { + // Replace group node with its children (ungroup) + removed.push(node.group) + items.splice(i, 1, ...node.branches[0].children) + } else { + seen.add(key) + } + } + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(nodes) + return removed +} diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index 192b5c0c34..4fd7641096 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -8,6 +8,14 @@ import { getFlowModuleAssets, type AssetWithAltAccessType } from '../assets/lib' import { assetDisplaysAsOutputInFlowGraph } from './renderers/nodes/AssetNode.svelte' import type { ModulesTestStates, ModuleTestState } from '../modulesTest.svelte' import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' +import { + type FlowStructureNode, + collectLeafIds, + findInsertIndexByNodeId, + buildStructureTree +} from './flowStructure' +import { groupKey, type FlowGroup } from './groupEditor.svelte' +import { computeGroupModuleIds } from './groupDetectionUtils' export type InsertKind = | 'script' @@ -62,6 +70,7 @@ export type GraphEventHandlers = { simplifyFlow: (b: boolean) => void expandSubflow: (id: string, path: string) => void minimizeSubflow: (id: string) => void + expandGroup: (groupId: string) => void updateMock: (detail: { mock: FlowModule['mock']; id: string }) => void testUpTo: (id: string) => void editInput: (moduleId: string, key: string) => void @@ -111,6 +120,9 @@ export type FlowNode = | AssetsOverflowedN | AiToolN | NewAiToolN + | CollapsedGroupN + | GroupHeadN + | GroupEndN export type InputN = { type: 'input2' @@ -316,6 +328,48 @@ export type NewAiToolN = { } } +export type CollapsedGroupN = { + type: 'collapsedGroup' + data: { + groupId: string + summary: string | undefined + note: string | undefined + color: string | undefined + autocollapse: boolean | undefined + stepCount: number + modules: FlowModule[] + flowModuleStates: Record | undefined + flowJob: Job | undefined + isOwner: boolean + suspendStatus: Record + showNotes: boolean + editMode: boolean + eventHandlers: GraphEventHandlers + } +} + +export type GroupHeadN = { + type: 'groupHead' + data: { + groupId: string + summary: string | undefined + note: string | undefined + color: string | undefined + autocollapse: boolean | undefined + editMode: boolean + showNotes: boolean + eventHandlers: GraphEventHandlers + wrapperWidth?: number + } +} + +export type GroupEndN = { + type: 'groupEnd' + data: { + groupId: string + } +} + export function topologicalSort( nodes: { id: string; parentIds?: string[] }[] ): { id: string; parentIds?: string[] }[] { @@ -336,22 +390,8 @@ export function topologicalSort( return result.reverse() } -// input2: InputNode, -// module: ModuleNode, -// branchAllStart: BranchAllStart, -// branchAllEnd: BranchAllEndNode, -// forLoopEnd: ForLoopEndNode, -// forLoopStart: ForLoopStartNode, -// result: ResultNode, -// whileLoopStart: ForLoopStartNode, -// whileLoopEnd: ForLoopEndNode, -// branchOneStart: BranchOneStart, -// branchOneEnd: BranchOneEndNode, -// subflowBound: SubflowBound, -// noBranch: NoBranchNode, -// trigger: TriggersNode - export function graphBuilder( + structureTree: FlowStructureNode[], modules: FlowModule[] | undefined, extra: { disableAi: boolean @@ -383,11 +423,9 @@ export function graphBuilder( selectedId: string | undefined, simplifiableFlow: SimplifiableFlow | undefined, flowPathForTriggerNode: string | undefined, - expandedSubflows: Record - // triggerProps?: { - // path?: string - // flowIsSimplifiable?: boolean - // } + expandedSubflows: Record, + showNotes: boolean, + collapsedGroupIds: Set ): { nodes: { [key: string]: NodeLayout } edges: Edge[] @@ -403,7 +441,13 @@ export function graphBuilder( const nodes: NodeLayout[] = [] const edges: Edge[] = [] - function addNode(module: FlowModule) { + // Lookup map from module ID to the original reactive FlowModule objects. + const moduleMap = new Map() + for (const m of getAllModules(modules, failureModule)) { + moduleMap.set(m.id, m) + } + + function addNode(module: FlowModule, extraData?: Record) { const duplicated = nodes.find((n) => n.id === module.id) if (duplicated) { console.log('Duplicated node detected: ', module, duplicated) @@ -424,7 +468,8 @@ export function graphBuilder( isOwner: extra.isOwner, flowJob: extra.flowJob, assets: getFlowModuleAssets(module, extra.additionalAssetsMap), - moduleAction: extra.moduleActions?.[module.id] + moduleAction: extra.moduleActions?.[module.id], + ...extraData }, type: 'module', selectable: true @@ -483,14 +528,20 @@ export function graphBuilder( customId?: string type?: string subModules?: FlowModule[] + currentItems?: FlowStructureNode[] disableMoveIds?: string[] } ) { parents[targetId] = [...(parents[targetId] ?? []), sourceId] - const mods = options?.subModules ?? modules - - let index = mods?.findIndex((m) => m.id === targetId) ?? -1 + let index: number + if (options?.currentItems) { + index = findInsertIndexByNodeId(options.currentItems, targetId) + } else { + const mods = options?.subModules ?? modules + const found = mods?.findIndex((m) => m.id === targetId) ?? -1 + index = found >= 0 ? found : (mods?.length ?? 0) + } const visited = new Set() const recStack = new Set() @@ -514,8 +565,7 @@ export function graphBuilder( simplifiedTriggerView: simplifiableFlow?.simplifiedFlow, disableMoveIds: options?.disableMoveIds, enableTrigger: sourceId === 'Input', - // If the index is -1, it means that the target module is not in the modules array, so we set it to the length of the array - index: index >= 0 ? index : (mods?.length ?? 0), + index, ...extra, insertable: extra.insertable && !options?.disableInsert && prefix == undefined, shouldOffsetInsertBtnDueToAssetNode: nodeIdsWithOutputAssets.has(sourceId) @@ -591,7 +641,7 @@ export function graphBuilder( } function processModules( - modules: FlowModule[], + items: FlowStructureNode[], branch: { rootId: string; branch: number } | undefined, beforeNode: NodeLayout, nextNode: NodeLayout | undefined, @@ -600,31 +650,166 @@ export function graphBuilder( disableMoveIds: string[] = [], parentIndex?: string ) { + // For subflow prefix rewriting, clone modules into moduleMap with prefixed IDs + // (avoid mutating reactive originals which would trigger state_unsafe_mutation in $derived) if (prefix != undefined) { - modules.forEach((m) => { - if (!m['oid']) { - m['oid'] = m.id + items.forEach((item) => { + if (item.kind === 'group') return + const m = moduleMap.get(item.id) + if (m) { + const oid = m['oid'] ?? m.id + const newId = 'subflow:' + prefix + oid + const clone = { ...m, id: newId, oid } as FlowModule & { oid: string } + clone['oid'] = oid + moduleMap.set(newId, clone) + item.id = newId } - m.id = 'subflow:' + prefix + m['oid'] }) } let previousId: string | undefined = undefined - if (modules.length === 0) { + if (items.length === 0) { if (nextNode) { addEdge(beforeNode.id, nextNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } } else { - modules.forEach((module, index) => { + items.forEach((item, index) => { + // --- Group items --- + if (item.kind === 'group') { + const g = item.group! + const gId = item.id + + if (collapsedGroupIds.has(gId)) { + // Collapsed group: single node + const nodeId = `collapsed-group:${gId}` + const leafIds = collectLeafIds(item.branches[0].children) + nodes.push({ + id: nodeId, + data: { + groupId: gId, + summary: g.summary, + note: g.note, + color: g.color, + autocollapse: g.autocollapse, + stepCount: item.moduleIds?.length ?? 0, + modules: leafIds + .map((id) => moduleMap.get(id)) + .filter((m): m is FlowModule => !!m), + flowModuleStates: extra.flowModuleStates, + flowJob: extra.flowJob, + isOwner: extra.isOwner, + suspendStatus: extra.suspendStatus, + showNotes, + editMode: prefix == undefined && extra.editMode, + eventHandlers + }, + type: 'collapsedGroup', + selectable: false + }) + + // Wire: previous → collapsedGroup + if (index > 0 && previousId) { + addEdge(previousId, nodeId, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + previousId = nodeId + } else { + // Expanded group: head → recurse → end + const headId = `group:${gId}` + const endId = `group:${gId}-end` + const localDisableMoveIds = [...disableMoveIds, headId] + + const headNode: NodeLayout = { + id: headId, + data: { + groupId: gId, + summary: g.summary, + note: g.note, + color: g.color, + autocollapse: g.autocollapse, + editMode: prefix == undefined && extra.editMode, + showNotes, + eventHandlers + }, + type: 'groupHead', + selectable: false + } + + const endNode: NodeLayout = { + id: endId, + data: { + groupId: gId + }, + type: 'groupEnd', + selectable: false + } + + nodes.push(headNode) + nodes.push(endNode) + + // Wire: previous → headNode + if (index > 0 && previousId) { + addEdge(previousId, headId, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + // Recurse inner modules + processModules( + item.branches[0].children, + { rootId: headId, branch: 0 }, + headNode, + endNode, + simplifiedTriggerView, + prefix, + localDisableMoveIds, + parentIndex + ) + + previousId = endId + } + + // Shared first/last edge wiring for groups + if (index === 0) { + addEdge( + beforeNode.id, + collapsedGroupIds.has(gId) ? `collapsed-group:${gId}` : `group:${gId}`, + undefined, + prefix, + { + currentItems: items, + disableMoveIds, + disableInsert: simplifiedTriggerView + } + ) + } + + if (index === items.length - 1 && previousId && nextNode) { + addEdge(previousId, nextNode.id, branch, prefix, { + currentItems: items, + disableMoveIds + }) + } + + return + } + + // --- Regular FlowModule items --- + const module = moduleMap.get(item.id) + if (!module) return const localDisableMoveIds = [...disableMoveIds, module.id] - // Add the edge between the previous node and the current one + // Inter-module edge: connect previous → current (expanded subflows handle their own) if (index > 0 && previousId && expandedSubflows[module.id] == undefined) { addEdge(previousId, module.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -700,7 +885,7 @@ export function graphBuilder( ) processModules( - branch.modules, + item.branches[branchIndex]?.children ?? [], { rootId: module.id, branch: branchIndex }, startNode, endNode, @@ -722,7 +907,7 @@ export function graphBuilder( id: `${module.id}-start`, data: { id: module.id, - module: module, + module: moduleMap.get(module.id) ?? module, simplifiedTriggerView, eventHandlers: eventHandlers, editMode: extra.editMode, @@ -759,7 +944,7 @@ export function graphBuilder( const selectedIterIndex = extra.flowModuleStates?.[module.id]?.selectedForloopIndex processModules( - module.value.modules, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, startNode, endNode, @@ -798,7 +983,7 @@ export function graphBuilder( const selectedIterIndex = extra.flowModuleStates?.[module.id]?.selectedForloopIndex processModules( - module.value.modules, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, startNode, endNode, @@ -825,21 +1010,6 @@ export function graphBuilder( } nodes.push(endNode) - // // Add default branch - // const defaultBranch: NodeLayout = { - // id: `${module.id}-default`, - // data: { - // offset: 0, - // label: 'Default', - // id: module.id, - // branchIndex: -1, - // eventHandlers: eventHandlers, - // branchOne: true, - // ...extra - // }, - // type: 'noBranch' - // } - const defaultBranch: NodeLayout = { id: `${module.id}-branch-default`, data: { @@ -863,7 +1033,7 @@ export function graphBuilder( }) processModules( - module.value.default, + item.branches[0]?.children ?? [], { rootId: module.id, branch: 0 }, defaultBranch, endNode, @@ -899,7 +1069,7 @@ export function graphBuilder( }) processModules( - branch.modules, + item.branches[branchIndex + 1]?.children ?? [], { rootId: module.id, branch: branchIndex + 1 }, startNode, endNode, @@ -912,9 +1082,9 @@ export function graphBuilder( previousId = endNode.id } else { - let expanded = expandedSubflows[module.id] - if (expanded) { - expanded = $state.snapshot(expanded) + const expandedData = expandedSubflows[module.id] + if (expandedData) { + const expandedMods = $state.snapshot(expandedData.modules) as FlowModule[] const startId = `${module.id}` const idWithoutPrefix = module.id.startsWith('subflow:') ? module.id.substring(8) @@ -936,12 +1106,12 @@ export function graphBuilder( if (previousId) { addEdge(previousId, startNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } else { addEdge(beforeNode.id, startNode.id, undefined, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -962,8 +1132,20 @@ export function graphBuilder( nodes.push(endNode) + // Register expanded subflow modules so prefix rewriting finds + // the inner modules (not the parent flow's modules with same IDs) + for (const em of getAllModules(expandedMods)) { + moduleMap.set(em.id, em) + } + + const expandedGroups = (expandedData.groups ?? []).map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: computeGroupModuleIds(g.start_id, g.end_id, getAllModules(expandedMods)) + })) + processModules( - expanded, + buildStructureTree(expandedMods, expandedGroups), undefined, startNode, endNode, @@ -981,15 +1163,15 @@ export function graphBuilder( if (index === 0 && expandedSubflows[module.id] == undefined) { addEdge(beforeNode.id, module.id, undefined, prefix, { - subModules: modules, + currentItems: items, disableMoveIds, disableInsert: simplifiedTriggerView }) } - if (index === modules.length - 1 && previousId && nextNode) { + if (index === items.length - 1 && previousId && nextNode) { addEdge(previousId, nextNode.id, branch, prefix, { - subModules: modules, + currentItems: items, disableMoveIds }) } @@ -997,10 +1179,12 @@ export function graphBuilder( } } + const topLevelItems = structureTree + if (simplifiableFlow?.simplifiedFlow === true && triggerNode) { - processModules(modules, undefined, triggerNode, undefined, true, undefined) + processModules(topLevelItems, undefined, triggerNode, undefined, true, undefined) } else { - processModules(modules, undefined, inputNode, resultNode, false, undefined) + processModules(topLevelItems, undefined, inputNode, resultNode, false, undefined) } if (failureModule) { diff --git a/frontend/src/lib/components/graph/graphContext.ts b/frontend/src/lib/components/graph/graphContext.ts index 8540982366..a6245769cf 100644 --- a/frontend/src/lib/components/graph/graphContext.ts +++ b/frontend/src/lib/components/graph/graphContext.ts @@ -4,6 +4,7 @@ import type { NoteManager } from './noteManager.svelte' import type { MoveManager } from './moveManager.svelte' import type { Writable } from 'svelte/store' import type { FlowDiffManager } from '../flows/flowDiffManager.svelte' +import type { GroupDisplayState } from './groupEditor.svelte' export type GraphContext = { selectionManager: SelectionManager @@ -14,6 +15,9 @@ export type GraphContext = { clearFlowSelection?: () => void yOffset?: number diffManager: FlowDiffManager + /** Current flow nodes for group validation (set by FlowGraphV2) */ + getFlowNodes?: () => { id: string; parentIds?: string[] }[] + groupDisplayState?: GroupDisplayState } const graphContextKey = 'FlowGraphContext' diff --git a/frontend/src/lib/components/graph/groupDetectionUtils.ts b/frontend/src/lib/components/graph/groupDetectionUtils.ts index e2c81c4d98..8dc6e3d8c2 100644 --- a/frontend/src/lib/components/graph/groupDetectionUtils.ts +++ b/frontend/src/lib/components/graph/groupDetectionUtils.ts @@ -1,7 +1,127 @@ +import { topologicalSort } from './graphBuilder.svelte' + +/** Node IDs synthesized by graphBuilder that are not real FlowModules */ +export const VIRTUAL_NODE_IDS = new Set(['Input', 'Result', 'Trigger']) + type FlowNode = { id: string; parentIds?: string[] } /** - * Use a simple algorithm to complete a group and split it into connected components + * Compute the set of module IDs that belong to a group defined by start_id and end_id. + * Uses the flattened module list (from getAllModules) and slices between start and end. + * Used for collapsed group icons, step count, and moduleToCollapsedGroup mapping. + */ +export function computeGroupModuleIds( + startId: string, + endId: string, + allModules: { id: string }[] +): string[] { + if (startId === endId) { + return allModules.some((m) => m.id === startId) ? [startId] : [] + } + + const startIdx = allModules.findIndex((m) => m.id === startId) + const endIdx = allModules.findIndex((m) => m.id === endId) + + if (startIdx === -1 || endIdx === -1 || startIdx > endIdx) { + if (startIdx > endIdx) { + console.warn( + `computeGroupModuleIds: inverted range for group ${startId}→${endId} (${startIdx} > ${endIdx})` + ) + } + return [] + } + + return allModules.slice(startIdx, endIdx + 1).map((m) => m.id) +} + +/** + * Check whether a set of selected node IDs can form a valid group. + * Normalizes marker IDs (branch/forloop) to parent module IDs, + * then uses topologicalSort to derive start and end boundaries. + */ +export function canFormValidGroup( + selectedIds: string[], + flowNodes: FlowNode[], + excludeIds?: Set +): { valid: true; startId: string; endId: string } | { valid: false } { + if (selectedIds.length === 0) return { valid: false } + + // Normalize marker IDs to parent module IDs. + // -start (forloop head) → parent ID. -end/-branch-* → skip if parent covered, else reject. + const rawSet = new Set(selectedIds) + const normalizedIds: string[] = [] + + for (const id of selectedIds) { + const parentId = id.replace(/-(end|start|branch-.*)$/, '') + if (parentId === id) { + normalizedIds.push(id) + continue + } + if (id.endsWith('-start')) { + normalizedIds.push(parentId) + continue + } + // -end or -branch-*: parent must be covered (directly or via -start) + if (!rawSet.has(parentId) && !rawSet.has(`${parentId}-start`)) { + return { valid: false } + } + } + + if (normalizedIds.length === 0) return { valid: false } + const normalizedSet = new Set(normalizedIds) + + // Topo sort full graph, filter to normalized selection. + // Include raw matches plus all markers (-start, -end, -branch-*) whose parent is selected. + const sorted = topologicalSort(flowNodes) + const selectedSorted = sorted.filter((n) => { + if (normalizedSet.has(n.id)) return true + const parentId = n.id.replace(/-(end|start|branch-.*)$/, '') + return parentId !== n.id && normalizedSet.has(parentId) + }) + + if (selectedSorted.length === 0) return { valid: false } + + // Reject virtual or excluded nodes + if (selectedSorted.some((n) => VIRTUAL_NODE_IDS.has(n.id) || excludeIds?.has(n.id))) { + return { valid: false } + } + + // Topo order is bottom-first: first = bottom (end), last = top (start). + // Use raw IDs for BFS traversal, normalize for the returned group boundaries. + const rawStartId = selectedSorted[selectedSorted.length - 1].id + const rawEndId = selectedSorted[0].id + const startId = rawStartId.replace(/-(end|start|branch-.*)$/, '') + const endId = rawEndId.replace(/-(end|start|branch-.*)$/, '') + + // Verify all selected nodes lie between start and end in the DAG. + // BFS backward from rawEndId to rawStartId to collect reachable nodes. + // Normalize collected IDs so container markers map to their parent module. + const between = new Set() + const queue = [rawEndId] + const visited = new Set() + const parentMap = new Map(flowNodes.map((n) => [n.id, n.parentIds ?? []])) + while (queue.length > 0) { + const cur = queue.shift()! + if (visited.has(cur)) continue + visited.add(cur) + const normalized = cur.replace(/-(end|start|branch-.*)$/, '') + between.add(cur) + between.add(normalized) + if (cur === rawStartId) continue + for (const p of parentMap.get(cur) ?? []) { + queue.push(p) + } + } + if (!normalizedIds.every((id) => between.has(id))) { + return { valid: false } + } + + return { valid: true, startId, endId } +} + +/** + * Legacy utility: complete a group and split it into connected components. + * Still used by NoteEditor for FlowNote group notes (contained_node_ids). */ export function completeAndSplitGroup(groupNodes: string[], flowNodes: FlowNode[]): string[][] { if (groupNodes.length <= 1) { diff --git a/frontend/src/lib/components/graph/groupEditor.svelte.ts b/frontend/src/lib/components/graph/groupEditor.svelte.ts new file mode 100644 index 0000000000..8405390b96 --- /dev/null +++ b/frontend/src/lib/components/graph/groupEditor.svelte.ts @@ -0,0 +1,325 @@ +import type { FlowModule } from '$lib/gen' +import type { StateStore } from '$lib/utils' +import type { ExtendedOpenFlow } from '../flows/types' + +import { canFormValidGroup } from './groupDetectionUtils' +import type { NoteColor } from './noteColors' +import { DEFAULT_GROUP_NOTE_COLOR, getNextAvailableColor } from './noteColors' +import { getContext, setContext } from 'svelte' + +/** + * Type for a flow group (matches the generated type from OpenAPI). + * Members are computed dynamically from all nodes on paths between start_id and end_id. + */ +export type FlowGroup = { + summary?: string + note?: string + autocollapse?: boolean + start_id: string + end_id: string + color?: string +} + +/** Derive a stable key from a group's boundaries. Used as ephemeral ID for graph nodes, runtime state, etc. */ +export function groupKey(g: { start_id: string; end_id: string }): string { + return `${g.start_id}:${g.end_id}` +} + +/** + * Display state for flow groups inside the graph. + * Handles runtime collapse state and note height tracking. + * Similar to NoteManager — instantiated inside FlowGraphV2. + */ +export class GroupDisplayState { + #getGroups: () => FlowGroup[] + #runtimeCollapsedIds = $state>(new Set()) + #runtimeInitialized = $state(false) + #noteHeights = $state>({}) + renderCount = $state(0) + + constructor(getGroups: () => FlowGroup[]) { + this.#getGroups = getGroups + } + + /** Initialize runtime state from autocollapse. Safe to call from event handlers. */ + private ensureRuntimeInitialized(): void { + if (this.#runtimeInitialized) return + const groups = this.#getGroups() + this.#runtimeCollapsedIds = new Set( + groups.filter((g) => g.autocollapse).map((g) => groupKey(g)) + ) + this.#runtimeInitialized = true + } + + /** Check if a group is currently collapsed (runtime). Safe to call from $derived. */ + isRuntimeCollapsed(groupId: string): boolean { + if (!this.#runtimeInitialized) { + return this.#getGroups().find((g) => groupKey(g) === groupId)?.autocollapse ?? false + } + return this.#runtimeCollapsedIds.has(groupId) + } + + /** Toggle runtime collapse (Minimize2 button) */ + toggleRuntimeCollapse(groupId: string): void { + this.ensureRuntimeInitialized() + const next = new Set(this.#runtimeCollapsedIds) + if (next.has(groupId)) next.delete(groupId) + else next.add(groupId) + this.#runtimeCollapsedIds = next + this.render() + } + + /** Expand a group at runtime (CollapsedGroupNode click) */ + expandGroup(groupId: string): void { + this.ensureRuntimeInitialized() + const next = new Set(this.#runtimeCollapsedIds) + next.delete(groupId) + this.#runtimeCollapsedIds = next + this.render() + } + + /** Set note height for a group (used for layout spacing) */ + setNoteHeight(groupId: string, height: number): void { + if (this.#noteHeights[groupId] !== height) { + this.#noteHeights[groupId] = height + this.render() + } + } + + /** Get all note heights */ + getNoteHeights(): Record { + return this.#noteHeights + } + + /** Bump render counter to trigger re-layout */ + render(): void { + this.renderCount++ + } + + /** Remap runtime state when a group's boundaries (and thus its key) change */ + remapGroupKey(oldKey: string, newKey: string): void { + if (this.#runtimeCollapsedIds.has(oldKey)) { + const next = new Set(this.#runtimeCollapsedIds) + next.delete(oldKey) + next.add(newKey) + this.#runtimeCollapsedIds = next + } + if (oldKey in this.#noteHeights) { + this.#noteHeights[newKey] = this.#noteHeights[oldKey] + delete this.#noteHeights[oldKey] + } + } + + /** Get currently collapsed groups for graph builder. Safe to call from $derived. */ + getCollapsedGroups(): FlowGroup[] { + if (!this.#runtimeInitialized) { + return this.#getGroups().filter((g) => g.autocollapse) + } + return this.#getGroups().filter((g) => this.#runtimeCollapsedIds.has(groupKey(g))) + } +} + +/** + * Utility class for editing flow groups via direct flowStore mutations. + * Follows the same pattern as NoteEditor. + */ +export class GroupEditor { + private flowStore: StateStore + + constructor(flowStore: StateStore) { + this.flowStore = flowStore + } + + getGroups(): FlowGroup[] { + return this.flowStore.val.value?.groups || [] + } + + private setGroups(groups: FlowGroup[]): void { + if (this.flowStore.val.value) { + this.flowStore.val.value.groups = groups + } + } + + /** IDs that cannot be part of a group (preprocessor, failure module) */ + getExcludeIds(): Set { + const excludeIds = new Set() + const pp = this.flowStore.val.value?.preprocessor_module?.id + if (pp) excludeIds.add(pp) + const fm = this.flowStore.val.value?.failure_module?.id + if (fm) excludeIds.add(fm) + return excludeIds + } + + /** Check whether the given selection can form a valid group */ + canCreateGroup( + selectedIds: string[], + flowNodes: { id: string; parentIds?: string[] }[] + ): boolean { + const result = canFormValidGroup(selectedIds, flowNodes, this.getExcludeIds()) + if (!result.valid) return false + // Reject if a group with the same boundaries already exists + return !this.getGroups().some((g) => g.start_id === result.startId && g.end_id === result.endId) + } + + /** + * Create a new group from selected node IDs. + * Uses canFormValidGroup to determine start_id and end_id. + * Returns the generated group ID. + */ + createGroup( + moduleIds: string[], + flowNodes: { id: string; parentIds?: string[] }[] + ): string | undefined { + // Filter subflow node IDs (same logic as NoteEditor.createGroupNote) + let filteredIds = [...moduleIds] + const subflowIds: string[] = [] + for (const id of moduleIds) { + if (id.startsWith('subflow:')) { + const match = id.match(/^subflow:([^:]+)/) + if (match) { + subflowIds.push(match[1]) + } + } + } + if (subflowIds.length > 0) { + filteredIds = filteredIds.filter((id) => !subflowIds.includes(id)) + filteredIds = [...filteredIds, ...subflowIds] + } + + const result = canFormValidGroup(filteredIds, flowNodes, this.getExcludeIds()) + if (!result.valid) return undefined + + const groups = this.getGroups() + + // Reject duplicate: a group with the same boundaries already exists + if (groups.some((g) => g.start_id === result.startId && g.end_id === result.endId)) { + return undefined + } + const usedColors = new Set() + for (const group of groups) { + if (group.color) { + usedColors.add(group.color as NoteColor) + } + } + const color = usedColors.size > 0 ? getNextAvailableColor(usedColors) : DEFAULT_GROUP_NOTE_COLOR + + const newGroup: FlowGroup = { + start_id: result.startId, + end_id: result.endId, + color + } + this.setGroups([...groups, newGroup]) + return groupKey(newGroup) + } + + deleteGroup(groupId: string): void { + const groups = this.getGroups() + this.setGroups(groups.filter((g) => groupKey(g) !== groupId)) + } + + updateColor(groupId: string, color: NoteColor): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, color } : g))) + } + + updateSummary(groupId: string, summary: string): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, summary } : g))) + } + + updateNote(groupId: string, note: string | undefined): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, note } : g))) + } + + /** Add a note to a group (sets note to empty string to trigger the placeholder UI) */ + addNote(groupId: string): void { + this.updateNote(groupId, '') + } + + /** Remove a note from a group */ + removeNote(groupId: string): void { + this.updateNote(groupId, undefined) + } + + updateAutocollapse(groupId: string, autocollapse: boolean): void { + const groups = this.getGroups() + this.setGroups(groups.map((g) => (groupKey(g) === groupId ? { ...g, autocollapse } : g))) + } +} + +export type GroupEditorContext = { + groupEditor: GroupEditor + canCreateGroup: StateStore +} + +const CONTEXT_KEY = 'GroupEditorContext' + +export function setGroupEditorContext( + groupEditor: GroupEditor, + canCreateGroup: StateStore +): void { + setContext(CONTEXT_KEY, { groupEditor, canCreateGroup }) +} + +export function getGroupEditorContext(): GroupEditorContext | undefined { + return getContext(CONTEXT_KEY) +} + +/** Height of the group header bar */ +export const GROUP_HEADER_HEIGHT = 22 + +/** Extra margin between the header and the first node */ +export const GROUP_TOP_MARGIN = 30 + +export type GraphGroup = FlowGroup & { + id: string + moduleIds: string[] +} + +export type ContainerInnerArray = { + get: () => FlowModule[] + set: (v: any) => void + label?: string +} + +/** Get inner arrays from a container FlowModule with direct get/set accessors. */ +export function getContainerInnerArrays(mod: FlowModule): ContainerInnerArray[] { + const val = mod.value as any + if (val.type === 'forloopflow' || val.type === 'whileloopflow') { + return [ + { + get: () => val.modules, + set: (v) => { + val.modules = v + } + } + ] + } else if (val.type === 'branchone') { + return [ + { + get: () => val.default, + set: (v) => { + val.default = v + }, + label: 'Default' + }, + ...val.branches.map((b: any, i: number) => ({ + get: () => b.modules, + set: (v: any) => { + b.modules = v + }, + label: b.summary || `Branch ${i + 1}` + })) + ] + } else if (val.type === 'branchall') { + return val.branches.map((b: any, i: number) => ({ + get: () => b.modules, + set: (v: any) => { + b.modules = v + }, + label: b.summary || `Branch ${i + 1}` + })) + } + return [] +} diff --git a/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts b/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts new file mode 100644 index 0000000000..6965baf531 --- /dev/null +++ b/frontend/src/lib/components/graph/groupedModulesProxy.svelte.ts @@ -0,0 +1,181 @@ +import { untrack } from 'svelte' +import type { FlowModule } from '$lib/gen' +import { type FlowGroup, type GraphGroup, groupKey } from './groupEditor.svelte' +import type { StateStore } from '$lib/utils' +import { getAllModules } from '../flows/flowExplorer' +import { computeGroupModuleIds } from './groupDetectionUtils' +import { stateSnapshot } from '$lib/svelte5Utils.svelte' +import { + buildStructureTree, + deriveGroupsFromStructure, + applyStructureToModules, + removeEmptyGroups, + findDuplicateGroups, + removeDuplicateGroups, + flattenStructureIds, + type FlowStructureNode +} from './flowStructure' + +export type ExtendedOpenFlow = { + value: { + modules: FlowModule[] + groups?: FlowGroup[] + [key: string]: any + } + [key: string]: any +} + +/** + * Reactive read-only view of the flow structure tree. + * The tree is always derived from flowStore (single source of truth). + * Mutations go through prepareMutation: snapshot → mutate → clean empty groups → commit. + */ +export class GroupedModulesProxy { + #items = $state([]) + #error = $state(undefined) + #flowStore: StateStore + + constructor(flowStore: StateStore) { + this.#flowStore = flowStore + this.rebuild() + + // Rebuild tree whenever store changes (undo/load/mutation) + $effect(() => { + void flowStore.val.value.modules + void flowStore.val.value.groups + untrack(() => this.rebuild()) + }) + } + + /** Reactive access to the structure tree (read-only view) */ + get items(): FlowStructureNode[] { + return this.#items + } + + /** Reactive access to build errors */ + get error(): unknown { + return this.#error + } + + /** + * Prepare a structural mutation without writing to the store yet. + * Returns the list of groups that became empty (already removed from the snapshot) + * and a `commit` function that writes the result to the store. + * + * If no groups were emptied, the caller can commit immediately. + * If groups were emptied, the caller should show a confirmation modal + * and call commit() only on user confirmation. + */ + prepareMutation( + mutate: (tree: FlowStructureNode[]) => void, + opts?: { + extraModules?: FlowModule[] + displayState?: import('./groupEditor.svelte').GroupDisplayState + } + ): { + emptiedGroups: FlowGroup[] + duplicateGroups: FlowGroup[] + commit: (commitOpts?: { removeDuplicates?: boolean }) => void + } { + const snapshot = $state.snapshot(this.#items) as FlowStructureNode[] + mutate(snapshot) + + // Clean up empty groups and collect which ones were removed + const emptiedGroups = removeEmptyGroups(snapshot) + // Detect groups that became duplicates after the mutation + const duplicateGroups = findDuplicateGroups(snapshot) + + const commit = (commitOpts?: { removeDuplicates?: boolean }) => { + if (commitOpts?.removeDuplicates && duplicateGroups.length > 0) { + removeDuplicateGroups(snapshot) + } + + // Remap runtime state for groups whose boundaries shifted + if (opts?.displayState) { + this.#remapChangedGroupKeys(snapshot, opts.displayState) + } + + // Build moduleMap lazily at commit time so it reflects the latest store state + const moduleMap = new Map() + for (const m of getAllModules(this.#flowStore.val.value.modules)) { + moduleMap.set(m.id, m) + } + if (opts?.extraModules) { + for (const m of opts.extraModules) { + moduleMap.set(m.id, m) + } + } + this.#flowStore.val.value.modules = applyStructureToModules(snapshot, moduleMap) + this.#flowStore.val.value.groups = deriveGroupsFromStructure(snapshot) + } + + return { emptiedGroups, duplicateGroups, commit } + } + + /** + * Convenience: prepare + auto-commit. Only use for mutations that cannot + * empty groups (e.g. inserts). Throws if groups are unexpectedly emptied. + * For mutations that may empty groups, use prepareMutation() directly. + */ + applyTreeMutation( + mutate: (tree: FlowStructureNode[]) => void, + opts?: { + extraModules?: FlowModule[] + displayState?: import('./groupEditor.svelte').GroupDisplayState + } + ): void { + const { emptiedGroups, duplicateGroups, commit } = this.prepareMutation(mutate, opts) + if (emptiedGroups.length > 0) { + console.error('applyTreeMutation: unexpected empty groups', emptiedGroups) + } + if (duplicateGroups.length > 0) { + console.error('applyTreeMutation: unexpected duplicate groups', duplicateGroups) + } + commit() + } + + /** Remap runtime state for group nodes whose boundaries shifted after a mutation. */ + #remapChangedGroupKeys( + snapshot: FlowStructureNode[], + displayState: import('./groupEditor.svelte').GroupDisplayState + ): void { + const walk = (nodes: FlowStructureNode[]) => { + for (const node of nodes) { + if (node.kind === 'group') { + const oldKey = node.id + const flatIds = flattenStructureIds(node.branches[0].children) + const newKey = flatIds.length > 0 ? `${flatIds[0]}:${flatIds[flatIds.length - 1]}` : null + if (newKey && oldKey !== newKey) { + displayState.remapGroupKey(oldKey, newKey) + } + walk(node.branches[0].children) + } else { + for (const branch of node.branches) { + walk(branch.children) + } + } + } + } + walk(snapshot) + } + + /** Rebuild from flowStore */ + private rebuild(): void { + const modules = stateSnapshot(this.#flowStore.val.value.modules) as FlowModule[] + const allGroups = this.#flowStore.val.value.groups ?? [] + const allModules = getAllModules(modules) + const graphGroups: GraphGroup[] = allGroups.map((g) => ({ + ...g, + id: groupKey(g), + moduleIds: computeGroupModuleIds(g.start_id, g.end_id, allModules) + })) + try { + this.#items = buildStructureTree(modules, graphGroups) + this.#error = undefined + } catch (e) { + // Intentionally preserve last-known-good #items so the graph + // can still render while the error is surfaced to the user. + this.#error = e + } + } +} diff --git a/frontend/src/lib/components/graph/moveManager.svelte.ts b/frontend/src/lib/components/graph/moveManager.svelte.ts index 605c4b849a..1539cc0802 100644 --- a/frontend/src/lib/components/graph/moveManager.svelte.ts +++ b/frontend/src/lib/components/graph/moveManager.svelte.ts @@ -204,9 +204,6 @@ export class MoveManager { for (const [edgeId, zone] of this.#registeredDropZones) { if (zone.disableMoveIds.includes(draggedId)) continue - // Skip edges adjacent to the dragged node (no-op move) - if (zone.sourceId === draggedId || zone.targetId === draggedId) continue - const dx = Math.abs(flowPos.x - zone.centerX) const dy = Math.abs(flowPos.y - zone.centerY) diff --git a/frontend/src/lib/components/graph/nodeExtraSpace.ts b/frontend/src/lib/components/graph/nodeExtraSpace.ts new file mode 100644 index 0000000000..f51309a60b --- /dev/null +++ b/frontend/src/lib/components/graph/nodeExtraSpace.ts @@ -0,0 +1,153 @@ +import type { FlowNote } from '../../gen' +import type { AssetWithAltAccessType } from '../assets/lib' +import { + assetDisplaysAsInputInFlowGraph, + assetDisplaysAsOutputInFlowGraph, + NODE_WITH_READ_ASSET_Y_OFFSET, + NODE_WITH_WRITE_ASSET_Y_OFFSET +} from './renderers/nodes/AssetNode.svelte' +import { + AI_TOOL_BASE_OFFSET, + AI_TOOL_ROW_OFFSET, + BELOW_ADDITIONAL_OFFSET +} from './renderers/nodes/AIToolNode.svelte' +import { topologicalSort } from './graphBuilder.svelte' +import { GROUP_HEADER_HEIGHT } from './groupEditor.svelte' +import type { GroupDisplayState } from './groupEditor.svelte' +import type { GraphModuleState } from '.' + +type NodeDep = { + id: string + parentIds?: string[] + data?: { assets?: AssetWithAltAccessType[]; module?: any } +} + +type ExtraSpace = { top: number; bottom: number; left: number; right: number } + +const MAX_TOOLS_PER_ROW = 2 + +/** + * Pre-compute extra top/bottom space each node needs for decorations + * (assets, AI tools, group headers, group notes). + */ +export function computeNodeExtraSpace( + graphNodes: NodeDep[], + opts: { + showAssets: boolean + showNotes: boolean + notes: FlowNote[] | undefined + noteTextHeights: Record + groupDisplayState: GroupDisplayState + insertable: boolean + flowModuleStates: Record | undefined + } +): Map | undefined { + const extraSpace = new Map() + + // 1. Assets + if (opts.showAssets) { + for (const node of graphNodes) { + const assets = node.data?.assets ?? [] + if (!assets.length) continue + const hasRead = assets.some(assetDisplaysAsInputInFlowGraph) + const hasWrite = assets.some(assetDisplaysAsOutputInFlowGraph) + if (hasRead || hasWrite) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + top: prev.top + (hasRead ? NODE_WITH_READ_ASSET_Y_OFFSET : 0), + bottom: prev.bottom + (hasWrite ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0) + }) + } + } + } + + // 2. AI tools + for (const node of graphNodes) { + const mod = node.data?.module + if (!mod || mod.value?.type !== 'aiagent') continue + + const agentActions = !opts.insertable && opts.flowModuleStates?.[node.id]?.agent_actions + + if (agentActions) { + // Execution mode: tools below + const totalRows = Math.ceil(agentActions.length / MAX_TOOLS_PER_ROW) + const space = AI_TOOL_BASE_OFFSET + AI_TOOL_ROW_OFFSET * totalRows + BELOW_ADDITIONAL_OFFSET + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { ...prev, bottom: prev.bottom + space }) + } else { + // Edit mode: tools above + const tools = mod.value.tools ?? [] + const totalRows = Math.ceil(tools.length / MAX_TOOLS_PER_ROW) + (opts.insertable ? 1 : 0) + const space = AI_TOOL_BASE_OFFSET + AI_TOOL_ROW_OFFSET * totalRows + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { ...prev, top: prev.top + space }) + } + } + + // Topological sort (reversed: top-of-graph first) — shared by group notes and group headers + const sortedNodes = topologicalSort(graphNodes).reverse() + + // 3. Group notes (text above topmost node in each group note) + if (opts.showNotes) { + const groupNotes = (opts.notes ?? []).filter((n) => n.type === 'group') + if (groupNotes.length > 0) { + for (const groupNote of groupNotes) { + if (!groupNote.contained_node_ids?.length) continue + const topmostNodeId = sortedNodes.find((node) => + groupNote.contained_node_ids?.includes(node.id) + )?.id + if (topmostNodeId) { + const textHeight = opts.noteTextHeights[groupNote.id] || 60 + const spacing = textHeight + 16 // padding + const prev = extraSpace.get(topmostNodeId) ?? { + top: 0, + bottom: 0, + left: 0, + right: 0 + } + extraSpace.set(topmostNodeId, { + ...prev, + top: Math.max(prev.top, spacing + prev.top) + }) + } + } + } + } + + // 4. Collapsed group nodes are taller than regular nodes (header + module icons) + for (const node of graphNodes) { + if (node.id.startsWith('collapsed-group:')) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + bottom: prev.bottom + GROUP_HEADER_HEIGHT + }) + } + } + + // 5. Group nodes (expanded heads and collapsed) with notes need extra height + if (opts.showNotes) { + const noteHeights = opts.groupDisplayState.getNoteHeights() + for (const node of graphNodes) { + let groupId: string | undefined + if (node.id.startsWith('group:') && !node.id.endsWith('-end')) { + groupId = node.id.slice('group:'.length) + } else if (node.id.startsWith('collapsed-group:')) { + groupId = node.id.slice('collapsed-group:'.length) + } + if (groupId) { + const noteHeight = noteHeights[groupId] + if (noteHeight && noteHeight > 0) { + const prev = extraSpace.get(node.id) ?? { top: 0, bottom: 0, left: 0, right: 0 } + extraSpace.set(node.id, { + ...prev, + bottom: prev.bottom + noteHeight + }) + } + } + } + } + + return extraSpace.size > 0 ? extraSpace : undefined +} diff --git a/frontend/src/lib/components/graph/noteColors.ts b/frontend/src/lib/components/graph/noteColors.ts index f9024adf7e..2a82ed1f40 100644 --- a/frontend/src/lib/components/graph/noteColors.ts +++ b/frontend/src/lib/components/graph/noteColors.ts @@ -14,6 +14,7 @@ export enum NoteColor { export interface NoteColorConfig { background: string + backgroundLight: string outline: string outlineHover: string text: string @@ -24,70 +25,80 @@ export interface NoteColorConfig { export const NOTE_COLORS: Record = { [NoteColor.YELLOW]: { background: 'bg-yellow-200 dark:bg-yellow-900', - outline: 'outline-yellow-300 dark:outline-yellow-600', + backgroundLight: 'bg-yellow-400/5 dark:bg-yellow-600/5', + outline: 'outline-yellow-200 dark:outline-yellow-900', outlineHover: 'outline-yellow-300/60 dark:outline-yellow-600/60', text: 'text-yellow-900 dark:text-yellow-100', hover: 'hover:bg-yellow-200 dark:hover:bg-yellow-800' }, [NoteColor.BLUE]: { background: 'bg-blue-100 dark:bg-blue-950', - outline: 'outline-blue-300 dark:outline-blue-600', + backgroundLight: 'bg-blue-400/5 dark:bg-blue-600/5', + outline: 'outline-blue-100 dark:outline-blue-950', outlineHover: 'outline-blue-300/60 dark:outline-blue-600/60', text: 'text-blue-900 dark:text-blue-100', hover: 'hover:bg-blue-200 dark:hover:bg-blue-800' }, [NoteColor.GREEN]: { background: 'bg-green-200 dark:bg-green-900', - outline: 'outline-green-300 dark:outline-green-600', + backgroundLight: 'bg-green-400/5 dark:bg-green-600/5', + outline: 'outline-green-200 dark:outline-green-900', outlineHover: 'outline-green-300/60 dark:outline-green-600/60', text: 'text-green-900 dark:text-green-100', hover: 'hover:bg-green-200 dark:hover:bg-green-800' }, [NoteColor.PURPLE]: { background: 'bg-purple-200 dark:bg-purple-900', - outline: 'outline-purple-300 dark:outline-purple-600', + backgroundLight: 'bg-purple-400/5 dark:bg-purple-600/5', + outline: 'outline-purple-200 dark:outline-purple-900', outlineHover: 'outline-purple-300/60 dark:outline-purple-600/60', text: 'text-purple-900 dark:text-purple-100', hover: 'hover:bg-purple-200 dark:hover:bg-purple-800' }, [NoteColor.PINK]: { background: 'bg-pink-200 dark:bg-pink-900', - outline: 'outline-pink-300 dark:outline-pink-600', + backgroundLight: 'bg-pink-400/5 dark:bg-pink-600/5', + outline: 'outline-pink-200 dark:outline-pink-900', outlineHover: 'outline-pink-300/60 dark:outline-pink-600/60', text: 'text-pink-900 dark:text-pink-100', hover: 'hover:bg-pink-200 dark:hover:bg-pink-800' }, [NoteColor.ORANGE]: { background: 'bg-orange-200 dark:bg-orange-900', - outline: 'outline-orange-300 dark:outline-orange-600', + backgroundLight: 'bg-orange-400/5 dark:bg-orange-600/5', + outline: 'outline-orange-200 dark:outline-orange-900', outlineHover: 'outline-orange-300/60 dark:outline-orange-600/60', text: 'text-orange-900 dark:text-orange-100', hover: 'hover:bg-orange-200 dark:hover:bg-orange-800' }, [NoteColor.RED]: { background: 'bg-red-200 dark:bg-red-900', - outline: 'outline-red-300 dark:outline-red-600', + backgroundLight: 'bg-red-400/5 dark:bg-red-600/5', + outline: 'outline-red-200 dark:outline-red-900', outlineHover: 'outline-red-300/60 dark:outline-red-600/60', text: 'text-red-900 dark:text-red-100', hover: 'hover:bg-red-200 dark:hover:bg-red-800' }, [NoteColor.CYAN]: { background: 'bg-cyan-200 dark:bg-cyan-900', - outline: 'outline-cyan-300 dark:outline-cyan-600', + backgroundLight: 'bg-cyan-400/5 dark:bg-cyan-600/5', + outline: 'outline-cyan-200 dark:outline-cyan-900', outlineHover: 'outline-cyan-300/60 dark:outline-cyan-600/60', text: 'text-cyan-900 dark:text-cyan-100', hover: 'hover:bg-cyan-200 dark:hover:bg-cyan-800' }, [NoteColor.LIME]: { background: 'bg-lime-200 dark:bg-lime-900', - outline: 'outline-lime-300 dark:outline-lime-600', + backgroundLight: 'bg-lime-400/5 dark:bg-lime-600/5', + outline: 'outline-lime-200 dark:outline-lime-900', outlineHover: 'outline-lime-300/60 dark:outline-lime-600/60', text: 'text-lime-900 dark:text-lime-100', hover: 'hover:bg-lime-200 dark:hover:bg-lime-800' }, [NoteColor.GRAY]: { background: 'bg-gray-200 dark:bg-gray-800', - outline: 'outline-gray-300 dark:outline-gray-600', + backgroundLight: 'bg-gray-400/5 dark:bg-gray-600/5', + outline: 'outline-gray-200 dark:outline-gray-800', outlineHover: 'outline-gray-300/60 dark:outline-gray-600/60', text: 'text-gray-900 dark:text-gray-100', hover: 'hover:bg-gray-200 dark:hover:bg-gray-700' diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts index 3350e07ce8..bd7fc2ce05 100644 --- a/frontend/src/lib/components/graph/noteUtils.svelte.ts +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -19,20 +19,6 @@ export type NodeDep = { export type NoteComputeResult = { noteNodes: (Node & NodeLayout)[] - newNodePositions: Record -} - -export type AIToolSpacingInfo = { - toolNodes: (Node & NodeLayout)[] - toolEdges: any[] - newNodePositions: Record -} - -export interface GroupNoteBounds { - x: number - y: number - width: number - height: number } let computeNoteNodesCache: @@ -283,14 +269,9 @@ export function computeNoteNodes( const allNoteNodes: (Node & NodeLayout)[] = [] - // Build a map of Y positions that need extra spacing for group notes - const yPosMap: Record = {} // Y position -> spacing needed - - // Group notes that need spacing + // Find topmost node per group note for layout calculation const groupNotes = notes.filter((n) => n.type === 'group') - const topMostNodesMap: Record = {} - const sortedNodes = topologicalSort(nodes).reverse() for (const groupNote of groupNotes) { @@ -298,47 +279,12 @@ export function computeNoteNodes( const topmostNodeId = sortedNodes.find((node) => groupNote.contained_node_ids?.includes(node.id) )?.id - const topmostNode = nodes.find((node) => node.id === topmostNodeId) - if (topmostNode) { - const textHeight = noteTextHeights[groupNote.id] || 60 - const spacing = textHeight + 16 // padding - // Mark this Y position as needing spacing - yPosMap[topmostNode.position.y] = Math.max(yPosMap[topmostNode.position.y] || 0, spacing) - topMostNodesMap[groupNote.id] = topmostNode.id + if (topmostNodeId) { + topMostNodesMap[groupNote.id] = topmostNodeId } } } - // Calculate new positions for nodes (offset by group notes) - const sortedNewNodes = nodes - .map((n) => ({ position: { ...n.position }, id: n.id })) - .sort((a, b) => a.position.y - b.position.y) - - let currentYOffset = 0 - let prevYPos = NaN - - for (const node of sortedNewNodes) { - if (node.position.y !== prevYPos) { - // Add spacing for group notes at this Y level - if (yPosMap[node.position.y]) { - currentYOffset += yPosMap[node.position.y] - } - prevYPos = node.position.y - } - node.position.y += currentYOffset - } - - // Create note nodes AFTER calculating adjusted node positions - // For group notes, we need to use the adjusted node positions - const adjustedNodes = sortedNewNodes.map((n) => { - const origNode = nodes.find((orig) => orig.id === n.id) - return { - ...n, - data: origNode?.data, - type: origNode?.type - } - }) - // Calculate all z-indexes at once using hierarchy information const noteZIndexes = calculateAllNoteZIndexes(notes, nodes) @@ -346,11 +292,11 @@ export function computeNoteNodes( const isGroupNote = note.type === 'group' const zIndex = noteZIndexes[note.id] - // Calculate position and size using adjusted node positions for group notes + // Calculate position and size using node positions for group notes const { position, size } = isGroupNote ? calculateGroupNoteLayout( note, - adjustedNodes, + nodes, noteTextHeights[note.id] || 60, topMostNodesMap[note.id] ) @@ -375,13 +321,8 @@ export function computeNoteNodes( allNoteNodes.push(noteNode) } - const newNodePositions: Record = Object.fromEntries( - sortedNewNodes.map((n) => [n.id, n.position]) - ) - const result: NoteComputeResult = { - noteNodes: allNoteNodes, - newNodePositions + noteNodes: allNoteNodes } // Cache the result diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index 8f1ca0017f..5fbf084374 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -12,11 +12,14 @@ import type { GraphModuleState } from '../../model' import InsertModuleButton from '$lib/components/flows/map/InsertModuleButton.svelte' import { getGraphContext } from '../../graphContext' + import { GROUP_TOP_PADDING } from '$lib/components/graph/compoundLayout' const { useDataflow, showAssets, moveManager } = getGraphContext() let { id, + source, + target, sourceX, sourceY, sourcePosition, @@ -45,6 +48,13 @@ } } = $props() + // Derive group boundary from source/target node IDs + let groupBoundary: 'top' | 'bottom' | undefined = $derived.by(() => { + if (source.startsWith('group:') && !source.endsWith('-end')) return 'top' + if (target.startsWith('group:') && target.endsWith('-end')) return 'bottom' + return undefined + }) + let [edgePath] = $derived( getBezierPath({ sourceX, @@ -75,9 +85,15 @@ ) let centerY = $derived( - sourceY + - 32 + - (data.shouldOffsetInsertBtnDueToAssetNode && $showAssets ? NODE_WITH_WRITE_ASSET_Y_OFFSET : 0) + groupBoundary === 'bottom' + ? targetY + : groupBoundary === 'top' + ? sourceY + GROUP_TOP_PADDING / 2 + : sourceY + + 32 + + (data.shouldOffsetInsertBtnDueToAssetNode && $showAssets + ? NODE_WITH_WRITE_ASSET_Y_OFFSET + : 0) ) let isDragging = $derived(!!moveManager?.dragging) @@ -87,13 +103,13 @@ data?.insertable && draggedId !== undefined && !data.disableMoveIds?.includes(draggedId) && - data.sourceId !== draggedId && - data.targetId !== draggedId + source !== draggedId && + target !== draggedId ) - let isNearestDrop = $derived(isValidDropTarget && moveManager?.nearestDropZone?.edgeId === id) - let isAdjacentToDragged = $derived( - isDragging && (data?.sourceId === draggedId || data?.targetId === draggedId) + let isNearestDrop = $derived( + isValidDropTarget && moveManager?.nearestDropZone?.edgeId === id ? true : false ) + let isAdjacentToDragged = $derived(isDragging && (source === draggedId || target === draggedId)) // Register this edge's drop zone position with the drag manager so proximity // detection uses the actual xyflow-computed position rather than re-deriving it. @@ -161,7 +177,7 @@ {@render dropTargetIndicator(isNearestDrop)}
    - {:else if data?.insertable && !$useDataflow && !moveManager?.movingModuleId && !isDragging} + {:else if data?.insertable && !groupBoundary && !$useDataflow && !moveManager?.movingModuleId && !isDragging}
    - {#if !(moveManager.movingIds ?? [moveManager.movingModuleId]).some((id) => data.disableMoveIds?.includes(id))} + {#if !(moveManager.movingIds ?? [moveManager.movingModuleId]).some( (id) => data.disableMoveIds?.includes(id) )} - {/if} -
    +
    + {/if} {/snippet} diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte index 2ffa6a8d49..ccc5643981 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte @@ -53,7 +53,7 @@