diff --git a/.claude/settings.json b/.claude/settings.json index fcd49c3140..cf8bfdd284 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -110,7 +110,6 @@ ] }, "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "code-review@claude-plugins-official": true } diff --git a/.wmdev.yaml b/.wmdev.yaml index c028c8f3bf..1a949c94e2 100644 --- a/.wmdev.yaml +++ b/.wmdev.yaml @@ -2,6 +2,8 @@ name: Windmill startupEnvs: CARGO_FEATURES: "quickjs" + WM_CLONE_DB: false + USE_RUST_PLUGIN: false services: - name: BE diff --git a/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json new file mode 100644 index 0000000000..18ad13d90f --- /dev/null +++ b/backend/.sqlx/query-19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "19a7ebb2e7e8e57b6e7c974da8eb7c6841a5c4ff12ba7c12c73d691c49dd99ed" +} diff --git a/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json new file mode 100644 index 0000000000..8c5f43ab07 --- /dev/null +++ b/backend/.sqlx/query-2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n flow_version.value -> 'flow_env' -> $3\n ELSE\n root_job.raw_flow -> 'flow_env' -> $3\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flow_env: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2f53576c2ad58abc24617e911e486d7c4b9bdb1e8fb1f7725060990ef8984943" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json b/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json deleted file mode 100644 index be352ce88e..0000000000 --- a/backend/.sqlx/query-c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n CASE\n WHEN flow_version.id IS NOT NULL THEN\n (flow_version.value -> 'flow_env' -> $3) #> $4\n ELSE\n (root_job.raw_flow -> 'flow_env' -> $3) #> $4\n END AS \"flow_env: sqlx::types::Json>\"\n FROM\n v2_job current_job\n JOIN\n v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id)\n AND root_job.workspace_id = current_job.workspace_id\n LEFT JOIN\n flow_version ON flow_version.id = root_job.runnable_id\n AND flow_version.path = root_job.runnable_path\n AND flow_version.workspace_id = root_job.workspace_id\n WHERE\n current_job.id = $1 AND\n current_job.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "flow_env: sqlx::types::Json>", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Text", - "TextArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "c23bea7db9623a60683596b7d6e689e2c0100c1569436a01b207876aaa470154" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d26d5b76d7..e52bb12fa1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16383,6 +16383,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "lazy_static", + "magic-crypt", "regex", "serde", "serde_json", diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs new file mode 100644 index 0000000000..1a60f8672c --- /dev/null +++ b/backend/tests/end_user_email.rs @@ -0,0 +1,323 @@ +//! Tests for WM_END_USER_EMAIL environment variable. +//! +//! These tests verify that WM_END_USER_EMAIL is populated with the authenticated +//! user's email when executing app components. +//! +//! TODO: Add tests for scripts and flows once public execution endpoints are identified. +//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::worker::Connection; +use windmill_test_utils::*; + +const SAME_WS_TOKEN: &str = "SECRET_TOKEN"; +const OTHER_WS_TOKEN: &str = "OTHER_WS_TOKEN"; +const NO_WS_TOKEN: &str = "NO_WS_TOKEN"; + +const SAME_WS_EMAIL: &str = "test@windmill.dev"; +const OTHER_WS_EMAIL: &str = "other-ws@windmill.dev"; +const NO_WS_EMAIL: &str = "no-ws@windmill.dev"; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// async fn run_script(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// async fn run_flow(port: u16, token: &str) -> anyhow::Result { +// let url = format!( +// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", +// port +// ); +// let resp = authed(client().post(&url), token) +// .json(&json!({})) +// .send() +// .await?; +// if !resp.status().is_success() { +// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); +// } +// Ok(resp.json::().await? +// .as_str().unwrap_or("").to_string()) +// } + +/// Create an app with inline script via API +async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test app for WM_END_USER_EMAIL", + "value": { + "type": "app", + "grid": [], + "subgrids": {}, + "hiddenInlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": "f/test/email_app/get_email" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +/// Create a raw app with inline script via API (uses regular app endpoint with rawapp type) +async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps/create", + port + ); + let resp = authed(client().post(&url), SAME_WS_TOKEN) + .json(&json!({ + "path": path, + "summary": "Test raw app for WM_END_USER_EMAIL", + "value": { + "type": "rawapp", + "css": "", + "inlineScripts": [{ + "name": "get_email", + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + }] + }, + "policy": { + "execution_mode": "anonymous", + "on_behalf_of": null, + "on_behalf_of_email": null, + "triggerables_v2": { + "get_email": { + "static_inputs": {}, + "one_of_inputs": {} + }, + // SHA256 hash of raw_code content for anonymous execution + "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("create raw app failed: {} - {}", resp.status(), resp.text().await?); + } + Ok(()) +} + +async fn run_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "path": format!("{}/get_email", app_path) + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn run_raw_app_inline_script(port: u16, token: &str, app_path: &str, force_viewer: bool) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/apps_u/execute_component/{}", + port, app_path + ); + let mut payload = json!({ + "args": {}, + "component": "get_email", + "raw_code": { + "language": "deno", + "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + } + }); + if force_viewer { + payload["force_viewer_static_fields"] = json!({}); + } + let resp = authed(client().post(&url), token) + .json(&payload) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!("raw app inline script run failed: {} - {}", resp.status(), resp.text().await?); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} + +async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs_u/completed/get_result/{}", + port, job_id + ); + for _ in 0..100 { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + let resp = authed(client().get(&url), token).send().await?; + if resp.status().is_success() { + return Ok(resp.json::().await? + .as_str().unwrap_or("").to_string()); + } + } + anyhow::bail!("timeout waiting for job result") +} + +// TODO: Script tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_script(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members +// #[cfg(feature = "deno_core")] +// #[sqlx::test(fixtures("base", "end_user_email"))] +// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { +// initialize_tracing().await; +// set_jwt_secret().await; +// let server = ApiServer::start(db.clone()).await?; +// let port = server.addr.port(); +// +// in_test_worker(Connection::Sql(db.clone()), async move { +// let result = run_flow(port, SAME_WS_TOKEN).await?; +// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); +// Ok::<(), anyhow::Error>(()) +// }, port).await?; +// +// Ok(()) +// } + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the app with inline script first + create_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} + +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_raw_app_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let app_path = "f/test/email_raw_app"; + + in_test_worker(Connection::Sql(db.clone()), async move { + // Create the raw app with inline script first + create_raw_app_with_inline_script(port, app_path).await?; + + // Same workspace user (force_viewer mode works for workspace members) + let result = run_raw_app_inline_script(port, SAME_WS_TOKEN, app_path, true).await?; + assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); + + // Other workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, OTHER_WS_TOKEN, app_path, false).await?; + assert_eq!(result, OTHER_WS_EMAIL, "other workspace user should get their email"); + + // No workspace user (uses app's anonymous policy + token lookup) + let result = run_raw_app_inline_script(port, NO_WS_TOKEN, app_path, false).await?; + assert_eq!(result, NO_WS_EMAIL, "no workspace user should get their email"); + + Ok::<(), anyhow::Error>(()) + }, port).await?; + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql new file mode 100644 index 0000000000..654ad93680 --- /dev/null +++ b/backend/tests/fixtures/end_user_email.sql @@ -0,0 +1,63 @@ +-- Fixture for WM_END_USER_EMAIL tests +-- Sets up 3 users with different workspace memberships: +-- 1. test@windmill.dev - in test-workspace (from base.sql) +-- 2. other-ws@windmill.dev - in other-workspace only +-- 3. no-ws@windmill.dev - not in any workspace + +-- Second workspace for cross-workspace user +INSERT INTO workspace (id, name, owner) +VALUES ('other-workspace', 'other-workspace', 'other-ws-user'); + +INSERT INTO workspace_key(workspace_id, kind, key) +VALUES ('other-workspace', 'cloud', 'other-key'); + +INSERT INTO workspace_settings (workspace_id) +VALUES ('other-workspace'); + +INSERT INTO group_ (workspace_id, name, summary, extra_perms) +VALUES ('other-workspace', 'all', 'All users', '{}'); + +-- User in other-workspace only (not in test-workspace) +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('other-ws@windmill.dev', 'hash', 'password', false, true, 'Other WS User'); + +INSERT INTO usr(workspace_id, email, username, is_admin, role) +VALUES ('other-workspace', 'other-ws@windmill.dev', 'other-ws-user', true, 'Admin'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('OTHER_WS_TOKEN', 'other-ws@windmill.dev', 'other ws token', false); + +-- User not in any workspace +INSERT INTO password(email, password_hash, login_type, super_admin, verified, name) +VALUES ('no-ws@windmill.dev', 'hash', 'password', false, true, 'No WS User'); + +INSERT INTO token(token, email, label, super_admin) +VALUES ('NO_WS_TOKEN', 'no-ws@windmill.dev', 'no ws token', false); + +-- Script that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) +VALUES ( + 'test-workspace', 'test-user', + 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + '{"g/all": true}' +); + +-- Flow that returns WM_END_USER_EMAIL (public via extra_perms) +INSERT INTO flow (workspace_id, summary, description, path, versions, schema, value, edited_by, extra_perms) +VALUES ( + 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user', + '{"g/all": true}' +); + +INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) +VALUES ( + 900002, 'test-workspace', 'f/test/get_end_user_email_flow', + '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + 'test-user' +); diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index 186f7af257..af212dc315 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -35,7 +35,45 @@ use windmill_common::{ lazy_static::lazy_static! { // Global auth cache accessible from main.rs for direct invalidation pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); + // Cache for token -> email lookups (for non-workspace-member authenticated users) + static ref TOKEN_EMAIL_CACHE: Cache> = Cache::new(500); +} +/// Get email from a valid token, with caching. +/// Used for WM_END_USER_EMAIL when user is authenticated but not a workspace member. +async fn get_email_from_token(db: &DB, token: &str) -> Option { + if let Some(cached) = TOKEN_EMAIL_CACHE.get(token) { + return cached; + } + + let email = sqlx::query_scalar!( + "SELECT email FROM token WHERE token = $1 AND (expiration > NOW() OR expiration IS NULL)", + token + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten(); // email column is nullable, so we get Option> + + TOKEN_EMAIL_CACHE.insert(token.to_string(), email.clone()); + email +} + +/// Get end user email from authenticated user or token. +/// Returns email if user is authenticated (workspace member) or has valid instance token. +pub async fn get_end_user_email( + db: &DB, + opt_authed: Option<&ApiAuthed>, + token: Option<&str>, +) -> Option { + if let Some(authed) = opt_authed { + return Some(authed.email.clone()); + } + if let Some(token) = token { + return get_email_from_token(db, token).await; + } + None } // Global function to invalidate a specific token from cache pub fn invalidate_token_from_cache(token: &str) { diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index efab57cf91..5acb696bd4 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -29,8 +29,8 @@ use scopes::ScopeDefinition; // Re-export key auth types and functions pub use auth::{ - invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, Tokened, - TruncatedTokenWithEmail, AUTH_CACHE, + get_end_user_email, invalidate_token_from_cache, AuthCache, ExpiringAuthCache, OptTokened, + Tokened, TruncatedTokenWithEmail, AUTH_CACHE, }; // ------------ ApiAuthed & OptJobAuthed types ------------ diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index bf493f9420..6b408724fd 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -43,8 +43,8 @@ use windmill_common::{ get_database_url, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, + EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -519,6 +519,7 @@ pub async fn get_global_setting( && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING + && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING { diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index b698426d53..a03bb3a490 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -29,6 +29,7 @@ windmill-dep-map.workspace = true axum.workspace = true chrono.workspace = true hex.workspace = true +magic-crypt.workspace = true http.workspace = true hyper.workspace = true lazy_static.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8f3311978a..80e497ba97 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -31,7 +31,9 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::variables::{ + build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, +}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -2418,20 +2420,28 @@ async fn set_encryption_key( )); } + // Build the previous cipher before the transaction (reads from cache/pool) let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + let mut tx = db.begin().await?; + sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), w_id ) - .execute(&db) + .execute(&mut *tx) .await?; - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + // Build the new cipher directly from the key string, since the transaction + // hasn't committed yet and build_crypt() would read the old key from the pool. + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", request.new_key, salt) + } else { + request.new_key.clone() + }; + let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); @@ -2445,7 +2455,7 @@ async fn set_encryption_key( "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", w_id ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; for variable in all_variables { @@ -2466,11 +2476,16 @@ async fn set_encryption_key( w_id, variable.path ) - .execute(&db) + .execute(&mut *tx) .await?; } } + tx.commit().await?; + + // Invalidate the cache only after the transaction has committed + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Trigger git sync for encryption key changes handle_deployment_metadata( &authed.email, diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 1a036f6dc9..7f04a7287e 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8857,9 +8857,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) @@ -14644,9 +14643,8 @@ paths: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index d156dc9c88..9846912192 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -8,7 +8,7 @@ use std::{collections::HashMap, sync::Arc}; * LICENSE-AGPL for a copy of the license. */ use crate::{ - auth::OptTokened, + auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, jobs::RunJobQuery, users::{require_owner_of_path, OptAuthed}, @@ -993,9 +993,18 @@ macro_rules! process_app_multipart { let mut uploaded_js = false; let mut multipart = $multipart; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + { + let name = field + .name() + .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? + .to_string(); + let data = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("failed to read multipart stream: {e}")) + })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; let (ntx, npath, nid) = $internal_fn( @@ -2149,7 +2158,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; - let end_user_email = opt_authed.as_ref().map(|a| a.email.clone()); + let end_user_email = + get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; let (uuid, mut tx) = push( &db, diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 144703ba5c..66a67c5f97 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,4 +1,5 @@ pub use windmill_api_auth::auth::{ - invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, AuthCache, - ExpiringAuthCache, OptTokened, Tokened, TruncatedTokenWithEmail, + get_end_user_email, invalidate_token_from_cache, list_tokens_internal, + transform_old_scope_to_new_scope, AuthCache, ExpiringAuthCache, OptTokened, Tokened, + TruncatedTokenWithEmail, }; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index d58dcd3c69..ac5a9a306b 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -448,14 +448,15 @@ async fn get_flow_env_by_flow_job_id( Path((w_id, flow_job_id, var_name)): Path<(String, Uuid, String)>, Query(JsonPath { json_path, .. }): Query, ) -> windmill_common::error::JsonResult> { - let flow_env = sqlx::query_scalar!( + // Fetch raw value (without json_path) to check for $var:/$res: references + let raw_value = sqlx::query_scalar!( r#" SELECT CASE WHEN flow_version.id IS NOT NULL THEN - (flow_version.value -> 'flow_env' -> $3) #> $4 + flow_version.value -> 'flow_env' -> $3 ELSE - (root_job.raw_flow -> 'flow_env' -> $3) #> $4 + root_job.raw_flow -> 'flow_env' -> $3 END AS "flow_env: sqlx::types::Json>" FROM v2_job current_job @@ -472,16 +473,86 @@ async fn get_flow_env_by_flow_job_id( flow_job_id, w_id, var_name, - json_path - .as_ref() - .map(|x| x.split(".").collect::>()) - .unwrap_or_default() as Vec<&str>, ) .fetch_optional(&db) .await? - .map(|r| r.map(|x| x.0)) - .flatten() - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + .and_then(|r| r.map(|x| x.0)); + + // Resolve $var:/$res: references if present + let resolved = if let Some(raw) = raw_value { + let raw_str = raw.get(); + let db_authed = windmill_common::db::DbWithOptAuthed::::from_authed( + &authed, + db.clone(), + None, + ); + if let Some(path) = raw_str + .strip_prefix("\"$var:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::variables::get_value_internal(&db_authed, &w_id, path, false) + .await + { + Ok(val) => to_raw_value(&serde_json::Value::String(val)), + Err(e) => { + tracing::warn!("Failed to resolve flow_env variable $var:{path}: {e}"); + raw + } + } + } else if let Some(path) = raw_str + .strip_prefix("\"$res:") + .and_then(|s| s.strip_suffix("\"")) + { + match windmill_store::resources::get_resource_value_interpolated_internal( + &db_authed, + &w_id, + path, + Some(flow_job_id), + Some(&tokened.token), + false, + ) + .await + { + Ok(Some(val)) => to_raw_value(&val), + Ok(None) => { + tracing::warn!( + "Failed to resolve flow_env resource $res:{path}: resource not found" + ); + raw + } + Err(e) => { + tracing::warn!("Failed to resolve flow_env resource $res:{path}: {e}"); + raw + } + } + } else { + raw + } + } else { + to_raw_value(&serde_json::Value::Null) + }; + + // Apply json_path navigation on the (possibly resolved) value + let flow_env = if let Some(ref jp) = json_path { + let mut value: serde_json::Value = + serde_json::from_str(resolved.get()).unwrap_or(serde_json::Value::Null); + for part in jp.split('.') { + value = match value { + serde_json::Value::Object(ref mut map) => { + map.remove(part).unwrap_or(serde_json::Value::Null) + } + serde_json::Value::Array(ref arr) => part + .parse::() + .ok() + .and_then(|i| arr.get(i).cloned()) + .unwrap_or(serde_json::Value::Null), + _ => serde_json::Value::Null, + }; + } + to_raw_value(&value) + } else { + resolved + }; log_job_view( &db, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3347127303..d4d8163ff2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -44,6 +44,7 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; +pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4de90be5c5..4ef88f5b78 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -230,6 +230,8 @@ pub struct GlobalSettings { pub no_default_maven: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_hub: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index 07e428c633..559196a3c2 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -2164,6 +2164,7 @@ version = "0.1.0" dependencies = [ "chrono", "duckdb", + "regex", "rust_decimal", "serde", "serde_json", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 7043b33ee5..7eb6869ab9 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" duckdb = { version = "1.4.4", features = ["bundled"] } +regex = "1" rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index c5c819e60b..2701319d6e 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,12 +1,14 @@ use std::{ collections::HashMap, - ffi::{CStr, CString, c_char, c_uint}, + ffi::{c_char, c_uint, CStr, CString}, ptr::null_mut, + sync::LazyLock, }; -use duckdb::{Row, core::LogicalTypeId, params_from_iter, types::TimeUnit}; -use rust_decimal::{Decimal, prelude::FromPrimitive}; -use serde::Deserialize; +use duckdb::{core::LogicalTypeId, params_from_iter, types::TimeUnit, Row}; +use regex::Regex; +use rust_decimal::{prelude::FromPrimitive, Decimal}; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; #[derive(Deserialize, Clone, Debug, PartialEq, Default)] @@ -96,6 +98,218 @@ pub extern "C" fn run_duckdb_ffi( }) } +#[derive(Serialize, Debug)] +struct PrepareQueryColumnInfo { + name: String, + #[serde(rename = "type")] + type_name: String, +} + +#[derive(Serialize, Debug)] +struct PrepareQueryResult { + #[serde(skip_serializing_if = "Option::is_none")] + columns: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + +fn is_setup_statement(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("ATTACH") + || upper.starts_with("USE") + || upper.starts_with("INSTALL") + || upper.starts_with("LOAD") + || upper.starts_with("SET") + || upper.starts_with("RESET") + || upper.starts_with("CREATE OR REPLACE SECRET") + || upper.starts_with("CREATE SECRET") +} + +/// Returns true if the query is expected to return a result set and can be wrapped with DESCRIBE. +fn is_describable_query(query: &str) -> bool { + let trimmed = query.trim_start(); + let upper = trimmed.to_uppercase(); + upper.starts_with("SELECT") + || upper.starts_with("WITH") + || upper.starts_with("VALUES") + || upper.starts_with("TABLE") + || upper.starts_with("FROM") +} + +static PARAM_RE: LazyLock = LazyLock::new(|| Regex::new(r"\$\d+").expect("invalid regex")); + +fn replace_params_with_null(query: &str) -> String { + PARAM_RE.replace_all(query, "NULL").to_string() +} + +#[unsafe(no_mangle)] +pub extern "C" fn prepare_duckdb_ffi( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> *mut c_char { + let r = match convert_prepare_args( + query_block_list, + query_block_list_count, + token, + base_internal_url, + w_id, + ) + .and_then(|(query_block_list, token, base_internal_url, w_id)| { + prepare_duckdb_internal(query_block_list, token, base_internal_url, w_id) + }) { + Ok(result) => result, + Err(err) => { + let err = serde_json::to_string(&err) + .unwrap_or_else(|_| "Unknown error in duckdb ffi lib".to_string()); + format!("ERROR {}", err) + } + }; + + CString::new(r).map(|s| s.into_raw()).unwrap_or_else(|e| { + println!("Failed to allocate error string in duckdb ffi lib: {:?}", e); + null_mut() + }) +} + +fn setup_duckdb_connection( + conn: &duckdb::Connection, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result<(), String> { + let (s3_access_key, s3_secret_key) = token.rsplit_once('.').unwrap_or(("", token)); + let (s3_endpoint_ssl, s3_endpoint) = base_internal_url + .split_once("://") + .unwrap_or(("http", &base_internal_url)); + let s3_endpoint_ssl = s3_endpoint_ssl == "https"; + + conn.execute_batch(&format!( + "INSTALL httpfs; LOAD httpfs; + INSTALL azure; LOAD azure; + CREATE OR REPLACE SECRET s3_secret ( + TYPE s3, + PROVIDER config, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + URL_STYLE path, + USE_SSL {s3_endpoint_ssl} + ); + CREATE OR REPLACE SECRET gcs_secret ( + TYPE gcs, + KEY_ID '{s3_access_key}', + SECRET '{s3_secret_key}', + ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', + USE_SSL {s3_endpoint_ssl} + ); + ", + )) + .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string())) +} + +fn convert_prepare_args<'a>( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, +) -> Result<(Vec<&'a str>, &'a str, &'a str, &'a str), String> { + let query_block_list = unsafe { + std::slice::from_raw_parts(query_block_list, query_block_list_count) + .iter() + .map(|q| { + CStr::from_ptr(*q).to_str().unwrap_or_else(|e| { + println!( + "Invalid query_block string pointer in duckdb ffi: {}", + e.to_string() + ); + "Invalid query_block string pointer in duckdb ffi" + }) + }) + .collect::>() + }; + let token = unsafe { CStr::from_ptr(token) } + .to_str() + .map_err(|e| format!("Invalid token string: {}", e.to_string()))?; + let base_internal_url = unsafe { CStr::from_ptr(base_internal_url) } + .to_str() + .map_err(|e| format!("Invalid base_internal_url string: {}", e.to_string()))?; + let w_id = unsafe { CStr::from_ptr(w_id) } + .to_str() + .map_err(|e| format!("Invalid w_id string: {}", e.to_string()))?; + Ok((query_block_list, token, base_internal_url, w_id)) +} + +fn prepare_duckdb_internal( + query_block_list: Vec<&str>, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result { + let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; + + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; + + let mut results: Vec = vec![]; + + // IMPORTANT: Setup statements (ATTACH, USE, INSTALL, etc.) are executed but intentionally + // do not produce a PrepareQueryResult entry. The frontend prepends these as connection setup + // before the actual user queries, and mapPrepareResults expects results.length to equal the + // number of user queries (not setup statements). If a new setup-like statement is added to + // the connection flow (e.g. in setup_duckdb_connection or transform_attach_ducklake) without + // also being caught by is_setup_statement, the result count will mismatch and the frontend + // will throw. + for query_block in &query_block_list { + if is_setup_statement(query_block) { + conn.execute_batch(query_block) + .map_err(|e| format!("Error executing setup statement: {}", e.to_string()))?; + continue; + } + + let modified_query = replace_params_with_null(query_block); + // Validate the query parses correctly by preparing it + if let Err(e) = conn.prepare(&modified_query) { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + continue; + } + + // DESCRIBE only works on queries that return result sets (SELECT, WITH, VALUES, TABLE, + // FROM). For non-returning statements (INSERT, UPDATE, DELETE, CREATE, DROP, ALTER, etc.) + // we skip DESCRIBE and assume no columns. + if !is_describable_query(&modified_query) { + results.push(PrepareQueryResult { columns: Some(vec![]), error: None }); + continue; + } + + // Note: We have to use a DESCRIBE statement and cannot simply use the + // methods returned by .prepare() because they panic if the statement was + // not executed at least once (which we specifically do not want to do). + let describe_query = format!("DESCRIBE {}", modified_query); + match conn.prepare(&describe_query).and_then(|mut stmt| { + let rows = stmt.query_map([], |row| { + Ok(PrepareQueryColumnInfo { + name: row.get::<_, String>(0)?, + type_name: row.get::<_, String>(1)?, + }) + })?; + rows.collect::, _>>() + }) { + Ok(columns) => { + results.push(PrepareQueryResult { columns: Some(columns), error: None }); + } + Err(e) => { + results.push(PrepareQueryResult { columns: None, error: Some(e.to_string()) }); + } + } + } + + serde_json::to_string(&results).map_err(|e| e.to_string()) +} + fn convert_args<'a>( query_block_list: *const *const c_char, query_block_list_count: usize, @@ -170,38 +384,7 @@ fn run_duckdb_internal<'a>( ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; - let (s3_access_key, s3_secret_key) = token.split_at(token.rfind('.').unwrap_or(0)); - let s3_secret_key = &s3_secret_key[1..]; - let (s3_endpoint_ssl, s3_endpoint) = base_internal_url - .split_once("://") - .unwrap_or(("http", &base_internal_url)); - let s3_endpoint_ssl = match s3_endpoint_ssl { - "https" => true, - _ => false, - }; - - conn.execute_batch(&format!( - "INSTALL httpfs; LOAD httpfs; - INSTALL azure; LOAD azure; - CREATE OR REPLACE SECRET s3_secret ( - TYPE s3, - PROVIDER config, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - URL_STYLE path, - USE_SSL {s3_endpoint_ssl} - ); - CREATE OR REPLACE SECRET gcs_secret ( - TYPE gcs, - KEY_ID '{s3_access_key}', - SECRET '{s3_secret_key}', - ENDPOINT '{s3_endpoint}/api/w/{w_id}/s3_proxy', - USE_SSL {s3_endpoint_ssl} - ); - ", - )) - .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string()))?; + setup_duckdb_connection(&conn, token, base_internal_url, w_id)?; let mut results: Vec>> = vec![]; let mut column_order = None; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 73136e1cc0..45e2f647a5 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -161,6 +161,22 @@ pub async fn do_duckdb( let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); + if annotations.prepare { + let result = tokio::task::spawn_blocking(move || { + prepare_duckdb_ffi_safe( + query_block_list.iter().map(String::as_str), + &token, + &base_internal_url, + &w_id, + ) + }) + .await + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r)?; + + return Ok(result); + } + let result = tokio::task::spawn_blocking(move || { run_duckdb_ffi_safe( query_block_list.iter().map(String::as_str), @@ -248,6 +264,18 @@ struct DuckDbFfiLib { collect_first_row_only: bool, ) -> *mut c_char, >, + prepare_duckdb_ffi: Option< + Symbol< + 'static, + unsafe extern "C" fn( + query_block_list: *const *const c_char, + query_block_list_count: usize, + token: *const c_char, + base_internal_url: *const c_char, + w_id: *const c_char, + ) -> *mut c_char, + >, + >, free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, } @@ -307,8 +335,11 @@ impl DuckDbFfiLib { } } + let prepare_duckdb_ffi = unsafe { lib.get(b"prepare_duckdb_ffi").ok() }; + Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, + prepare_duckdb_ffi, free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, }) } @@ -388,6 +419,56 @@ fn run_duckdb_ffi_safe<'a>( } } +fn prepare_duckdb_ffi_safe<'a>( + query_block_list: impl Iterator, + token: &str, + base_internal_url: &str, + w_id: &str, +) -> Result> { + let query_block_list = query_block_list + .map(|s| { + CString::new(s).map_err(|e| { + Error::ExecutionErr(format!("Failed CString conversion: {}", e.to_string())) + }) + }) + .collect::>>()?; + let query_block_list = query_block_list + .iter() + .map(|s| s.as_ptr()) + .collect::>(); + + let token = CString::new(token).map_err(to_anyhow)?; + let base_internal_url = CString::new(base_internal_url).map_err(to_anyhow)?; + let w_id = CString::new(w_id).map_err(to_anyhow)?; + + let lib = DuckDbFfiLib::get_singleton()?; + let prepare_fn = lib.prepare_duckdb_ffi.as_ref().ok_or_else(|| { + Error::InternalErr( + "prepare_duckdb_ffi not available in duckdb ffi library. Please update to the latest windmill_duckdb_ffi_lib.".to_string(), + ) + })?; + let free_cstr = &lib.free_cstr; + + let result_str = unsafe { + let ptr = prepare_fn( + query_block_list.as_ptr(), + query_block_list.len(), + token.as_ptr(), + base_internal_url.as_ptr(), + w_id.as_ptr(), + ); + let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); + free_cstr(ptr); + str + }; + + if result_str.starts_with("ERROR") { + Err(Error::ExecutionErr(result_str[6..].to_string())) + } else { + Ok(serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?) + } +} + struct ParsedAttachDbResource<'a> { resource_path: &'a str, name: &'a str, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index d0b7fc7a58..33dafbb482 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::common::{cached_result_path, get_root_job_id, save_in_cache}; +use crate::common::{cached_result_path, get_root_job_id, save_in_cache, transform_json}; use crate::js_eval::{eval_timeout, IdContext}; use crate::worker_utils::get_tag_and_concurrency; use crate::{ @@ -53,7 +53,7 @@ use windmill_common::runnable_settings::{ use windmill_common::scripts::{ScriptHash, ScriptRunnableSettingsInline}; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::WarnAfterExt; -use windmill_common::worker::to_raw_value; +use windmill_common::worker::{to_raw_value, Connection}; use windmill_common::{ add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, DB, @@ -2245,6 +2245,35 @@ pub async fn handle_flow( killpill_rx: &tokio::sync::broadcast::Receiver<()>, ) -> anyhow::Result<()> { let flow = flow_data.value(); + + // Resolve $var: and $res: references in flow_env. + // We resolve into a separate variable to avoid cloning the entire FlowValue + // (which includes modules, failure_module, etc.) just to replace flow_env. + let resolved_env; + let flow_env = if let Some(ref env) = flow.flow_env { + match transform_json( + client, + &flow_job.workspace_id, + env, + &flow_job, + &Connection::Sql(db.clone()), + ) + .await + { + Ok(Some(resolved)) => { + resolved_env = resolved; + Some(&resolved_env) + } + Ok(None) => flow.flow_env.as_ref(), + Err(e) => { + tracing::warn!("Failed to resolve flow_env references: {e}"); + flow.flow_env.as_ref() + } + } + } else { + None + }; + let status = flow_job .parse_flow_status() .with_context(|| "Unable to parse flow status")?; @@ -2348,6 +2377,7 @@ pub async fn handle_flow( flow_job, status, flow, + flow_env, db, client, last_result.clone(), @@ -2448,6 +2478,7 @@ async fn push_next_flow_job( flow_job: Arc, mut status: FlowStatus, flow: &FlowValue, + flow_env: Option<&HashMap>>, db: &sqlx::Pool, client: &AuthedClient, last_job_result: Option>>, @@ -2580,7 +2611,7 @@ async fn push_next_flow_job( let skip = compute_bool_from_expr( &skip_expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Arc::new(to_raw_value(&json!("{}"))), None, None, @@ -2705,7 +2736,7 @@ async fn push_next_flow_job( expr.to_string(), context, Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, None, None, None @@ -2966,7 +2997,7 @@ async fn push_next_flow_job( &input_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), None, ) @@ -3004,7 +3035,7 @@ async fn push_next_flow_job( &status.retry, arc_last_job_result.clone(), arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, Some(client), ) .await? @@ -3092,7 +3123,7 @@ async fn push_next_flow_job( compute_bool_from_expr( &skip_if.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), @@ -3182,7 +3213,7 @@ async fn push_next_flow_job( }; transform_input( arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3209,7 +3240,7 @@ async fn push_next_flow_job( let next_flow_transform = compute_next_flow_transform( arc_flow_job_args.clone(), arc_last_job_result.clone(), - flow.flow_env.as_ref(), + flow_env, &flow_job, &flow, transform_context, @@ -3373,7 +3404,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, "", &status); let ti = transform_input( Marc::new(args), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3428,7 +3459,7 @@ async fn push_next_flow_job( let ctx = get_transform_context(&flow_job, &previous_id, &status); let ti = transform_input( Marc::new(hm), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), input_transforms, resumes.clone(), @@ -3546,7 +3577,7 @@ async fn push_next_flow_job( timeout_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -3625,7 +3656,7 @@ async fn push_next_flow_job( parallelism_transform, arc_last_job_result.clone(), Some(arc_flow_job_args.clone()), - flow.flow_env.as_ref(), + flow_env, Some(client), Some(&ctx), ) @@ -4461,7 +4492,7 @@ async fn compute_next_flow_transform( let pred = compute_bool_from_expr( &b.expr, arc_flow_job_args.clone(), - flow.flow_env.as_ref(), + flow_env, arc_last_job_result.clone(), None, Some(&idcontext), diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index c5986e18a4..5996d98075 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -6,27 +6,20 @@ import DrawerContent from './common/drawer/DrawerContent.svelte' import Select from './select/Select.svelte' import { ArrowLeft, Expand, LoaderCircle, Minimize, RefreshCcw } from 'lucide-svelte' - import type { DbInput } from './dbTypes' import DBManagerContent from './DBManagerContent.svelte' import { resource } from 'runed' + import { untrack } from 'svelte' + import type { DbManagerUriState } from './dbManagerDrawerModel.svelte' interface Props { + uriState: DbManagerUriState /** Z-index offset for the drawer, useful when opening from within modals */ offset?: number } - let { offset = 0 }: Props = $props() + let { uriState, offset = 0 }: Props = $props() - let input: DbInput | undefined = $state() - let open = $derived(!!input) - - // For datatable inputs, track the selected datatable separately - let selectedDatatable = $state(undefined) - - // Check if input is a datatable type - const isDatatableInput = $derived( - input?.type === 'database' && input.resourcePath.startsWith('datatable://') - ) + let open = $derived(uriState.open) // Load available datatables when drawer opens with datatable input const datatables = resource([], async () => { @@ -39,16 +32,6 @@ } }) - // Computed input that updates when selectedDatatable changes - const effectiveInput: DbInput | undefined = $derived.by(() => { - if (!input) return undefined - if (!isDatatableInput || !selectedDatatable) return input - return { - ...input, - resourcePath: `datatable://${selectedDatatable}` - } - }) - const datatableItems = $derived( datatables.current.map((dt) => ({ value: dt, @@ -56,32 +39,26 @@ })) ) - export function openDrawer(nInput: DbInput) { - input = nInput - if (isDatatableInput) { - datatables.refetch() + // Refetch datatables when switching to a datatable input + $effect(() => { + if (uriState.isDatatableInput) { + untrack(() => datatables.refetch()) } - // If it's a datatable input, extract the datatable name for the selector - if (nInput.type === 'database' && nInput.resourcePath.startsWith('datatable://')) { - selectedDatatable = nInput.resourcePath.replace('datatable://', '') - datatables.refetch() - } else { - selectedDatatable = undefined - } - } - export function closeDrawer() { - input = undefined - selectedDatatable = undefined + }) + + function handleClose() { + uriState.closeDrawer() dbManagerContent?.clearReplResult() - if (window.location.hash.startsWith('#dbmanager:')) - history.replaceState('', document.title, window.location.href.replace(/#dbmanager:.*$/, '')) } let windowWidth = $state(window.innerWidth) let expand = $state(false) $effect(() => { - if (!open) expand = false + if (!open) { + expand = false + uriState.closeDrawer() + } }) let dbManagerContent: DBManagerContent | undefined = $state() @@ -96,7 +73,7 @@ size={expand ? `${windowWidth}px` : '1200px'} preventEscape {offset} - on:close={closeDrawer} + on:close={handleClose} > - {#if effectiveInput && $workspaceStore} - {#key selectedDatatable} - + {#if uriState.effectiveInput && $workspaceStore} + {#key uriState.selectedDatatable} + {#snippet dbSelector()} - {#if isDatatableInput} + {#if uriState.isDatatableInput} {#if datatables.loading}
@@ -125,7 +108,7 @@ updateEnvValue(entry.key, e.currentTarget.value, 'string')} - disabled={noEditor} - class="input w-full" - placeholder="Variable value" - /> +
+ + updateEnvValue(entry.key, e.currentTarget.value, 'string')} + disabled={noEditor} + class="input w-full" + placeholder="Variable value" + /> + {#if !noEditor} +
+ {#if typeof entry.value === 'string' && entry.value.startsWith('$var:') && entry.value.length > 5} +
+ Linked to variable {entry.value.slice(5)} +
+ {/if} {/if} -
+ {/each} @@ -285,3 +367,20 @@ + + { + if (pickForKey) { + setVarPath(pickForKey, path) + pickForKey = undefined + } + }} + itemName="Variable" + extraField="path" + loadItems={async () => + (await VariableService.listVariable({ workspace: $workspaceStore ?? '' })).map((x) => ({ + name: x.path, + ...x + }))} +/> diff --git a/frontend/src/lib/components/flows/content/FlowLoop.svelte b/frontend/src/lib/components/flows/content/FlowLoop.svelte index f087079356..886f57386f 100644 --- a/frontend/src/lib/components/flows/content/FlowLoop.svelte +++ b/frontend/src/lib/components/flows/content/FlowLoop.svelte @@ -425,7 +425,6 @@ on:blur={() => { iteratorFieldFocused = false }} - autofocus lang="javascript" bind:code={mod.value.iterator.expr} class="h-full" diff --git a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte index b14f090afc..7efb44b82d 100644 --- a/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowSelectionPanel.svelte @@ -2,14 +2,29 @@ import FlowCard from '../common/FlowCard.svelte' import type { SelectionManager } from '$lib/components/graph/selectionUtils.svelte' import { Button } from '$lib/components/common' + import DropdownV2 from '$lib/components/DropdownV2.svelte' import { getNoteEditorContext } from '$lib/components/graph/noteEditor.svelte' - import { StickyNote } from 'lucide-svelte' + import { StickyNote, Move, Copy, Trash2 } from 'lucide-svelte' + import type { Item } from '$lib/utils' interface Props { selectionManager: SelectionManager noEditor: boolean + onDeleteSelected?: () => void + onDuplicateSelected?: () => void + onMoveSelected?: () => void + canMoveSelected?: boolean + resolvedCount?: number } - let { selectionManager, noEditor }: Props = $props() + let { + selectionManager, + noEditor, + onDeleteSelected, + onDuplicateSelected, + onMoveSelected, + canMoveSelected = false, + resolvedCount = 0 + }: Props = $props() const noteEditorContext = getNoteEditorContext() @@ -19,20 +34,44 @@ noteEditorContext.noteEditor.createGroupNote(selectionManager.selectedIds) } } + + let menuItems: Item[] = $derived([ + { + displayName: 'Move', + icon: Move, + action: () => onMoveSelected?.(), + disabled: !canMoveSelected + }, + { + displayName: 'Duplicate', + icon: Copy, + action: () => onDuplicateSelected?.() + }, + { + displayName: `Delete (${resolvedCount})`, + icon: Trash2, + type: 'delete', + action: () => onDeleteSelected?.() + } + ]) {#snippet action()} - +
+ + {#if resolvedCount > 0} + + {/if} +
{/snippet}
-

{selectionManager.selectedIds.length} nodes selected

{#each selectionManager.selectedIds as nodeId}
diff --git a/frontend/src/lib/components/flows/flowModuleNextId.ts b/frontend/src/lib/components/flows/flowModuleNextId.ts index 34bed860f8..e10c900982 100644 --- a/frontend/src/lib/components/flows/flowModuleNextId.ts +++ b/frontend/src/lib/components/flows/flowModuleNextId.ts @@ -17,3 +17,15 @@ export function nextId(flowState: FlowState, fullFlow: OpenFlow): string { }, 0) return numberToChars(max) } + +// Computes a copy id like "a2", "a3", etc. based on the original id +export function copyId(originalId: string, flowState: FlowState, fullFlow: OpenFlow): string { + const allIds = new Set(dfs(fullFlow.value.modules, (fm) => fm.id).concat(Object.keys(flowState))) + for (let n = 2; n < 10000; n++) { + const candidate = `${originalId}${n}` + if (!allIds.has(candidate)) { + return candidate + } + } + return `${originalId}10000` +} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 8836edb708..28c66c05ca 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -2,12 +2,13 @@ import { preventDefault, stopPropagation } from 'svelte/legacy' import Popover from '$lib/components/Popover.svelte' - import { classNames, type StateStore } from '$lib/utils' + import DropdownV2 from '$lib/components/DropdownV2.svelte' + import { classNames, type Item, type StateStore } from '$lib/utils' import { Bed, Database, Gauge, - Move, + EllipsisVertical, PhoneIncoming, Repeat, Square, @@ -20,7 +21,7 @@ Timer, Maximize2 } from 'lucide-svelte' - import { createEventDispatcher, getContext, onDestroy } from 'svelte' + import { createEventDispatcher, getContext } from 'svelte' import { fade } from 'svelte/transition' import type { FlowEditorContext } from '../types' import { twMerge } from 'tailwind-merge' @@ -48,6 +49,7 @@ import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' import DiffActionBar from './DiffActionBar.svelte' import { getGraphContext } from '$lib/components/graph/graphContext' + import MoveHandleButton from '$lib/components/graph/MoveHandleButton.svelte' interface Props { selected?: boolean @@ -69,7 +71,6 @@ id?: string | undefined label: string path?: string - modType?: string | undefined nodeState?: FlowNodeState concurrency?: boolean // TODO: Implement for this one. See how concurrency is implemented. @@ -89,6 +90,7 @@ isOwner?: boolean enableTestRun?: boolean maximizeSubflow?: () => void + menuItems?: Item[] } let { @@ -106,7 +108,6 @@ id = undefined, label, path = '', - modType = undefined, nodeState, concurrency = false, debouncing = false, @@ -123,7 +124,8 @@ onEditInput, flowJob, enableTestRun = false, - maximizeSubflow = undefined + maximizeSubflow = undefined, + menuItems = undefined }: Props = $props() // AI action colors take priority over execution state @@ -138,6 +140,9 @@ const diffManager = flowGraphContext?.diffManager const moveManager = flowGraphContext?.moveManager + // Hide per-node action buttons when multiple nodes are selected (multi-select mode) + let isMultiSelected = $derived((flowGraphContext?.selectionManager?.selectedIds?.length ?? 0) > 1) + let pickableIds: Record | undefined = $state(undefined) const dispatch = createEventDispatcher() @@ -161,6 +166,7 @@ let outputPicker: OutputPicker | undefined = $state(undefined) let testJob: any | undefined = $state(undefined) let outputPickerBarOpen = $state(false) + let dropdownOpen = $state(false) let flowStateStore = $derived(flowEditorContext?.flowStateStore) @@ -188,42 +194,6 @@ let isDragging = $derived(!!moveManager?.dragging) - // --- Drag handle logic --- - let dragCleanup: (() => void) | undefined - - function onMovePointerDown(e: Event) { - const pe = e as PointerEvent - const startX = pe.clientX - const startY = pe.clientY - let didDrag = false - - function onMovePointer(me: PointerEvent) { - const dx = me.clientX - startX - const dy = me.clientY - startY - if (!didDrag && Math.sqrt(dx * dx + dy * dy) > 5) { - didDrag = true - if (moveManager && id) { - moveManager.startDrag(id, startX, startY) - } - } - } - - function onUp() { - document.removeEventListener('pointermove', onMovePointer) - document.removeEventListener('pointerup', onUp) - dragCleanup = undefined - if (!didDrag) { - dispatch('move') - } - } - - document.addEventListener('pointermove', onMovePointer) - document.addEventListener('pointerup', onUp) - dragCleanup = onUp - } - - onDestroy(() => dragCleanup?.()) - const outputPickerVisible = $derived( editMode && (isConnectingCandidate || alwaysShowOutputPicker) && !!id && !isDragging ) @@ -467,6 +437,7 @@ {deletable} {bold} bind:editId + disableEditId={isMultiSelected} {hover} {colorClasses} > @@ -475,7 +446,7 @@ {/snippet} - {#if outputPickerVisible} + {#if outputPickerVisible && !isMultiSelected} -
- -
- {/if} - -
- -
- {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)} {/if} + + {#if !isMultiSelected && id !== 'preprocessor' && moveManager && id} +
+ dispatch('move')} + class="trash group-hover:block" + /> +
+ {/if} + + {#if !isMultiSelected && menuItems && menuItems.length > 0} +
+ + {#snippet buttonReplacement()} + + {/snippet} + +
+ {/if} {:else if maximizeSubflow !== undefined} {@render buttonMaximizeSubflow?.()} {/if} @@ -603,7 +576,7 @@ onmouseenter={() => (hover = true)} onmouseleave={() => (hover = false)} > - {#if (hover || selected || testRunDropdownOpen) && outputPickerVisible} + {#if !isMultiSelected && (hover || selected || testRunDropdownOpen) && outputPickerVisible}
{#if !testIsLoading}
diff --git a/frontend/src/lib/components/flows/map/MapItem.svelte b/frontend/src/lib/components/flows/map/MapItem.svelte index 417ce58346..1a65452873 100644 --- a/frontend/src/lib/components/flows/map/MapItem.svelte +++ b/frontend/src/lib/components/flows/map/MapItem.svelte @@ -5,7 +5,7 @@ import FlowModuleSchemaItem from './FlowModuleSchemaItem.svelte' import FlowModuleIcon from '../FlowModuleIcon.svelte' import { prettyLanguage } from '$lib/common' - import { msToSec } from '$lib/utils' + import { msToSec, type Item } from '$lib/utils' import FlowJobsMenu from './FlowJobsMenu.svelte' import { isTriggerStep, @@ -47,6 +47,7 @@ flowJob?: Job | undefined isOwner?: boolean maximizeSubflow?: () => void + menuItems?: Item[] } let { @@ -67,7 +68,8 @@ onEditInput, flowJob, isOwner = false, - maximizeSubflow + maximizeSubflow, + menuItems = undefined }: Props = $props() const { selectionManager, moveManager } = getGraphContext() @@ -117,12 +119,11 @@ : '' : '' ) - {#if mod}
- {#if moveManager?.movingModuleId == mod.id} + {#if moveManager?.movingModuleId == mod.id && !moveManager?.movingIds?.includes(mod.id)}
{/if}
- {#if outputPickerVisible} + {#if outputPickerVisible && !isMultiSelected} - {#if outputPickerVisible} + {#if outputPickerVisible && !isMultiSelected}
> { + const ancestors = new Map>() + + function walk(mods: FlowModule[], parentAncestors: Set) { + for (const mod of mods) { + ancestors.set(mod.id, parentAncestors) + const childAncestors = new Set([...parentAncestors, mod.id]) + + const val = mod.value + if (val.type === 'forloopflow' || val.type === 'whileloopflow') { + walk(val.modules, childAncestors) + } else if (val.type === 'branchall') { + for (const branch of val.branches) walk(branch.modules, childAncestors) + } else if (val.type === 'branchone') { + for (const branch of val.branches) walk(branch.modules, childAncestors) + walk(val.default, childAncestors) + } + } + } + + walk(modules, new Set()) + return ancestors +} + +/** + * Filter raw selected node IDs down to the minimal set of top-level module IDs: + * 1. Filter out virtual graph nodes (Input, Result, Trigger, -start, -end, -branch-*, subflow:*, preprocessor, failure) + * 2. Verify each ID exists as a real module in the flow module tree + * 3. Deduplicate nested: if a container (loop/branch) AND its children are both selected, keep only the container + */ +export function resolveSelectedModuleIds(rawIds: string[], modules: FlowModule[]): string[] { + // Step 1: Filter out virtual IDs + const candidateIds = rawIds.filter((id) => !isVirtualId(id)) + + // Step 2+3: Single DFS to build ancestor map (also verifies existence) + const ancestorMap = buildAncestorMap(modules) + const verifiedIds = candidateIds.filter((id) => ancestorMap.has(id)) + + // If any ancestor of this module is also selected, it's a nested child — drop it + const selectedSet = new Set(verifiedIds) + return verifiedIds.filter((id) => { + const ancestors = ancestorMap.get(id)! + for (const ancestor of ancestors) { + if (selectedSet.has(ancestor)) return false + } + return true + }) +} + +export type ModuleLocation = { + id: string + parentArray: FlowModule[] + index: number +} + +/** + * For each ID, find its parent array (reference) and index using DFS. + */ +export function locateModules(ids: string[], modules: FlowModule[]): ModuleLocation[] { + const idSet = new Set(ids) + const locations: ModuleLocation[] = [] + + dfs(modules, (mod, parentModules) => { + if (idSet.has(mod.id)) { + const index = parentModules.findIndex((m) => m.id === mod.id) + if (index !== -1) { + locations.push({ id: mod.id, parentArray: parentModules, index }) + } + } + }) + + return locations +} + +/** + * Group locations that share the same parent array, sorted by index within each group. + */ +export function groupByParent(locations: ModuleLocation[]): ModuleLocation[][] { + const groups = new Map() + for (const loc of locations) { + const existing = groups.get(loc.parentArray) + if (existing) { + existing.push(loc) + } else { + groups.set(loc.parentArray, [loc]) + } + } + // Sort each group by index + for (const group of groups.values()) { + group.sort((a, b) => a.index - b.index) + } + return Array.from(groups.values()) +} + +/** + * True if all locations share the same parent array and have consecutive indices. + * Required for move to be valid. + */ +export function areContiguousSiblings(locations: ModuleLocation[]): boolean { + if (locations.length === 0) return false + if (locations.length === 1) return true + + // All must share the same parent + const parent = locations[0].parentArray + if (!locations.every((loc) => loc.parentArray === parent)) return false + + // Sort by index and check contiguity + const sorted = [...locations].sort((a, b) => a.index - b.index) + for (let i = 1; i < sorted.length; i++) { + if (sorted[i].index !== sorted[i - 1].index + 1) return false + } + return true +} diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 7f534a924a..234f053f7b 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -7,6 +7,8 @@ import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { Alert } from '$lib/components/common' + import { disableHubStore } from '$lib/stores' interface Props { filter?: string @@ -30,11 +32,22 @@ const dispatch = createEventDispatcher() + let hubNotAvailable = $state(false) + onMount(async () => { - hubApps = await loadHubApps() + if ($disableHubStore) return + const result = await loadHubApps() + if (result === undefined) { + hubNotAvailable = true + } else { + hubApps = result + } }) +{#if $disableHubStore} + +{:else} -{#if hubApps} +{#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + +{:else if hubApps} {#if filteredItems.length == 0} {:else} @@ -93,3 +110,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 2ab12aea36..1f3c659bf8 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -7,6 +7,8 @@ import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { Alert } from '$lib/components/common' + import { disableHubStore } from '$lib/stores' interface Props { filter?: string @@ -30,11 +32,22 @@ const dispatch = createEventDispatcher() + let hubNotAvailable = $state(false) + onMount(async () => { - hubFlows = await loadHubFlows() + if ($disableHubStore) return + const result = await loadHubFlows() + if (result === undefined) { + hubNotAvailable = true + } else { + hubFlows = result + } }) +{#if $disableHubStore} + +{:else} -{#if hubFlows} +{#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + +{:else if hubFlows} {#if filteredItems.length == 0} {:else} @@ -95,3 +112,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index b5ba87327e..7ec0a8ccc3 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -8,6 +8,7 @@ import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { disableHubStore } from '$lib/stores' interface Props { kind?: HubScriptKind & string @@ -47,6 +48,7 @@ ) async function getAllApps(filterKind: typeof kind) { + if ($disableHubStore) return try { hubNotAvailable = false allApps = ( @@ -67,6 +69,7 @@ filterKind: typeof kind, appFilter: string | undefined ) { + if ($disableHubStore) return try { loading = true hubNotAvailable = false @@ -138,6 +141,9 @@ }) +{#if $disableHubStore} + +{:else}
{@render children?.()}
@@ -156,7 +162,9 @@
{#if hubNotAvailable} - + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + {:else if (items.length > 0 && apps.length > 0) || !loading} {#if items.length == 0} @@ -204,3 +212,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 2091f76340..3d929dead7 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -24,7 +24,7 @@ []) : undefined } catch (err) { - sendUserToast('Failed to fetch hub scripts: ' + err, 'error') + console.error('Failed to fetch hub scripts:', err) return undefined } }, @@ -44,9 +44,10 @@ import { Circle, ExternalLink } from 'lucide-svelte' import Popover from '$lib/components/Popover.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' - import { hubBaseUrlStore, userStore } from '$lib/stores' + import { disableHubStore, hubBaseUrlStore, userStore } from '$lib/stores' import { get } from 'svelte/store' import Button from '$lib/components/common/button/Button.svelte' + import { Alert } from '$lib/components/common' let hubNotAvailable = $state(false) @@ -94,13 +95,14 @@ }) async function getAllApps(filterKind: typeof kind) { + if ($disableHubStore) return try { hubNotAvailable = false allApps = (await listHubIntegrationsCached({ kind: filterKind, refreshCount })).map( (x) => x.name ) } catch (err) { - sendUserToast('Failed to fetch hub integrations: ' + err, 'error') + console.error('Failed to fetch hub integrations:', err) allApps = [] hubNotAvailable = true } @@ -112,7 +114,9 @@ ) $effect(() => { ;[filter, kind, appFilter, refreshCount] - hubScriptsFilteredPromise.refresh() + if (!$disableHubStore) { + hubScriptsFilteredPromise.refresh() + } }) $effect(() => { loading = hubScriptsFilteredPromise.status === 'loading' @@ -175,9 +179,13 @@ -{#if hubNotAvailable} -
- Hub not available +{#if $disableHubStore} + +{:else if hubNotAvailable} +
+ + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. +
{:else if loading} {#each Array(15).fill(0) as _} diff --git a/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte b/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte index 6910fabbb5..ead5ad9ad3 100644 --- a/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte +++ b/frontend/src/lib/components/flows/propPicker/PropPickerWrapper.svelte @@ -27,7 +27,7 @@ import type { PickableProperties } from '../previousResults' import AnimatedButton from '$lib/components/common/button/AnimatedButton.svelte' import type { PropPickerContext } from '$lib/components/prop_picker' - import type { FlowEditorContext } from '../types' + interface Props { pickableProperties: PickableProperties | undefined @@ -67,9 +67,8 @@ const { flowPropPickerConfig } = getContext('PropPickerContext') flowPropPickerConfig.set(undefined) - const { flowStore } = getContext('FlowEditorContext') - let flow_env = $derived(pickableProperties?.flow_env || flowStore.val.value.flow_env) + setContext('PropPickerWrapper', { propPickerConfig, inputMatches, @@ -156,7 +155,6 @@ {extraResults} {displayContext} {error} - {flow_env} previousId={pickableProperties?.previousId} {pickableProperties} allowCopy={!notSelectable && !$propPickerConfig} diff --git a/frontend/src/lib/components/graph/DragCoordinator.svelte b/frontend/src/lib/components/graph/DragCoordinator.svelte index ed277da2f0..cba81e71d4 100644 --- a/frontend/src/lib/components/graph/DragCoordinator.svelte +++ b/frontend/src/lib/components/graph/DragCoordinator.svelte @@ -21,7 +21,15 @@ onMount(() => { moveManager.setScreenToFlowPosition(screenToFlowPosition) - moveManager.setComputeDraggedNodeIds((moduleId) => getSubflowNodeIds(moduleId, nodes, edges)) + moveManager.setComputeDraggedNodeIds((moduleIds) => { + const combined = new Set() + for (const id of moduleIds) { + for (const nodeId of getSubflowNodeIds(id, nodes, edges)) { + combined.add(nodeId) + } + } + return combined + }) }) $effect(() => { @@ -33,11 +41,17 @@ function onPointerUp(_e: PointerEvent) { const moduleId = moveManager.dragging?.moduleId + const selectedIds = moveManager.dragging?.selectedIds const zone = moveManager.endDrag() if (zone && moduleId) { - // Set movingModuleId directly (non-toggle) so the insert handler knows which module to relocate - moveManager.setMoving(moduleId) - // Then trigger the insert, which detects movingModuleId and performs the splice + // Set moving state so the insert handler knows which module(s) to relocate + if (selectedIds && selectedIds.length > 1) { + moveManager.movingModuleId = selectedIds[0] + moveManager.movingIds = selectedIds + } else { + moveManager.setMoving(moduleId) + } + // Then trigger the insert, which detects movingModuleId/movingIds and performs the splice eventHandlers.insert({ sourceId: zone.sourceId, targetId: zone.targetId, diff --git a/frontend/src/lib/components/graph/DragGhost.svelte b/frontend/src/lib/components/graph/DragGhost.svelte index 1ad0ca5a63..6d2e26e10e 100644 --- a/frontend/src/lib/components/graph/DragGhost.svelte +++ b/frontend/src/lib/components/graph/DragGhost.svelte @@ -46,8 +46,19 @@ return { x: n.position.x, y: n.position.y } } - function computeGhost(moduleId: string, allNodes: Node[], allEdges: Edge[]) { - const { sfNodes, sfEdges } = getSubflowNodesAndEdges(moduleId, allNodes, allEdges) + function computeGhost(moduleId: string, draggedNodeIds: Set, allNodes: Node[], allEdges: Edge[]) { + // Use pre-computed draggedNodeIds when available (covers multi-select), + // otherwise fall back to single-module subflow computation. + let sfNodes: Node[] + let sfEdges: Edge[] + if (draggedNodeIds.size > 0) { + sfNodes = allNodes.filter((n) => draggedNodeIds.has(n.id)) + sfEdges = allEdges.filter((e) => draggedNodeIds.has(e.source) && draggedNodeIds.has(e.target)) + } else { + const result = getSubflowNodesAndEdges(moduleId, allNodes, allEdges) + sfNodes = result.sfNodes + sfEdges = result.sfEdges + } if (sfNodes.length === 0) return undefined // Compute bounding box using absolute positions @@ -112,8 +123,7 @@ let ghost = $derived.by(() => { const moduleId = moveManager.dragging?.moduleId if (!moduleId) return undefined - // Compute ghost once at drag start — don't react to node/edge changes during drag - return untrack(() => computeGhost(moduleId, nodes, edges)) + return untrack(() => computeGhost(moduleId, moveManager.draggedNodeIds, nodes, edges)) }) diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index cc316630f5..b2346a1b84 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -78,6 +78,11 @@ import { computeNoteNodes } from './noteUtils.svelte' import { Tooltip } from '../meltComponents' import { getNoteEditorContext } from './noteEditor.svelte' + import { + resolveSelectedModuleIds, + locateModules, + areContiguousSiblings + } from '../flows/multiSelectUtils' let useDataflow: Writable = writable(false) let showAssets: Writable = writable(true) @@ -131,6 +136,10 @@ notes?: FlowNote[] chatInputEnabled?: boolean multiSelectEnabled?: boolean + onDeleteMultiple?: (ids: string[]) => void + onDuplicateMultiple?: (ids: string[]) => void + onMoveMultiple?: (ids: string[]) => void + movingIds?: string[] onDelete?: (id: string) => void onInsert?: (detail: { sourceId?: string @@ -150,6 +159,7 @@ onDeleteBranch?: (detail: { id: string; index: number }) => Promise onChangeId?: (detail: { id: string; newId: string; deps: Record }) => void onMove?: (id: string) => void + onDuplicate?: (id: string) => void onUpdateMock?: (detail: { mock: FlowModule['mock']; id: string }) => void onTestUpTo?: ((id: string) => void) | undefined onSelectedIteration?: onSelectedIteration @@ -175,6 +185,7 @@ onInsert = undefined, onDelete = undefined, onMove = undefined, + onDuplicate = undefined, onDeleteBranch = undefined, onNewBranch = undefined, onSelect = undefined, @@ -231,7 +242,11 @@ diffBeforeFlow = undefined, currentInputSchema = undefined, markRemovedAsShadowed = false, - multiSelectEnabled = false + multiSelectEnabled = false, + onDeleteMultiple = undefined, + onDuplicateMultiple = undefined, + onMoveMultiple = undefined, + movingIds = undefined }: Props = $props() // Initialize note manager with fine-grained reactivity @@ -428,6 +443,9 @@ move: (detail) => { onMove?.(detail.id) }, + duplicate: (detail) => { + onDuplicate?.(detail.id) + }, selectedIteration: (detail) => { onSelectedIteration?.(detail) }, @@ -508,6 +526,15 @@ let canUseDiffDrawer = $derived(diffBeforeFlow || moduleActions || editMode) + // Derived state for multi-select operations + let resolvedModuleIds = $derived( + resolveSelectedModuleIds(selectionManager.selectedIds, effectiveModules ?? []) + ) + let canMoveSelected = $derived( + resolvedModuleIds.length > 0 && + areContiguousSiblings(locateModules(resolvedModuleIds, effectiveModules ?? [])) + ) + // Initialize moduleTracker with effectiveModules let moduleTracker = $state(new ChangeTracker([])) @@ -566,6 +593,31 @@ exitNoteMode?.() } } + if ((event.key === 'Backspace' || event.key === 'Delete') && editMode) { + const active = document.activeElement + if (active && active !== document.body && !flowContainer?.contains(active)) { + return + } + if ( + active instanceof HTMLInputElement || + active instanceof HTMLTextAreaElement || + active?.getAttribute('contenteditable') === 'true' + ) { + return + } + if (noteManager.selectedNoteId && noteEditorContext) { + noteEditorContext.noteEditor.deleteNote(noteManager.selectedNoteId) + noteManager.clearNoteSelection() + return + } + if (resolvedModuleIds.length > 1) { + onDeleteMultiple?.(resolvedModuleIds) + } else if (resolvedModuleIds.length === 1) { + onDelete?.(resolvedModuleIds[0]) + } else if (selectedId) { + onDelete?.(selectedId) + } + } } async function updateStores() { @@ -967,6 +1019,7 @@ elevateNodesOnSelect={false} {proOptions} multiSelectionKey={'Shift'} + deleteKey={null} nodesDraggable={false} --background-color={false} > @@ -978,8 +1031,17 @@ {#if multiSelectEnabled} + nodesWithOffset.some(n => n.id === id) + )} allNodes={nodesWithOffset as (Node & { type: string })[]} + onDeleteSelected={() => onDeleteMultiple?.(resolvedModuleIds)} + onDuplicateSelected={() => onDuplicateMultiple?.(resolvedModuleIds)} + onMoveSelected={() => onMoveMultiple?.(resolvedModuleIds)} + onCancelMove={() => onMoveMultiple?.(movingIds ?? [])} + {canMoveSelected} + isMoving={movingIds != null && movingIds.length > 0} + {resolvedModuleIds} /> {/if} @@ -1094,4 +1156,8 @@ display: none; pointer-events: none; } + + :global(.svelte-flow__selection-wrapper) { + pointer-events: none !important; + } diff --git a/frontend/src/lib/components/graph/MoveHandleButton.svelte b/frontend/src/lib/components/graph/MoveHandleButton.svelte new file mode 100644 index 0000000000..d077f0d03d --- /dev/null +++ b/frontend/src/lib/components/graph/MoveHandleButton.svelte @@ -0,0 +1,74 @@ + + + diff --git a/frontend/src/lib/components/graph/SelectionBoundingBox.svelte b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte index 5faff92d28..e1248b52ef 100644 --- a/frontend/src/lib/components/graph/SelectionBoundingBox.svelte +++ b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte @@ -1,23 +1,46 @@ -{#if bounds() && selectedNodes.length > 1} - {@const currentBounds = bounds()!} +{#if bounds && selectedNodes.length > 1} + {@const currentBounds = bounds!}
- - {#if noteEditorContext?.noteEditor} -
- - Create group note ({selectedNodes.length} nodes) - -
- {/if} + {:else if resolvedCount > 0} + {#if canMoveSelected && moveManager && resolvedModuleIds.length > 0} +
+ onMoveSelected?.()} + /> +
+ {/if} + + {#snippet buttonReplacement()} + + {/snippet} + + {/if} +
{/if} diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index e1714f6d33..9b578030e2 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -56,6 +56,7 @@ export type GraphEventHandlers = { delete: (detail: { id: string }, label: string) => void newBranch: (id: string) => void move: (detail: { id: string }) => void + duplicate: (detail: { id: string }) => void selectedIteration: onSelectedIteration changeId: (newId: string) => void simplifyFlow: (b: boolean) => void diff --git a/frontend/src/lib/components/graph/moveManager.svelte.ts b/frontend/src/lib/components/graph/moveManager.svelte.ts index 7f13c44171..605c4b849a 100644 --- a/frontend/src/lib/components/graph/moveManager.svelte.ts +++ b/frontend/src/lib/components/graph/moveManager.svelte.ts @@ -22,6 +22,7 @@ export type DropZoneRegistration = { type DragInfo = { moduleId: string + selectedIds?: string[] } /** @@ -54,22 +55,13 @@ export function getSubflowNodeIds( // Include child nodes (e.g. asset/AI tool nodes) of nodes added via edges. // Nodes found through disableMoveIds (like inner module "b") may have children // ("b-asset-in-...") that weren't caught by the initial prefix match on moduleId. - // Only scan children of edge-added nodes that aren't already covered by the - // moduleId prefix (those children were already captured in the first pass). - const newFromEdges: string[] = [] - for (const id of nodeIds) { - if (id !== moduleId && !id.startsWith(nodeIdPrefix)) { - newFromEdges.push(id) - } - } - if (newFromEdges.length > 0) { - for (const n of allNodes) { - if (!nodeIds.has(n.id)) { - for (const id of newFromEdges) { - if (n.id.startsWith(id + '-')) { - nodeIds.add(n.id) - break - } + const edgeMatchedIds = [...nodeIds] + for (const n of allNodes) { + if (!nodeIds.has(n.id)) { + for (const id of edgeMatchedIds) { + if (n.id.startsWith(id + '-')) { + nodeIds.add(n.id) + break } } } @@ -88,35 +80,57 @@ export class MoveManager { /** The module ID currently being moved via legacy click-to-move */ movingModuleId = $state(undefined) + /** Multiple module IDs being moved together (multi-select move) */ + movingIds = $state(undefined) + toggleMoving(id: string) { if (this.movingModuleId === id) { this.movingModuleId = undefined this.#updateDraggedNodeIds(undefined) } else { this.movingModuleId = id - this.#updateDraggedNodeIds(id) + this.#updateDraggedNodeIds([id]) + } + } + + toggleMovingMultiple(ids: string[]) { + if ( + this.movingIds && + this.movingIds.length === ids.length && + this.movingIds.every((id, i) => id === ids[i]) + ) { + this.movingModuleId = undefined + this.movingIds = undefined + this.#updateDraggedNodeIds(undefined) + } else { + this.movingModuleId = ids[0] + this.movingIds = ids + this.#updateDraggedNodeIds(ids) } } setMoving(id: string) { this.movingModuleId = id - this.#updateDraggedNodeIds(id) + this.#updateDraggedNodeIds([id]) } clearMoving() { this.movingModuleId = undefined + this.movingIds = undefined this.#updateDraggedNodeIds(undefined) } - #computeDraggedNodeIds: ((moduleId: string) => Set) | undefined + #computeDraggedNodeIds: ((moduleIds: string[]) => Set) | undefined - setComputeDraggedNodeIds(fn: (moduleId: string) => Set) { + setComputeDraggedNodeIds(fn: (moduleIds: string[]) => Set) { this.#computeDraggedNodeIds = fn } - #updateDraggedNodeIds(moduleId: string | undefined) { + #updateDraggedNodeIds(moduleIds: string[] | undefined) { this.draggedNodeIds = - moduleId && this.#computeDraggedNodeIds ? this.#computeDraggedNodeIds(moduleId) : new Set() + moduleIds && moduleIds.length > 0 && this.#computeDraggedNodeIds + ? this.#computeDraggedNodeIds(moduleIds) + : new Set() } #screenToFlowPosition: ((pos: { x: number; y: number }) => { x: number; y: number }) | undefined @@ -134,14 +148,19 @@ export class MoveManager { this.#registeredDropZones.delete(edgeId) } - startDrag(moduleId: string, screenX: number, screenY: number) { + startDrag(moduleId: string, screenX: number, screenY: number, selectedIds?: string[]) { // Clear any active click-to-move so only drag mode is active this.movingModuleId = undefined - this.dragging = { moduleId } + this.dragging = { moduleId, selectedIds } this.ghostScreenX = screenX this.ghostScreenY = screenY this.nearestDropZone = undefined - this.#updateDraggedNodeIds(moduleId) + // Compute dragged node IDs for the primary module plus any additional selected modules + const allIds = + selectedIds && selectedIds.length > 0 + ? [moduleId, ...selectedIds.filter((id) => id !== moduleId)] + : [moduleId] + this.#updateDraggedNodeIds(allIds) } updateDrag(screenX: number, screenY: number) { diff --git a/frontend/src/lib/components/graph/noteManager.svelte.ts b/frontend/src/lib/components/graph/noteManager.svelte.ts index 6e605327a8..3ce16495a8 100644 --- a/frontend/src/lib/components/graph/noteManager.svelte.ts +++ b/frontend/src/lib/components/graph/noteManager.svelte.ts @@ -133,6 +133,13 @@ export class NoteManager { } } + /** + * Get the currently selected note ID + */ + get selectedNoteId(): string | undefined { + return this.#selectedNoteId + } + /** * Check if a note is currently selected */ diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index de74cd22cd..8f1ca0017f 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -213,7 +213,7 @@ {#if moveManager?.movingModuleId && data?.insertable}
- {#if !data.disableMoveIds?.includes(moveManager.movingModuleId)} + {#if !(moveManager.movingIds ?? [moveManager.movingModuleId]).some((id) => data.disableMoveIds?.includes(id))}
diff --git a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte index 27cb72457c..01953f2d8b 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte @@ -1,12 +1,11 @@ - + {#snippet children({ darkMode })} @@ -22,10 +24,27 @@ offset = 0, wrapperClass = '', contextMenuItems = undefined, + menuItems = undefined, nodeId = undefined, children }: Props = $props() + let resolvedContextMenuItems: ContextMenuItem[] | undefined = $derived( + contextMenuItems ?? + menuItems?.flatMap((item) => [ + ...(item.separatorTop ? [{ id: `${item.displayName}-divider`, label: '', divider: true }] : []), + { + id: item.displayName, + label: item.displayName, + icon: item.icon, + disabled: item.disabled, + type: item.type, + shortcut: item.shortcut, + onClick: item.action as (() => void) | undefined + } + ]) + ) + const { moveManager } = getGraphContext() let faded = $derived( @@ -37,8 +56,8 @@ -{#if contextMenuItems && contextMenuItems.length > 0} - +{#if resolvedContextMenuItems && resolvedContextMenuItems.length > 0} +
{@render children?.({ darkMode })}
diff --git a/frontend/src/lib/components/graph/selectionUtils.svelte.ts b/frontend/src/lib/components/graph/selectionUtils.svelte.ts index 8abc7fa563..daa9de3ecf 100644 --- a/frontend/src/lib/components/graph/selectionUtils.svelte.ts +++ b/frontend/src/lib/components/graph/selectionUtils.svelte.ts @@ -78,13 +78,24 @@ export class SelectionManager { } // If the new selection is the same as the current selection, do nothing - if (JSON.stringify(nodes) === JSON.stringify($state.snapshot(this.#selectedNodes))) { + const newIds = nodes.map((n) => n.id).join(',') + const currentIds = this.#selectedNodes.map((n) => n.id).join(',') + if (newIds === currentIds) { return } this.#selectedNodes = nodes } + // Select multiple nodes by their IDs + selectByIds(ids: string[]) { + if (!ids || ids.length === 0) { + this.clearSelection() + return + } + this.#selectedNodes = ids.map((id) => ({ id })) + } + // Clear all selections clearSelection() { this.#selectedNodes = [{ id: 'settings' }] diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index de4cd1528b..25955b74db 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -331,6 +331,16 @@ export const settings: Record = { storage: 'setting', ee_only: '', hiddenIfEmpty: true + }, + { + label: 'Disable Hub', + description: + 'Disable the Windmill Hub integration entirely. Enable this if your instance runs in a closed environment without internet access and you do not have a private hub setup.', + key: 'disable_hub', + fieldType: 'boolean', + storage: 'setting', + ee_only: '', + requiresReloadOnChange: true } ], SMTP: [ diff --git a/frontend/src/lib/components/propertyPicker/PropPicker.svelte b/frontend/src/lib/components/propertyPicker/PropPicker.svelte index d04683e226..6910750480 100644 --- a/frontend/src/lib/components/propertyPicker/PropPicker.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPicker.svelte @@ -19,7 +19,6 @@ error?: boolean allowCopy?: boolean previousId?: string | undefined - flow_env?: Record | undefined result?: any | undefined extraResults?: any } @@ -30,7 +29,6 @@ error = false, allowCopy = false, previousId = undefined, - flow_env = undefined, result = undefined, extraResults = undefined }: Props = $props() @@ -39,7 +37,6 @@ let resources: Record = $state({}) let displayVariable = $state(false) let displayResources = $state(false) - let displayFlowEnv = $state(false) let allResultsCollapsed = $state(true) let collapsableInitialState: @@ -47,7 +44,6 @@ allResultsCollapsed: boolean displayVariable: boolean displayResources: boolean - displayFlowEnv: boolean } | undefined @@ -139,7 +135,9 @@ resultByIdFiltered = {} } if (!$inputMatches?.some((match) => match.word === 'flow_env')) { - flowEnvFiltered = {} + if (search === EMPTY_STRING) { + flowEnvFiltered = pickableProperties.flow_env + } } if ($inputMatches?.length == 1) { filteringFlowInputsOrResult = $inputMatches[0].value @@ -185,8 +183,7 @@ collapsableInitialState = { allResultsCollapsed, displayVariable, - displayResources, - displayFlowEnv + displayResources } } @@ -200,10 +197,6 @@ displayResources = true return } - if ($inputMatches[0].word === 'flow_env') { - displayFlowEnv = true - return - } if ($inputMatches[0].word === 'results') { allResultsCollapsed = false return @@ -214,8 +207,7 @@ if (!collapsableInitialState) { return } - ;({ allResultsCollapsed, displayVariable, displayResources, displayFlowEnv } = - collapsableInitialState) + ;({ allResultsCollapsed, displayVariable, displayResources } = collapsableInitialState) collapsableInitialState = undefined } @@ -279,6 +271,18 @@ />
{/if} + {#if flowEnvFiltered && Object.keys(flowEnvFiltered ?? {}).length > 0} + Flow Env Variables +
+ +
+ {/if} {#if error} Error
@@ -445,45 +449,6 @@ {/if}
{/if} - {#if flow_env && Object.keys(flow_env).length > 0 && $inputMatches?.some((match) => match.word === 'flow_env')} -
- Flow Env Variables: - - {#if displayFlowEnv} - - - {:else} - - {/if} -
- {/if} {/if} diff --git a/frontend/src/lib/components/sidebar/FavoriteMenu.svelte b/frontend/src/lib/components/sidebar/FavoriteMenu.svelte index d0a14143ca..056bca1489 100644 --- a/frontend/src/lib/components/sidebar/FavoriteMenu.svelte +++ b/frontend/src/lib/components/sidebar/FavoriteMenu.svelte @@ -10,7 +10,7 @@ flow: `/flows/get/${path}`, app: `/apps/get/${path}`, raw_app: `/apps_raw/get/${path}`, - asset: `#dbmanager:${path}` + asset: '#' }[kind] } export function getFavoriteLabel(path: string, kind: FavoriteKind): string { diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte index b74d7c9b92..44a4dcc6bf 100644 --- a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbSelect.svelte @@ -8,14 +8,12 @@ import CustomInstanceDbWizardModal from './CustomInstanceDbWizardModal.svelte' import { ArrowRight, TriangleAlert } from 'lucide-svelte' import type { ConfirmationModalHandle } from '../common/confirmationModal/asyncConfirmationModal.svelte' - import DBManagerDrawer from '../DBManagerDrawer.svelte' import type { Snippet } from 'svelte' type Props = { value: string | undefined customInstanceDbs: ResourceReturn confirmationModal: ConfirmationModalHandle - dbManagerDrawer: DBManagerDrawer | undefined wizardBottomHint?: Snippet | undefined class?: string tag?: CustomInstanceDbTag @@ -24,7 +22,6 @@ value = $bindable(), customInstanceDbs, confirmationModal, - dbManagerDrawer, wizardBottomHint, class: className, tag @@ -90,7 +87,6 @@ confirmationModal: ConfirmationModalHandle - dbManagerDrawer: any | undefined bottomHint?: Snippet | undefined opened: { status: CustomInstanceDb | undefined; dbname: string } | undefined tag?: CustomInstanceDbTag @@ -33,7 +32,6 @@ let { customInstanceDbs, confirmationModal, - dbManagerDrawer, bottomHint, opened = $bindable(), tag @@ -76,7 +74,6 @@ class="flex-1" asset={{ kind: 'resource', path: 'CUSTOM_INSTANCE_DB/' + dbname }} _resourceMetadata={{ resource_type: 'postgresql' }} - {dbManagerDrawer} disabled={!$isCustomInstanceDbEnabled || !enableManageButton} onClick={() => (opened = undefined)} /> diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 188d736e9f..6410e6a1af 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -62,7 +62,7 @@ import { random_adj } from '../random_positive_adjetive' import { sendUserToast } from '$lib/toast' import { SettingService, WorkspaceService, type GetSettingsResponse } from '$lib/gen' - import { globalDbManagerDrawer, workspaceStore } from '$lib/stores' + import { workspaceStore } from '$lib/stores' import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte' import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte' import { resource } from 'runed' @@ -141,7 +141,6 @@ } let confirmationModal = createAsyncConfirmationModal() - let dbManagerDrawer = $derived(globalDbManagerDrawer.val) let dirtyMap = $derived.by(() => { const map: Record = {} for (let i = 0; i < tempSettings.dataTables.length; i++) { @@ -245,7 +244,6 @@ @@ -275,7 +272,6 @@ {/if} diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index ac8c20377d..c264219922 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -68,7 +68,7 @@ import { SettingService, WorkspaceService } from '$lib/gen' import type { GetSettingsResponse } from '$lib/gen' - import { globalDbManagerDrawer, workspaceStore } from '$lib/stores' + import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import ExploreAssetButton from '../ExploreAssetButton.svelte' import Tooltip from '../Tooltip.svelte' @@ -187,7 +187,6 @@ 'Where the data is actually stored, in parquet format. You need to configure a workspace storage first' } - let dbManagerDrawer = $derived(globalDbManagerDrawer.val) let confirmationModal = createAsyncConfirmationModal() @@ -293,7 +292,6 @@ bind:value={ducklake.catalog.resource_path} {customInstanceDbs} {confirmationModal} - {dbManagerDrawer} tag="ducklake" > {#snippet wizardBottomHint()} @@ -372,7 +370,6 @@ {:else} {/if}
diff --git a/frontend/src/lib/infer.svelte.ts b/frontend/src/lib/infer.svelte.ts index 419020631c..cbdfa95505 100644 --- a/frontend/src/lib/infer.svelte.ts +++ b/frontend/src/lib/infer.svelte.ts @@ -4,6 +4,13 @@ import { ChangeOnDeepInequality, MapResource } from './svelte5Utils.svelte' import { sqlDataTypeToJsTypeHeuristic } from './components/apps/components/display/dbtable/utils' import { chunkBy, clone, getQueryStmtCountHeuristic } from './utils' +function extractErrorMessage(e: unknown): string { + if (e != null && typeof e === 'object' && 'body' in e) { + return (e as any).body?.error?.message ?? JSON.stringify(e) + } + return e instanceof Error ? e.message : JSON.stringify(e) +} + function computeQueryKey(query: InferAssetsSqlQueryDetails, workspace?: string) { return `${query.source_kind}::${query.source_name}::${query.source_schema}::${workspace}::${query.query_string}` } @@ -21,66 +28,26 @@ export function usePreparedAssetSqlQueries( ), async (toFetch) => { let queries = Object.entries(clone(toFetch)) - // We only support preparing datatable source kinds for now. - queries = queries.filter(([_, q]) => q.source_kind === 'datatable') + queries = queries.filter( + ([_, q]) => q.source_kind === 'datatable' || q.source_kind === 'ducklake' + ) // We only support preparing single-statement queries for now. queries = queries.filter(([_, q]) => getQueryStmtCountHeuristic(q.query_string) === 1) if (!queries?.length) return {} - try { - // We chunk by source_name to minimize the number of requests. - // For example if we have 10 queries on the same data table, - // we can prepare them all with a single script. - queries.sort((a, b) => a[1].source_name.localeCompare(b[1].source_name)) - let results = ( - await Promise.all( - chunkBy(queries, ([key, q]) => q.source_name).map(async (chunk) => { - console.log( - 'Preparing chunk of queries:', - chunk.map(([_, q]) => q) - ) - let queryContent = chunk - .flatMap(([key, q]) => [ - q.source_schema ? `SET search_path TO ${q.source_schema};` : 'RESET search_path;', - q.query_string + (q.query_string.trim().endsWith(';') ? '' : ';') - ]) - .join('\n') - queryContent = - '-- prepare\n--result_collection=all_statements_first_row\n' + queryContent + let datatableQueries = queries.filter(([_, q]) => q.source_kind === 'datatable') + let ducklakeQueries = queries.filter(([_, q]) => q.source_kind === 'ducklake') - let res = (await JobService.runScriptPreviewAndWaitResult({ - workspace: getWorkspace()!, - requestBody: { - language: 'postgresql', - content: queryContent, - args: { database: `datatable://${chunk[0][1]?.source_name}` } - } - })) as { error?: string; columns?: { name: string; type: string }[] }[] + let allResults: [string, PreparedAssetsSqlQuery][] = [] - console.log('Prepared query content:', res) - - let res2: [string, PreparedAssetsSqlQuery][] = res.map((r, i) => [ - chunk[i][0], - r.columns - ? { - columns: Object.fromEntries( - r.columns.map(({ name, type }) => [ - name, - sqlDataTypeToJsTypeHeuristic(type) - ]) - ) - } - : { error: r.error ?? "Couldn't prepare query " } - ]) - return res2 - }) - ) - ).flat() - - return Object.fromEntries(results) - } catch (e) { - throw e + if (datatableQueries.length) { + allResults.push(...(await prepareDatatableQueries(datatableQueries, getWorkspace))) } + if (ducklakeQueries.length) { + allResults.push(...(await prepareDucklakeQueries(ducklakeQueries, getWorkspace))) + } + + return Object.fromEntries(allResults) } ) @@ -96,3 +63,104 @@ export function usePreparedAssetSqlQueries( } } } + +type QueryEntry = [string, InferAssetsSqlQueryDetails] + +function mapPrepareResults( + res: { error?: string; columns?: { name: string; type: string }[] }[], + chunk: QueryEntry[] +): [string, PreparedAssetsSqlQuery][] { + if (res.length !== chunk.length) { + throw new Error(`Prepare results count mismatch: got ${res.length}, expected ${chunk.length}`) + } + return res.map((r, i) => [ + chunk[i]?.[0], + r.columns + ? { + columns: Object.fromEntries( + r.columns.map(({ name, type: t }) => [name, sqlDataTypeToJsTypeHeuristic(t)]) + ) + } + : { error: r.error ?? "Couldn't prepare query " } + ]) +} + +async function prepareDatatableQueries( + queries: QueryEntry[], + getWorkspace: () => string | undefined +): Promise<[string, PreparedAssetsSqlQuery][]> { + queries.sort((a, b) => a[1].source_name.localeCompare(b[1].source_name)) + let results = ( + await Promise.all( + chunkBy(queries, ([_, q]) => q.source_name).map(async (chunk) => { + let queryContent = chunk + .flatMap(([_, q]) => [ + q.source_schema ? `SET search_path TO ${q.source_schema};` : 'RESET search_path;', + q.query_string + (q.query_string.trim().endsWith(';') ? '' : ';') + ]) + .join('\n') + queryContent = '-- prepare\n--result_collection=all_statements_first_row\n' + queryContent + + try { + let res = (await JobService.runScriptPreviewAndWaitResult({ + workspace: getWorkspace()!, + requestBody: { + language: 'postgresql', + content: queryContent, + args: { database: `datatable://${chunk[0][1]?.source_name}` } + } + })) as { error?: string; columns?: { name: string; type: string }[] }[] + + return mapPrepareResults(res, chunk) + } catch (e) { + const error = extractErrorMessage(e) + return chunk.map(([key]) => [key, { error }] as [string, PreparedAssetsSqlQuery]) + } + }) + ) + ).flat() + return results +} + +async function prepareDucklakeQueries( + queries: QueryEntry[], + getWorkspace: () => string | undefined +): Promise<[string, PreparedAssetsSqlQuery][]> { + queries.sort((a, b) => a[1].source_name.localeCompare(b[1].source_name)) + let results = ( + await Promise.all( + chunkBy(queries, ([_, q]) => `${q.source_name}::${q.source_schema ?? ''}`).map( + async (chunk) => { + let sourceName = chunk[0][1].source_name + let sourceSchema = chunk[0][1].source_schema + let attachSetup = `ATTACH 'ducklake://${sourceName}' AS dl;\n` + attachSetup += sourceSchema ? `USE dl.${sourceSchema};\n` : `USE dl;\n` + + let queryContent = chunk + .map(([_, q]) => q.query_string + (q.query_string.trim().endsWith(';') ? '' : ';')) + .join('\n') + queryContent = + '-- prepare\n--result_collection=all_statements_first_row\n' + + attachSetup + + queryContent + + try { + let res = (await JobService.runScriptPreviewAndWaitResult({ + workspace: getWorkspace()!, + requestBody: { + language: 'duckdb', + content: queryContent, + args: {} + } + })) as { error?: string; columns?: { name: string; type: string }[] }[] + return mapPrepareResults(res, chunk) + } catch (e) { + const error = extractErrorMessage(e) + return chunk.map(([key]) => [key, { error }] as [string, PreparedAssetsSqlQuery]) + } + } + ) + ) + ).flat() + return results +} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index f8fc95c8f3..6485970f27 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -30,6 +30,9 @@ def main(): # wmill.setState(newState) # 4. Return the new rows # return range from (state to newState) + # + # For more complex states, consider using Data Tables: + # https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables return [1, 2, 3]` const PYTHON_INIT_CODE = `import os @@ -554,6 +557,9 @@ export async function main() { // await wmill.setState(newState) // 4. Return the new rows // return range from (state to newState) + // + // For more complex states, consider using Data Tables: + // https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables return [1,2,3] @@ -575,6 +581,9 @@ export async function main() { // await wmill.setState(newState) // 4. Return the new rows // return range from (state to newState) + // + // For more complex states, consider using Data Tables: + // https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables return [1,2,3] @@ -599,6 +608,9 @@ func main() (interface{}, error) { // 3. Compare the two states and update the internal state wmill.SetState(4) // 4. Return the new rows + // + // For more complex states, consider using Data Tables: + // https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables return state, nil diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index a397cb4b7f..fd0a66cfb3 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -14,6 +14,7 @@ import { import { getLocalSetting, type StateStore } from './utils' import { createState } from './svelte5Utils.svelte' import { DEFAULT_HUB_BASE_URL } from './hub' +import type { DbManagerUriState } from './components/dbManagerDrawerModel.svelte' export interface UserExt { email: string @@ -82,6 +83,7 @@ export const superadmin = writable(undefined) export const devopsRole = writable(undefined) export const lspTokenStore = writable(undefined) export const hubBaseUrlStore = writable(DEFAULT_HUB_BASE_URL) +export const disableHubStore = writable(false) export const userWorkspaces: Readable> = derived( [usersWorkspaceStore, superadmin], ([store, superadmin]) => { @@ -126,9 +128,7 @@ export const codeCompletionSessionEnabled = writable( export const usedTriggerKinds = writable([]) -export let globalDbManagerDrawer: StateStore = createState({ - val: undefined -}) +export let globalDbManagerDrawer: StateStore = { val: undefined } type SQLBaseSchema = { [schemaKey: string]: { diff --git a/frontend/src/lib/svelte5UtilsKit.svelte.ts b/frontend/src/lib/svelte5UtilsKit.svelte.ts index cadd2dee1e..921d41614e 100644 --- a/frontend/src/lib/svelte5UtilsKit.svelte.ts +++ b/frontend/src/lib/svelte5UtilsKit.svelte.ts @@ -107,9 +107,10 @@ export function useSearchParams(schema: S): SearchParamsRes } else { sp.set(key, serializeParam(v)) } + const hash = window.location.hash const newUrl = sp.toString() - ? `${window.location.pathname}?${sp}` - : window.location.pathname + ? `${window.location.pathname}?${sp}${hash}` + : `${window.location.pathname}${hash}` history.replaceState(history.state, '', newUrl) }, enumerable: true, diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 0847fc35cb..496dba427b 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -17,6 +17,7 @@ export { sendUserToast } import type { AnyMeltElement } from '@melt-ui/svelte' import type { TriggerKind } from './components/triggers' import { stateSnapshot } from './svelte5Utils.svelte' + export namespace OpenApi { export enum OpenApiVersion { V2, @@ -1495,6 +1496,7 @@ export type Item = { tooltip?: string separatorTop?: boolean submenuItems?: Item[] + shortcut?: string } export function isObjectTooBig(obj: any): boolean { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 4260bbeede..78de6c1535 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -11,13 +11,7 @@ UserService, WorkspaceService } from '$lib/gen' - import { - capitalize, - classNames, - getModifierKey, - parseDbInputFromAssetSyntax, - sendUserToast - } from '$lib/utils' + import { capitalize, classNames, getModifierKey, sendUserToast } from '$lib/utils' import WorkspaceMenu from '$lib/components/sidebar/WorkspaceMenu.svelte' import SidebarContent from '$lib/components/sidebar/SidebarContent.svelte' import CriticalAlertModal from '$lib/components/sidebar/CriticalAlertModal.svelte' @@ -32,6 +26,7 @@ type UserExt, defaultScripts, hubBaseUrlStore, + disableHubStore, usedTriggerKinds, devopsRole, whitelabelNameStore, @@ -66,8 +61,8 @@ import AiChatLayout from '$lib/components/copilot/chat/AiChatLayout.svelte' import { DEFAULT_HUB_BASE_URL } from '$lib/hub' import DBManagerDrawer from '$lib/components/DBManagerDrawer.svelte' - import { watchOnce } from 'runed' import { useIsDarkMode } from '$lib/components/DarkModeObserver.svelte' + import { useDbManagerUriState } from '$lib/components/dbManagerDrawerModel.svelte' interface Props { children?: import('svelte').Snippet } @@ -163,6 +158,7 @@ loadUsage() syncTutorialsTodos() loadHubBaseUrl() + loadDisableHub() loadUsedTriggerKinds() } @@ -182,6 +178,11 @@ DEFAULT_HUB_BASE_URL } + async function loadDisableHub() { + $disableHubStore = + ((await SettingService.getGlobal({ key: 'disable_hub' })) as boolean) ?? false + } + async function loadFavorites() { const scripts = await ScriptService.listScripts({ workspace: $workspaceStore ?? '', @@ -439,18 +440,8 @@ untrack(() => loadProtectionRules(workspace)) } }) - watchOnce( - () => globalDbManagerDrawer.val, - () => { - if (!globalDbManagerDrawer.val) return - const hash = window.location.hash - if (hash.startsWith('#dbmanager:')) { - const [_, path] = hash.split('#dbmanager:') - const dbInput = parseDbInputFromAssetSyntax(path) - if (dbInput) globalDbManagerDrawer.val?.openDrawer(dbInput) - } - } - ) + + globalDbManagerDrawer.val = useDbManagerUriState() @@ -786,6 +777,6 @@ {/if} -{#if $workspaceStore} - +{#if $workspaceStore && globalDbManagerDrawer.val} + {/if} diff --git a/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte b/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte index 6181204e3b..2bb6671a23 100644 --- a/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte @@ -41,10 +41,18 @@ const templateId = $page.url.searchParams.get('template_id') const hubId = $page.url.searchParams.get('hub') - const importRaw = $importStore + // Check in-memory store first, then sessionStorage (used when full page reload occurs) + let importRaw = $importStore if ($importStore) { $importStore = undefined } + if (!importRaw) { + const sessionData = sessionStorage.getItem('rawAppImport') + if (sessionData) { + sessionStorage.removeItem('rawAppImport') + importRaw = JSON.parse(sessionData) + } + } const appState = nodraft || hubId ? undefined : localStorage.getItem('rawapp') @@ -189,7 +197,7 @@ files: svelte5Template } ] - let templatePicker = $state(nodraft != null) + let templatePicker = $state(nodraft != null && !importRaw) let reloadCounter = $state(0) // Modal state diff --git a/frontend/src/routes/(root)/(logged)/assets/+page.svelte b/frontend/src/routes/(root)/(logged)/assets/+page.svelte index 7eac2cc535..032a60af66 100644 --- a/frontend/src/routes/(root)/(logged)/assets/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/assets/+page.svelte @@ -99,7 +99,6 @@ let assets = $derived(_assets.current?.flatMap((page) => page.assets)) let s3FilePicker: S3FilePicker | undefined = $state() - let dbManagerDrawer = $derived(globalDbManagerDrawer.val) as any let assetsUsageDropdown: AssetsUsageDrawer | undefined = $state() let allS3Storages = resource( @@ -192,7 +191,6 @@
@@ -337,7 +335,6 @@ {/if} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index f0ac8ae9d6..abf17b88bb 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -35,8 +35,7 @@ enterpriseLicense, userStore, workspaceStore, - userWorkspaces, - globalDbManagerDrawer + userWorkspaces } from '$lib/stores' import { sendUserToast } from '$lib/toast' import { @@ -557,8 +556,6 @@ } }) - let dbManagerDrawer = $derived(globalDbManagerDrawer.val) as any - let showTable = $derived( tab == 'workspace' || tab == 'states' || tab == 'cache' || tab == 'theme' ) @@ -1064,7 +1061,6 @@ {#if path && assetCanBeExplored({ kind: 'resource', path }, { resource_type }) && !$userStore?.operator} diff --git a/frontend/src/routes/(root)/(logged)/variables/+page.svelte b/frontend/src/routes/(root)/(logged)/variables/+page.svelte index 4082e5f8ff..870bf8e7c4 100644 --- a/frontend/src/routes/(root)/(logged)/variables/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/variables/+page.svelte @@ -40,7 +40,8 @@ EyeOff, Circle } from 'lucide-svelte' - import { untrack } from 'svelte' + import { onMount, untrack } from 'svelte' + import { page } from '$app/stores' type ListableVariableW = ListableVariable & { canWrite: boolean } @@ -202,6 +203,14 @@ loadContextualVariables() }, 5000) } + + onMount(() => { + let hash = $page.url.hash + if (hash.length > 1) { + let path = hash.slice(1) + variableEditor?.editVariable(path) + } + }) diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 2b181a6c7b..5031c7e889 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -96,9 +96,8 @@ components: type: boolean flow_env: type: object - description: Environment variables available to all steps - additionalProperties: - type: string + description: "Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource)." + additionalProperties: {} priority: type: number description: Execution priority (higher numbers run first) diff --git a/scripts/worktree-env b/scripts/worktree-env index ae83b09405..5fd8490bc2 100755 --- a/scripts/worktree-env +++ b/scripts/worktree-env @@ -61,7 +61,7 @@ if command -v psql &>/dev/null; then if psql "$db_conn/postgres" -tc "SELECT 1 FROM pg_database WHERE datname = '${db_name}'" 2>/dev/null | grep -q 1; then echo "Database $db_name already exists" else - if [[ -n "${WM_CLONE_DB:-}" ]]; then + if [[ "${WM_CLONE_DB:-}" == "1" || "${WM_CLONE_DB:-}" == "true" ]]; then # Terminate active connections so CREATE DATABASE ... TEMPLATE works psql "$db_conn/postgres" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'windmill' AND pid <> pg_backend_pid();" 2>/dev/null || true psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name} TEMPLATE windmill" 2>/dev/null \ @@ -178,13 +178,20 @@ if [ -n "$ee_repo" ]; then if [ -d "$ee_worktree_dir" ]; then ee_rel=$(python3 -c "import os; print(os.path.relpath('$ee_worktree_dir', '$(pwd)'))" 2>/dev/null || echo "$ee_worktree_dir") mkdir -p .claude + rust_plugin="" + if [[ "${USE_RUST_PLUGIN:-}" == "1" || "${USE_RUST_PLUGIN:-}" == "true" ]]; then + rust_plugin=', + "enabledPlugins": { + "rust-analyzer-lsp@claude-plugins-official": true + }' + fi cat > .claude/settings.local.json <