From e432f8234815f88690158e752326b698f30dfbc7 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 4 Mar 2026 14:44:33 +0000 Subject: [PATCH 01/57] fix: handle multipart stream errors gracefully instead of panicking (#8226) Co-authored-by: Claude Opus 4.6 --- backend/windmill-api/src/apps.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 1cfbfa6e19..9846912192 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -993,9 +993,18 @@ macro_rules! process_app_multipart { let mut uploaded_js = false; let mut multipart = $multipart; - while let Some(field) = multipart.next_field().await.unwrap() { - let name = field.name().unwrap().to_string(); - let data = field.bytes().await.unwrap(); + while let Some(field) = multipart + .next_field() + .await + .map_err(|e| Error::BadRequest(format!("failed to read multipart field: {e}")))? + { + let name = field + .name() + .ok_or_else(|| Error::BadRequest("multipart field missing name".to_string()))? + .to_string(); + let data = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("failed to read multipart stream: {e}")) + })?; if name == "app" { let app = serde_json::from_slice(&data).map_err(to_anyhow)?; let (ntx, npath, nid) = $internal_fn( @@ -2149,7 +2158,8 @@ async fn execute_component( (email.as_str(), permissioned_as) }; - let end_user_email = get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; + let end_user_email = + get_end_user_email(&db, opt_authed.as_ref(), tokened.token.as_deref()).await; let (uuid, mut tx) = push( &db, From 13f94b9677cfa3c13f1a0d5e8cde708cb24daca8 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 4 Mar 2026 15:53:56 +0100 Subject: [PATCH 02/57] fix: wrap set_encryption_key in a single database transaction (#8212) Prevent workspace corruption when re-encryption fails mid-loop by wrapping the key update and variable re-encryption in a single transaction. If any step fails, the entire operation rolls back. Co-authored-by: Claude Opus 4.6 --- backend/Cargo.lock | 1 + backend/windmill-api-workspaces/Cargo.toml | 1 + .../windmill-api-workspaces/src/workspaces.rs | 29 ++++++++++++++----- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d26d5b76d7..e52bb12fa1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16383,6 +16383,7 @@ dependencies = [ "http 1.4.0", "hyper 1.8.1", "lazy_static", + "magic-crypt", "regex", "serde", "serde_json", diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index b698426d53..a03bb3a490 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -29,6 +29,7 @@ windmill-dep-map.workspace = true axum.workspace = true chrono.workspace = true hex.workspace = true +magic-crypt.workspace = true http.workspace = true hyper.workspace = true lazy_static.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8f3311978a..80e497ba97 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -31,7 +31,9 @@ use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; +use windmill_common::variables::{ + build_crypt, decrypt, encrypt, SECRET_SALT, WORKSPACE_CRYPT_CACHE, +}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -2418,20 +2420,28 @@ async fn set_encryption_key( )); } + // Build the previous cipher before the transaction (reads from cache/pool) let previous_encryption_key = build_crypt(&db, w_id.as_str()).await?; + let mut tx = db.begin().await?; + sqlx::query!( "UPDATE workspace_key SET key = $1 WHERE workspace_id = $2", request.new_key.clone(), w_id ) - .execute(&db) + .execute(&mut *tx) .await?; - WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); - if !request.skip_reencrypt.unwrap_or(false) { - let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; + // Build the new cipher directly from the key string, since the transaction + // hasn't committed yet and build_crypt() would read the old key from the pool. + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", request.new_key, salt) + } else { + request.new_key.clone() + }; + let new_encryption_key = magic_crypt::new_magic_crypt!(crypt_key, 256); let mut truncated_new_key = request.new_key.clone(); truncated_new_key.truncate(8); @@ -2445,7 +2455,7 @@ async fn set_encryption_key( "SELECT path, value, is_secret FROM variable WHERE workspace_id = $1", w_id ) - .fetch_all(&db) + .fetch_all(&mut *tx) .await?; for variable in all_variables { @@ -2466,11 +2476,16 @@ async fn set_encryption_key( w_id, variable.path ) - .execute(&db) + .execute(&mut *tx) .await?; } } + tx.commit().await?; + + // Invalidate the cache only after the transaction has committed + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + // Trigger git sync for encryption key changes handle_deployment_metadata( &authed.email, From e66a682c4a4a3449b2730e4b17de600f4efcb600 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Wed, 4 Mar 2026 16:09:42 +0100 Subject: [PATCH 03/57] chore: make rust-analyzer plugin opt-in via USE_RUST_PLUGIN env var (#8227) * feat: optionally enable rust-analyzer plugin in worktree settings When USE_RUST_PLUGIN env var is set, the worktree-env script now includes the rust-analyzer-lsp plugin in .claude/settings.local.json. Co-Authored-By: Claude Opus 4.6 * chore: remove rust-analyzer plugin from default settings The rust-analyzer plugin is now opt-in via USE_RUST_PLUGIN env var in worktree-env, so it no longer needs to be in the shared settings. Co-Authored-By: Claude Opus 4.6 * chore: add WM_CLONE_DB and USE_RUST_PLUGIN to wmdev startup envs Defaults both to false so they can be toggled per-worktree. Co-Authored-By: Claude Opus 4.6 * fix: use explicit truthy checks for WM_CLONE_DB and USE_RUST_PLUGIN Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .claude/settings.json | 1 - .wmdev.yaml | 2 ++ scripts/worktree-env | 11 +++++++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.claude/settings.json b/.claude/settings.json index fcd49c3140..cf8bfdd284 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -110,7 +110,6 @@ ] }, "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true, "typescript-lsp@claude-plugins-official": true, "code-review@claude-plugins-official": true } diff --git a/.wmdev.yaml b/.wmdev.yaml index c028c8f3bf..1a949c94e2 100644 --- a/.wmdev.yaml +++ b/.wmdev.yaml @@ -2,6 +2,8 @@ name: Windmill startupEnvs: CARGO_FEATURES: "quickjs" + WM_CLONE_DB: false + USE_RUST_PLUGIN: false services: - name: BE diff --git a/scripts/worktree-env b/scripts/worktree-env index ae83b09405..5fd8490bc2 100755 --- a/scripts/worktree-env +++ b/scripts/worktree-env @@ -61,7 +61,7 @@ if command -v psql &>/dev/null; then if psql "$db_conn/postgres" -tc "SELECT 1 FROM pg_database WHERE datname = '${db_name}'" 2>/dev/null | grep -q 1; then echo "Database $db_name already exists" else - if [[ -n "${WM_CLONE_DB:-}" ]]; then + if [[ "${WM_CLONE_DB:-}" == "1" || "${WM_CLONE_DB:-}" == "true" ]]; then # Terminate active connections so CREATE DATABASE ... TEMPLATE works psql "$db_conn/postgres" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'windmill' AND pid <> pg_backend_pid();" 2>/dev/null || true psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name} TEMPLATE windmill" 2>/dev/null \ @@ -178,13 +178,20 @@ if [ -n "$ee_repo" ]; then if [ -d "$ee_worktree_dir" ]; then ee_rel=$(python3 -c "import os; print(os.path.relpath('$ee_worktree_dir', '$(pwd)'))" 2>/dev/null || echo "$ee_worktree_dir") mkdir -p .claude + rust_plugin="" + if [[ "${USE_RUST_PLUGIN:-}" == "1" || "${USE_RUST_PLUGIN:-}" == "true" ]]; then + rust_plugin=', + "enabledPlugins": { + "rust-analyzer-lsp@claude-plugins-official": true + }' + fi cat > .claude/settings.local.json < Date: Wed, 4 Mar 2026 16:12:00 +0100 Subject: [PATCH 04/57] feat: replace hub error toasts with warning alerts and add disable hub setting (#8225) * feat: replace hub error toasts with warning alerts and add disable hub setting Co-Authored-By: Claude Opus 4.6 * fix: guard hub script cache refresh when hub is disabled Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- backend/windmill-api-settings/src/lib.rs | 5 +++-- .../windmill-common/src/global_settings.rs | 1 + .../windmill-common/src/instance_config.rs | 2 ++ .../flows/pickers/PickHubApp.svelte | 22 +++++++++++++++++-- .../flows/pickers/PickHubFlow.svelte | 22 +++++++++++++++++-- .../flows/pickers/PickHubScript.svelte | 11 +++++++++- .../flows/pickers/PickHubScriptQuick.svelte | 22 +++++++++++++------ .../src/lib/components/instanceSettings.ts | 10 +++++++++ frontend/src/lib/stores.ts | 1 + .../src/routes/(root)/(logged)/+layout.svelte | 7 ++++++ 10 files changed, 89 insertions(+), 14 deletions(-) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index bf493f9420..6b408724fd 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -43,8 +43,8 @@ use windmill_common::{ get_database_url, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, - ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, + EMAIL_DOMAIN_SETTING, ENV_SETTINGS, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, }, instance_config::{self, ApplyMode, InstanceConfig}, server::Smtp, @@ -519,6 +519,7 @@ pub async fn get_global_setting( && key != DEFAULT_TAGS_WORKSPACES_SETTING && key != HUB_BASE_URL_SETTING && key != HUB_ACCESSIBLE_URL_SETTING + && key != DISABLE_HUB_SETTING && key != EMAIL_DOMAIN_SETTING && key != APP_WORKSPACED_ROUTE_SETTING { diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 3347127303..d4d8163ff2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -44,6 +44,7 @@ pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; +pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4de90be5c5..4ef88f5b78 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -230,6 +230,8 @@ pub struct GlobalSettings { pub no_default_maven: Option, #[serde(skip_serializing_if = "Option::is_none")] pub default_tags_per_workspace: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_hub: Option, // String settings #[serde(skip_serializing_if = "Option::is_none")] diff --git a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte index 7f534a924a..234f053f7b 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubApp.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubApp.svelte @@ -7,6 +7,8 @@ import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubApps } from '$lib/hub' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { Alert } from '$lib/components/common' + import { disableHubStore } from '$lib/stores' interface Props { filter?: string @@ -30,11 +32,22 @@ const dispatch = createEventDispatcher() + let hubNotAvailable = $state(false) + onMount(async () => { - hubApps = await loadHubApps() + if ($disableHubStore) return + const result = await loadHubApps() + if (result === undefined) { + hubNotAvailable = true + } else { + hubApps = result + } }) +{#if $disableHubStore} + +{:else} -{#if hubApps} +{#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + +{:else if hubApps} {#if filteredItems.length == 0} {:else} @@ -93,3 +110,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte index 2ab12aea36..1f3c659bf8 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubFlow.svelte @@ -7,6 +7,8 @@ import RowIcon from '$lib/components/common/table/RowIcon.svelte' import { loadHubFlows } from '$lib/hub' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { Alert } from '$lib/components/common' + import { disableHubStore } from '$lib/stores' interface Props { filter?: string @@ -30,11 +32,22 @@ const dispatch = createEventDispatcher() + let hubNotAvailable = $state(false) + onMount(async () => { - hubFlows = await loadHubFlows() + if ($disableHubStore) return + const result = await loadHubFlows() + if (result === undefined) { + hubNotAvailable = true + } else { + hubFlows = result + } }) +{#if $disableHubStore} + +{:else} -{#if hubFlows} +{#if hubNotAvailable} + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + +{:else if hubFlows} {#if filteredItems.length == 0} {:else} @@ -95,3 +112,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte index b5ba87327e..7ec0a8ccc3 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScript.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScript.svelte @@ -8,6 +8,7 @@ import { IntegrationService, ScriptService, type HubScriptKind } from '$lib/gen' import { Loader2 } from 'lucide-svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' + import { disableHubStore } from '$lib/stores' interface Props { kind?: HubScriptKind & string @@ -47,6 +48,7 @@ ) async function getAllApps(filterKind: typeof kind) { + if ($disableHubStore) return try { hubNotAvailable = false allApps = ( @@ -67,6 +69,7 @@ filterKind: typeof kind, appFilter: string | undefined ) { + if ($disableHubStore) return try { loading = true hubNotAvailable = false @@ -138,6 +141,9 @@ }) +{#if $disableHubStore} + +{:else}
{@render children?.()}
@@ -156,7 +162,9 @@
{#if hubNotAvailable} - + + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. + {:else if (items.length > 0 && apps.length > 0) || !loading} {#if items.length == 0} @@ -204,3 +212,4 @@ {/each} {/if} +{/if} diff --git a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte index 2091f76340..3d929dead7 100644 --- a/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte +++ b/frontend/src/lib/components/flows/pickers/PickHubScriptQuick.svelte @@ -24,7 +24,7 @@ []) : undefined } catch (err) { - sendUserToast('Failed to fetch hub scripts: ' + err, 'error') + console.error('Failed to fetch hub scripts:', err) return undefined } }, @@ -44,9 +44,10 @@ import { Circle, ExternalLink } from 'lucide-svelte' import Popover from '$lib/components/Popover.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' - import { hubBaseUrlStore, userStore } from '$lib/stores' + import { disableHubStore, hubBaseUrlStore, userStore } from '$lib/stores' import { get } from 'svelte/store' import Button from '$lib/components/common/button/Button.svelte' + import { Alert } from '$lib/components/common' let hubNotAvailable = $state(false) @@ -94,13 +95,14 @@ }) async function getAllApps(filterKind: typeof kind) { + if ($disableHubStore) return try { hubNotAvailable = false allApps = (await listHubIntegrationsCached({ kind: filterKind, refreshCount })).map( (x) => x.name ) } catch (err) { - sendUserToast('Failed to fetch hub integrations: ' + err, 'error') + console.error('Failed to fetch hub integrations:', err) allApps = [] hubNotAvailable = true } @@ -112,7 +114,9 @@ ) $effect(() => { ;[filter, kind, appFilter, refreshCount] - hubScriptsFilteredPromise.refresh() + if (!$disableHubStore) { + hubScriptsFilteredPromise.refresh() + } }) $effect(() => { loading = hubScriptsFilteredPromise.status === 'loading' @@ -175,9 +179,13 @@ -{#if hubNotAvailable} -
- Hub not available +{#if $disableHubStore} + +{:else if hubNotAvailable} +
+ + Could not connect to the Windmill Hub. If you are in a closed environment, you can disable the Hub in the instance settings. +
{:else if loading} {#each Array(15).fill(0) as _} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index de4cd1528b..25955b74db 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -331,6 +331,16 @@ export const settings: Record = { storage: 'setting', ee_only: '', hiddenIfEmpty: true + }, + { + label: 'Disable Hub', + description: + 'Disable the Windmill Hub integration entirely. Enable this if your instance runs in a closed environment without internet access and you do not have a private hub setup.', + key: 'disable_hub', + fieldType: 'boolean', + storage: 'setting', + ee_only: '', + requiresReloadOnChange: true } ], SMTP: [ diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 6a9a8e7805..fd0a66cfb3 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -83,6 +83,7 @@ export const superadmin = writable(undefined) export const devopsRole = writable(undefined) export const lspTokenStore = writable(undefined) export const hubBaseUrlStore = writable(DEFAULT_HUB_BASE_URL) +export const disableHubStore = writable(false) export const userWorkspaces: Readable> = derived( [usersWorkspaceStore, superadmin], ([store, superadmin]) => { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index e9b1884df2..78de6c1535 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -26,6 +26,7 @@ type UserExt, defaultScripts, hubBaseUrlStore, + disableHubStore, usedTriggerKinds, devopsRole, whitelabelNameStore, @@ -157,6 +158,7 @@ loadUsage() syncTutorialsTodos() loadHubBaseUrl() + loadDisableHub() loadUsedTriggerKinds() } @@ -176,6 +178,11 @@ DEFAULT_HUB_BASE_URL } + async function loadDisableHub() { + $disableHubStore = + ((await SettingService.getGlobal({ key: 'disable_hub' })) as boolean) ?? false + } + async function loadFavorites() { const scripts = await ScriptService.listScripts({ workspace: $workspaceStore ?? '', From ac71232b95d99b6545bae211af5ee0a3d13e7f87 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 4 Mar 2026 20:20:18 +0000 Subject: [PATCH 05/57] fix: improve windows compatibility * ci: add Windows backend integration test workflow Co-Authored-By: Claude Opus 4.5 * ci: temporarily add push trigger for testing Co-Authored-By: Claude Opus 4.5 * ci: add --no-fail-fast to run all test binaries Co-Authored-By: Claude Opus 4.5 * fix: Windows path handling for backend integration tests - WINDMILL_DIR: use std::env::temp_dir() on Windows instead of /tmp/windmill - HOME_ENV: fall back to USERPROFILE on Windows when HOME is not set - loader.bun.js: normalize paths to forward slashes for consistent comparison with Bun's resolver output on Windows - bun_executor.rs: convert job_dir to forward slashes in JS template strings to avoid backslash escape issues (\t -> tab, etc.) - go_executor.rs: fix windows_gopath() double backslash bug (r"\\" -> "\\") - bash_executor.rs: default to "bash" (in PATH) on Windows instead of /bin/bash Co-Authored-By: Claude Opus 4.6 * fix: improve Windows diagnostics and fix onLoad handler - Include path in create_directory_async/sync panic messages - Add WINDMILL_DIR initialization debug output - Fix loader.bun.js onLoad: use properly escaped regex instead of returning undefined (Bun requires onLoad to return an object) - Add env var debug output to CI workflow Co-Authored-By: Claude Opus 4.6 * fix: sanitize Windows-invalid characters in test worker names and fix cargo path - Replace :: with __ in worker names (colons illegal in Windows dir names) - Fix HOME_DIR to fall back to USERPROFILE on Windows - Add PATH fallback for cargo discovery on Windows - Add debug logging to bun loader for fetch errors Co-Authored-By: Claude Opus 4.6 * fix: handle single colons in worker names, pass MSVC linker env vars, revert bun debug Co-Authored-By: Claude Opus 4.5 * fix: use .exe binary name on Windows and normalize bun import URL paths Co-Authored-By: Claude Opus 4.5 * fix: use absolute path for rust binary, normalize bun resolve paths Co-Authored-By: Claude Opus 4.5 * fix: use .wurl extension instead of .url for bun import resolution on Windows Co-Authored-By: Claude Opus 4.5 * fix: use custom namespace for bun plugin to bypass default file resolution Co-Authored-By: Claude Opus 4.5 * fix: use virtual namespace for bun import resolution to avoid Windows path issues Co-Authored-By: Claude Opus 4.5 * fix: handle Windows 8.3 paths and namespace-prefixed importers in bun loader Co-Authored-By: Claude Opus 4.5 * fix: strip namespace prefix from args.path and handle absolute imports without leading slash in bun loader Co-Authored-By: Claude Opus 4.5 * refactor: simplify bun loader and remove redundant cargo path lookups Co-Authored-By: Claude Opus 4.5 * fix: use platform-specific cargo binary path with .exe on Windows Co-Authored-By: Claude Opus 4.5 * refactor: replace HOME_DIR with HOME_ENV in rust_executor to remove duplication Co-Authored-By: Claude Opus 4.5 * refactor: keep original bun loader on linux, use virtual namespace loader only on windows Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- .github/workflows/backend-test-windows.yml | 165 ++++++++++++++++++ backend/windmill-common/src/utils.rs | 4 +- backend/windmill-common/src/worker.rs | 12 +- backend/windmill-test-utils/src/lib.rs | 2 + backend/windmill-worker/loader.bun.windows.js | 123 +++++++++++++ backend/windmill-worker/src/bash_executor.rs | 7 +- backend/windmill-worker/src/bun_executor.rs | 15 +- backend/windmill-worker/src/go_executor.rs | 3 +- backend/windmill-worker/src/rust_executor.rs | 63 ++++--- backend/windmill-worker/src/worker.rs | 11 +- 10 files changed, 371 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/backend-test-windows.yml create mode 100644 backend/windmill-worker/loader.bun.windows.js diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml new file mode 100644 index 0000000000..ca9ce2aaac --- /dev/null +++ b/.github/workflows/backend-test-windows.yml @@ -0,0 +1,165 @@ +name: Backend integration tests (Windows) + +on: + workflow_dispatch: + push: + branches: + - "ci-windows-tests" + +env: + CARGO_INCREMENTAL: 0 + SQLX_OFFLINE: true + DISABLE_EMBEDDING: true + +jobs: + cargo_test_windows: + runs-on: blacksmith-16vcpu-windows-2025 + steps: + - uses: actions/checkout@v4 + + - name: Read EE repo commit hash + shell: pwsh + run: | + $ee_repo_ref = Get-Content .\backend\ee-repo-ref.txt + echo "ee_repo_ref=$ee_repo_ref" | Out-File -FilePath $env:GITHUB_ENV -Append + + - name: Checkout windmill-ee-private repository + uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + - name: Substitute EE code + shell: bash + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Setup PostgreSQL + uses: ikalnytskyi/action-setup-postgres@v6 + with: + username: postgres + password: changeme + database: windmill + port: 5432 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.93.0 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: "9.0.x" + + - uses: denoland/setup-deno@v2 + with: + deno-version: v2.x + + - uses: actions/setup-go@v2 + with: + go-version: 1.21.5 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.3.10 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + + - uses: astral-sh/setup-uv@v6.2.1 + with: + version: "0.9.24" + + - uses: shivammathur/setup-php@v2 + with: + php-version: "8.3" + tools: composer + + - name: Install windmill CLI + shell: bash + run: | + cd cli + bash gen_wm_client.sh + bun install + mkdir -p "$HOME/.local/bin" + printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill" + chmod +x "$HOME/.local/bin/wmill" + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install OpenSSL via vcpkg + run: | + vcpkg.exe install openssl-windows:x64-windows + vcpkg.exe install openssl:x64-windows-static + vcpkg.exe integrate install + + - name: Get runtime paths + id: runtime-paths + shell: pwsh + run: | + echo "DENO_PATH=$($(Get-Command deno).Source)" >> $env:GITHUB_OUTPUT + echo "BUN_PATH=$($(Get-Command bun).Source)" >> $env:GITHUB_OUTPUT + echo "NODE_BIN_PATH=$($(Get-Command node).Source)" >> $env:GITHUB_OUTPUT + echo "GO_PATH=$($(Get-Command go).Source)" >> $env:GITHUB_OUTPUT + echo "UV_PATH=$($(Get-Command uv).Source)" >> $env:GITHUB_OUTPUT + echo "PHP_PATH=$($(Get-Command php).Source)" >> $env:GITHUB_OUTPUT + echo "COMPOSER_PATH=$($(Get-Command composer).Source)" >> $env:GITHUB_OUTPUT + echo "POWERSHELL_PATH=$($(Get-Command pwsh).Source)" >> $env:GITHUB_OUTPUT + echo "DOTNET_PATH=$($(Get-Command dotnet).Source)" >> $env:GITHUB_OUTPUT + + - name: Build DuckDB FFI module + working-directory: backend/windmill-duckdb-ffi-internal + timeout-minutes: 30 + run: | + cargo build --release -p windmill_duckdb_ffi_internal + New-Item -ItemType Directory -Path ..\target\debug -Force + Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\ + + - name: Print runtime versions and env + shell: pwsh + run: | + deno --version + bun -v + node --version + go version + python3 --version + php --version + pwsh --version + dotnet --version + echo "TEMP=$env:TEMP" + echo "TMP=$env:TMP" + echo "USERPROFILE=$env:USERPROFILE" + echo "HOME=$env:HOME" + + - name: cargo test + working-directory: backend + timeout-minutes: 60 + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: "off" + RUST_LOG_STYLE: never + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_BUILD_JOBS: 12 + VCPKGRS_DYNAMIC: 1 + OPENSSL_DIR: ${{ env.VCPKG_INSTALLATION_ROOT }}\installed\x64-windows-static + DENO_PATH: ${{ steps.runtime-paths.outputs.DENO_PATH }} + BUN_PATH: ${{ steps.runtime-paths.outputs.BUN_PATH }} + NODE_BIN_PATH: ${{ steps.runtime-paths.outputs.NODE_BIN_PATH }} + GO_PATH: ${{ steps.runtime-paths.outputs.GO_PATH }} + UV_PATH: ${{ steps.runtime-paths.outputs.UV_PATH }} + PHP_PATH: ${{ steps.runtime-paths.outputs.PHP_PATH }} + COMPOSER_PATH: ${{ steps.runtime-paths.outputs.COMPOSER_PATH }} + POWERSHELL_PATH: ${{ steps.runtime-paths.outputs.POWERSHELL_PATH }} + DOTNET_PATH: ${{ steps.runtime-paths.outputs.DOTNET_PATH }} + WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1 + WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1 + WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1 + run: > + cargo test + --no-fail-fast + --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,csharp,php,quickjs,mcp,run_inline + --all + -- --nocapture --test-threads=10 diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index e4c5315cba..10b1a3b408 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -314,14 +314,14 @@ pub async fn create_directory_async(directory_path: &str) { .recursive(true) .create(directory_path) .await - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } pub fn create_directory_sync(directory_path: &str) { SyncDirBuilder::new() .recursive(true) .create(directory_path) - .expect("could not create dir"); + .unwrap_or_else(|e| panic!("could not create dir '{}': {}", directory_path, e)); } #[track_caller] diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 8d95541f61..3195562368 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -494,7 +494,17 @@ pub async fn store_pull_query(wc: &WorkerConfig) { lazy_static::lazy_static! { pub static ref WINDMILL_DIR: String = { let dir = std::env::var("WINDMILL_DIR") - .unwrap_or_else(|_| "/tmp/windmill".to_string()); + .unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/tmp/windmill".to_string() } + #[cfg(windows)] + { + let temp = std::env::temp_dir(); + let temp_str = temp.to_string_lossy(); + let normalized = temp_str.trim_end_matches(&['/', '\\'][..]).replace('\\', "/"); + format!("{}/windmill", normalized) + } + }); if dir.is_empty() { panic!("WINDMILL_DIR must not be empty"); } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 69e7fc0558..68da452335 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -66,6 +66,8 @@ fn next_worker_name() -> String { .unwrap_or(s) }) .unwrap_or("no thread name"); + // Replace colons because they are illegal in Windows directory names + let thread_name = thread_name.replace(':', "_"); format!("{id}/worker-{thread_name}") } diff --git a/backend/windmill-worker/loader.bun.windows.js b/backend/windmill-worker/loader.bun.windows.js new file mode 100644 index 0000000000..227c68a56c --- /dev/null +++ b/backend/windmill-worker/loader.bun.windows.js @@ -0,0 +1,123 @@ +// Windows-specific bun loader that uses a virtual "windmill-url" namespace instead +// of writing .url files to disk. This avoids Windows path issues (backslashes in +// resolve(), 8.3 short filenames, drive letter prefixes). The virtual namespace +// approach is likely better on all fronts but we keep the original .url-file loader +// on Linux to avoid breaking back-compat. +const p = { + name: "windmill-relative-resolver", + async setup(build) { + const { readFileSync } = await import("fs"); + const { resolve } = await import("node:path"); + + const base_internal_url = "BASE_INTERNAL_URL".replace( + "localhost", + "127.0.0.1" + ); + + const w_id = "W_ID"; + const current_path = "CURRENT_PATH"; + const token = "TOKEN"; + + const cdir = resolve("./"); + const cdirNoPrivate = cdir.replace(/^\/private/, ""); // for macos + // Normalize path to forward slashes to match Bun's resolver output on Windows + const cdirFwd = cdir.replace(/\\/g, "/"); + const cdirPosix = cdirFwd.replace(/^[a-zA-Z]:/, ""); + const filterResolve = new RegExp( + `^(?!\\.\/main\\.ts)(?!${cdirFwd}\/main\\.ts)(?!${cdirPosix}\/main\\.ts)(?!(?:/private)?${cdirNoPrivate}\/wrapper\\.mjs).*\\.ts$` + ); + + let cdirNodeModules = `${cdirFwd}/node_modules/`; + + const filterLoad = new RegExp(`^${cdir}\/main\\.ts$`); + const transpiler = new Bun.Transpiler({ + loader: "ts", + }); + + function replaceRelativeImports(code) { + const imports = transpiler.scanImports(code); + for (const imp of imports) { + if (imp.kind == "import-statement") { + if ( + (imp.path.startsWith(".") || + imp.path.startsWith("/u/") || + imp.path.startsWith("/f/")) && + !imp.path.endsWith(".ts") + ) { + code = code.replaceAll(imp.path, imp.path + ".ts"); + } + } + } + return { + contents: code, + }; + } + + function normalizePath(rawPath) { + return rawPath.split("/").reduce((acc, seg) => { + if (seg === "..") acc.pop(); + else if (seg !== "." && seg !== "") acc.push(seg); + return acc; + }, []).join("/"); + } + + // Resolve a windmill script import path relative to an importer path. + // Bun on Windows may prefix args with "windmill-url:" or strip leading "/". + function resolveWindmillImport(importerPath, importPath) { + const path = importPath.replace(/^windmill-url:/, "").replace(/^\//, ""); + const isAbsolute = path.startsWith("f/") || path.startsWith("u/"); + const endExt = path.endsWith(".ts") ? "" : ".ts"; + const rawScriptPath = isAbsolute + ? `${path}${endExt}` + : `${importerPath}/../${path}${endExt}`; + return { path: normalizePath(rawScriptPath), namespace: "windmill-url" }; + } + + build.onLoad({ filter: filterLoad }, async (args) => { + const code = readFileSync(args.path, "utf8"); + return replaceRelativeImports(code); + }); + + // Load windmill scripts by fetching from the API + build.onLoad({ filter: /.*/, namespace: "windmill-url" }, async (args) => { + const path = args.path.replace(/^windmill-url:/, ""); + const url = `${base_internal_url}/api/w/${w_id}/scripts/RAW_GET_ENDPOINT/p/${path}`; + const req = await fetch(url, { + method: "GET", + headers: { + Authorization: "Bearer " + token, + }, + }); + if (!req.ok) { + throw new Error( + `Failed to find relative import at ${url} (status ${req.status})` + ); + } + const contents = await req.text(); + return { + contents: replaceRelativeImports(contents).contents, + loader: "tsx", + }; + }); + + // Resolve windmill script imports from the file namespace (e.g. from main.ts) + build.onResolve({ filter: filterResolve }, (args) => { + const importerFwd = args.importer?.replace(/\\/g, "/") ?? ""; + if (importerFwd.startsWith(cdirNodeModules)) { + return undefined; + } + const isMainTs = + args.importer == "./main.ts" || importerFwd.endsWith("/main.ts"); + const file_path = isMainTs + ? current_path + : importerFwd.replace(cdirFwd + "/", ""); + return resolveWindmillImport(file_path, args.path); + }); + + // Resolve nested imports from within windmill-url modules + build.onResolve({ filter: /\.ts$/, namespace: "windmill-url" }, (args) => { + const importer = args.importer.replace(/^windmill-url:/, ""); + return resolveWindmillImport(importer, args.path); + }); + }, +}; diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 30348895cf..429fd88c94 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -26,7 +26,12 @@ use windmill_queue::{ }; lazy_static::lazy_static! { - pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); + pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| { + #[cfg(not(windows))] + { "/bin/bash".to_string() } + #[cfg(windows)] + { "bash".to_string() } + }); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 08a61645ce..40f4b9f77e 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -53,7 +53,14 @@ use windmill_object_store::attempt_fetch_bytes; use windmill_parser::Typ; +// The Windows loader uses a virtual "windmill-url" namespace instead of writing .url +// files to disk, which avoids Windows path issues. The virtual namespace approach is +// likely better on all fronts but we keep the original .url-file loader on Linux to +// avoid breaking back-compat. +#[cfg(not(windows))] pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.js"); +#[cfg(windows)] +pub const RELATIVE_BUN_LOADER: &str = include_str!("../loader.bun.windows.js"); pub const RELATIVE_BUN_BUILDER: &str = include_str!("../loader_builder.bun.js"); @@ -527,6 +534,8 @@ pub async fn build_loader( current_path: &str, mode: LoaderMode, ) -> Result<()> { + // Use forward slashes in JS strings to avoid backslash escape issues on Windows + let job_dir_js = job_dir.replace('\\', "/"); let loader = RELATIVE_BUN_LOADER .replace("W_ID", w_id) .replace("BASE_INTERNAL_URL", base_internal_url) @@ -549,13 +558,13 @@ import {{ readdir }} from "node:fs/promises"; let fileNames = [] try {{ - fileNames = await readdir("{job_dir}/node_modules") + fileNames = await readdir("{job_dir_js}/node_modules") }} catch (e) {{ }} try {{ await Bun.build({{ - entrypoints: ["{job_dir}/wrapper.mjs"], + entrypoints: ["{job_dir_js}/wrapper.mjs"], outdir: "./", target: "node", plugins: [p], @@ -597,7 +606,7 @@ plugin(p) try {{ await Bun.build({{ - entrypoints: ["{job_dir}/main.ts"], + entrypoints: ["{job_dir_js}/main.ts"], outdir: "./", target: "{}", plugins: [p], diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 29fff1a2ba..ddcc108659 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -53,8 +53,7 @@ fn get_windows_program_files() -> String { #[cfg(windows)] fn windows_gopath() -> String { - let tmp_dir = get_windows_tmp_dir(); - GO_CACHE_DIR.replace("/tmp", &tmp_dir).replace("/", r"\\") + GO_CACHE_DIR.replace('/', "\\") } #[cfg(windows)] diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 216c59f2cd..a1f29696c4 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -41,20 +41,30 @@ const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.co const NSJAIL_CONFIG_COMPILE_RUST_CONTENT: &str = include_str!("../nsjail/download.rust.config.proto"); +#[cfg(windows)] +const RUST_BIN_NAME: &str = "main.exe"; +#[cfg(not(windows))] +const RUST_BIN_NAME: &str = "main"; + fn find_cargo_path() -> String { if let Ok(p) = std::env::var("CARGO_PATH") { return p; } - let from_home = format!("{}/bin/cargo", CARGO_HOME.as_str()); - if std::path::Path::new(&from_home).exists() { - return from_home; - } - for p in ["/usr/local/cargo/bin/cargo", "/usr/bin/cargo"] { + let candidates = if cfg!(windows) { + vec![format!("{}\\bin\\cargo.exe", CARGO_HOME.as_str())] + } else { + vec![ + format!("{}/bin/cargo", CARGO_HOME.as_str()), + "/usr/local/cargo/bin/cargo".to_string(), + "/usr/bin/cargo".to_string(), + ] + }; + for p in &candidates { if std::path::Path::new(p).exists() { - return p.to_string(); + return p.clone(); } } - from_home + candidates.into_iter().next().unwrap() } #[cfg(not(windows))] @@ -71,7 +81,6 @@ fn find_preinstalled_dir(env_var: &str, candidates: &[&str]) -> String { } lazy_static::lazy_static! { - static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); static ref CARGO_PATH: String = find_cargo_path(); @@ -81,14 +90,14 @@ lazy_static::lazy_static! { #[cfg(windows)] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}\\.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}\\.rustup", HOME_ENV.as_str()); } #[cfg(not(windows))] lazy_static::lazy_static! { - static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", *HOME_DIR); - static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", *HOME_DIR); + static ref CARGO_HOME_DEFAULT: String = format!("{}/.cargo", HOME_ENV.as_str()); + static ref RUSTUP_HOME_DEFAULT: String = format!("{}/.rustup", HOME_ENV.as_str()); } const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; @@ -97,11 +106,11 @@ const RUST_OBJECT_STORE_PREFIX: &str = "rustbin/"; lazy_static::lazy_static! { static ref PREINSTALLED_CARGO: String = find_preinstalled_dir( "CARGO_PREINSTALL_DIR", - &["/usr/local/cargo", &format!("{}/.cargo", *HOME_DIR)], + &["/usr/local/cargo", &format!("{}/.cargo", HOME_ENV.as_str())], ); static ref PREINSTALLED_RUSTUP: String = find_preinstalled_dir( "RUSTUP_PREINSTALL_DIR", - &["/usr/local/rustup", &format!("{}/.rustup", *HOME_DIR)], + &["/usr/local/rustup", &format!("{}/.rustup", HOME_ENV.as_str())], ); } @@ -521,6 +530,13 @@ pub async fn build_rust_crate( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + // MSVC linker needs LIB and INCLUDE to find kernel32.lib etc. + if let Ok(lib) = std::env::var("LIB") { + build_rust_cmd.env("LIB", lib); + } + if let Ok(include) = std::env::var("INCLUDE") { + build_rust_cmd.env("INCLUDE", include); + } } start_child_process(build_rust_cmd, CARGO_PATH.as_str(), false).await? }; @@ -545,30 +561,29 @@ pub async fn build_rust_crate( tokio::fs::copy( &format!( - "{build_dir}/target/{}/main", + "{build_dir}/target/{}/{RUST_BIN_NAME}", if is_preview { "debug" } else { "release" }, ), - format! {"{job_dir}/main"}, + format!("{job_dir}/{RUST_BIN_NAME}"), ) .await .map_err(|e| { Error::ExecutionErr(format!( - "could not copy built binary from [...]/target/.../main to {job_dir}/main: {e:?}" + "could not copy built binary from [...]/target/.../{RUST_BIN_NAME} to {job_dir}/{RUST_BIN_NAME}: {e:?}" )) })?; match save_cache( &bin_path, &format!("{RUST_OBJECT_STORE_PREFIX}{hash}"), - &format!("{job_dir}/main"), + &format!("{job_dir}/{RUST_BIN_NAME}"), false, ) .await { Err(e) => { let em = format!( - "could not save {bin_path} to {} to rust cache: {e:?}", - format!("{job_dir}/main"), + "could not save {bin_path} to {job_dir}/{RUST_BIN_NAME} to rust cache: {e:?}", ); tracing::error!(em); Ok(em) @@ -618,16 +633,16 @@ pub async fn handle_rust_job( let (cache, cache_logs) = crate::global_cache::load_cache(&bin_path, &remote_path, false).await; let cache_logs = if cache { - let target = format!("{job_dir}/main"); + let target = format!("{job_dir}/{RUST_BIN_NAME}"); #[cfg(unix)] let symlink = std::os::unix::fs::symlink(&bin_path, &target); #[cfg(windows)] - let symlink = std::os::windows::fs::symlink_dir(&bin_path, &target); + let symlink = std::os::windows::fs::symlink_file(&bin_path, &target); symlink.map_err(|e| { Error::ExecutionErr(format!( - "could not copy cached binary from {bin_path} to {job_dir}/main: {e:?}" + "could not copy cached binary from {bin_path} to {target}: {e:?}" )) })?; @@ -694,7 +709,7 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let compiled_executable_name = "./main"; + let compiled_executable_name = &format!("{job_dir}/{RUST_BIN_NAME}"); let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 49a1048afd..1497d3ebb8 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -563,7 +563,16 @@ lazy_static::lazy_static! { pub static ref DOTNET_PATH: String = std::env::var("DOTNET_PATH").unwrap_or_else(|_| DOTNET_DEFAULT_PATH.to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); - pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref HOME_ENV: String = { + #[cfg(not(windows))] + { std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()) } + #[cfg(windows)] + { + std::env::var("HOME") + .or_else(|_| std::env::var("USERPROFILE")) + .unwrap_or_else(|_| std::env::temp_dir().to_string_lossy().to_string()) + } + }; pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); From 32afdd480c74f637d82ef24bdb226a888a6e4a2a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Mar 2026 06:13:42 +0100 Subject: [PATCH 06/57] chore: upgrade rquickjs from 0.8 to 0.11 (#8233) Co-authored-by: Claude Opus 4.6 --- backend/Cargo.lock | 53 ++++++++++++++++++++++++---------------------- backend/Cargo.toml | 2 +- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index e52bb12fa1..d853faa2a0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1900,7 +1900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c" dependencies = [ "once_cell", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -2550,6 +2550,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "cooked-waker" version = "5.0.0" @@ -8683,7 +8692,7 @@ dependencies = [ "darling 0.20.11", "heck 0.5.0", "num-bigint", - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro-error2", "proc-macro2", "quote", @@ -9252,7 +9261,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate 3.4.0", + "proc-macro-crate", "proc-macro2", "quote", "syn 2.0.117", @@ -10331,16 +10340,6 @@ dependencies = [ "elliptic-curve", ] -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", -] - [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -11094,9 +11093,12 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "relative-path" -version = "1.9.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] [[package]] name = "rend" @@ -11390,9 +11392,9 @@ dependencies = [ [[package]] name = "rquickjs" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16661bff09e9ed8e01094a188b463de45ec0693ade55b92ed54027d7ba7c40c" +checksum = "c50dc6d6c587c339edb4769cf705867497a2baf0eca8b4645fa6ecd22f02c77a" dependencies = [ "rquickjs-core", "rquickjs-macro", @@ -11400,26 +11402,27 @@ dependencies = [ [[package]] name = "rquickjs-core" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8db6379e204ef84c0811e90e7cc3e3e4d7688701db68a00d14a6db6849087b" +checksum = "b8bf7840285c321c3ab20e752a9afb95548c75cd7f4632a0627cea3507e310c1" dependencies = [ "async-lock", + "hashbrown 0.16.0", "relative-path", "rquickjs-sys", ] [[package]] name = "rquickjs-macro" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6041104330c019fcd936026ae05e2446f5e8a2abef329d924f25424b7052a2f3" +checksum = "7106215ff41a5677b104906a13e1a440b880f4b6362b5dc4f3978c267fad2b80" dependencies = [ - "convert_case 0.6.0", + "convert_case 0.10.0", "fnv", "ident_case", "indexmap 2.11.1", - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", "rquickjs-core", @@ -11428,9 +11431,9 @@ dependencies = [ [[package]] name = "rquickjs-sys" -version = "0.8.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc352c6b663604c3c186c000cfcc6c271f4b50bc135a285dd6d4f2a42f9790a" +checksum = "27344601ef27460e82d6a4e1ecb9e7e99f518122095f3c51296da8e9be2b9d83" dependencies = [ "cc", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 195d8bd30d..bc1ddf9829 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -512,7 +512,7 @@ nu-parser = { version = "0.101.0", default-features = false } globset = "0.4.16" croner = "2.2.0" rmcp = { version = "=0.15.0", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } -rquickjs = { version = "0.8", features = ["futures", "parallel", "macro"] } +rquickjs = { version = "0.11", features = ["futures", "parallel", "macro"] } process-wrap = { version = "8.2.1", features = ["tokio1"] } systemstat = "0.2.4" From 58ea38502bdbf3704c0d94015973d447e38b598f Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 5 Mar 2026 06:22:46 +0100 Subject: [PATCH 07/57] feat: token expiration notifications (#8190) * feat: add token expiration notifications via email, critical alerts, and webhooks - Monitor loop checks for tokens expiring within 7 days and sends email notifications to token owners. Tracks notification state via new `expiry_notified` column on the token table to avoid duplicates. - When tokens expire and are deleted, owners are also notified. - Critical alerts (in-app UI) are gated behind a new instance setting `critical_alerts_on_token_expiry` (off by default); emails are always sent regardless of the setting. - Add TokenExpiringSoon and TokenExpired webhook message variants for workspace webhook integrations. - Frontend: show expiration badges and a warning banner on the tokens table for tokens expiring within 30 days. - Exclude session and ephemeral tokens from all notifications. Co-Authored-By: Claude Opus 4.6 * refactor: use separate token_expiry_notification table for dedup - Replace `expiry_notified` column on token table with a dedicated `token_expiry_notification` table (token, expiration) - Insert notification row on token creation via shared `register_token_expiry_notification()` helper - Delete notification row atomically when sending the notification - Clean up orphaned rows in `delete_expired_items()` - No FK constraint to avoid cascade overhead on token deletions - Add index on expiration column for efficient range queries Co-Authored-By: Claude Opus 4.6 * fix: calendar-based expiration badge and move notification cleanup - Fix daysUntilExpiration to compare calendar dates instead of time diff - Move notification row cleanup from delete_expired_items to check_expiring_tokens to keep it off the hot path - Use simple expiration <= now() index scan instead of NOT EXISTS join Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- ...da8d6a3b425ea0590a66f1db6692dd2ddb437.json | 15 ++ ...856a39c89cdc658b09c478050de5145a45ca4.json | 12 ++ ...aa27e233e60e18b3d35545005eb680701241f.json | 38 ++++ ...7288b39ea7c802007f112eb3d62230d07abb6.json | 38 ++++ ...2000000_add_token_expiry_notified.down.sql | 1 + ...302000000_add_token_expiry_notified.up.sql | 8 + backend/src/main.rs | 23 ++- backend/src/monitor.rs | 167 ++++++++++++++++-- backend/summarized_schema.txt | 2 + backend/windmill-api-auth/src/lib.rs | 33 ++++ backend/windmill-api-users/src/users.rs | 8 + .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/lib.rs | 1 + backend/windmill-common/src/webhook.rs | 146 ++++++++++++--- .../src/lib/components/instanceSettings.ts | 9 + .../components/settings/TokensTable.svelte | 61 ++++++- 16 files changed, 509 insertions(+), 54 deletions(-) create mode 100644 backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json create mode 100644 backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json create mode 100644 backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json create mode 100644 backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json create mode 100644 backend/migrations/20260302000000_add_token_expiry_notified.down.sql create mode 100644 backend/migrations/20260302000000_add_token_expiry_notified.up.sql diff --git a/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json new file mode 100644 index 0000000000..af35d619fa --- /dev/null +++ b/backend/.sqlx/query-a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "a4d973d0f1c293345ad2bfd2472da8d6a3b425ea0590a66f1db6692dd2ddb437" +} diff --git a/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json new file mode 100644 index 0000000000..00604f4bc9 --- /dev/null +++ b/backend/.sqlx/query-a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification WHERE expiration <= now()", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "a6b1c8808c892e62ae4ba04171d856a39c89cdc658b09c478050de5145a45ca4" +} diff --git a/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json new file mode 100644 index 0000000000..9085383617 --- /dev/null +++ b/backend/.sqlx/query-bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE expiration <= now()\n RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "bb446cbb20166f274a7ee6e88abaa27e233e60e18b3d35545005eb680701241f" +} diff --git a/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json new file mode 100644 index 0000000000..015aa7b05a --- /dev/null +++ b/backend/.sqlx/query-d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token_expiry_notification n\n USING token t\n WHERE n.token = t.token\n AND n.expiration > now()\n AND n.expiration <= now() + interval '7 days'\n RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token_prefix", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "label", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + true, + true, + true + ] + }, + "hash": "d7e9b69fef8369117ce057d01d87288b39ea7c802007f112eb3d62230d07abb6" +} diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.down.sql b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql new file mode 100644 index 0000000000..ab827c5de5 --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS token_expiry_notification; diff --git a/backend/migrations/20260302000000_add_token_expiry_notified.up.sql b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql new file mode 100644 index 0000000000..61883f070d --- /dev/null +++ b/backend/migrations/20260302000000_add_token_expiry_notified.up.sql @@ -0,0 +1,8 @@ +-- Tracks pending expiry notifications: row exists = not yet notified. +-- Deleted once the notification is sent. Orphaned rows are harmless (filtered out by the join). +CREATE TABLE token_expiry_notification ( + token VARCHAR(255) PRIMARY KEY, + expiration TIMESTAMPTZ NOT NULL +); + +CREATE INDEX idx_token_expiry_notification_expiration ON token_expiry_notification (expiration); diff --git a/backend/src/main.rs b/backend/src/main.rs index 97980484a5..c8a1021ced 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -38,11 +38,11 @@ use windmill_common::{ agent_workers::AgentConfig, global_settings::{ APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, - CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, + DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, + ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, @@ -99,10 +99,10 @@ use crate::monitor::{ load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db, reload_app_workspaced_route_setting, reload_base_url_setting, reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting, - reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, - reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, - reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, - reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, + reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting, + reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting, + reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, + reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration, }; @@ -1717,6 +1717,11 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload critical alert UI setting"); } } + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING => { + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!(error = %e, "Could not reload critical alerts on token expiry setting"); + } + } "workspace_telemetry_enabled" => { // Read the new value from the database and log it let enabled = sqlx::query_scalar!( diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6baba34a02..37190a5f7a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -44,19 +44,20 @@ use windmill_common::{ apps::APP_WORKSPACED_ROUTE, auth::create_token_for_owner, ee_oss::CriticalErrorChannel, + email_oss::send_email_if_possible, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, - CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, - DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, - KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, - NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, - OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, - POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, + JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, + MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, + POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_INDEX_STRATEGY_SETTING, @@ -76,10 +77,11 @@ use windmill_common::{ DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR, WORKER_CONFIG, WORKER_GROUP, }, - KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERT_MUTE_UI_ENABLED, - CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, - METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, - OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, + KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, + CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, + HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; #[cfg(feature = "parquet")] @@ -207,6 +209,10 @@ pub async fn initial_load( tracing::error!("Error loading critical alert mute ui setting: {e:#}"); } + if let Err(e) = reload_critical_alerts_on_token_expiry_setting(conn).await { + tracing::error!("Error loading critical alerts on token expiry setting: {e:#}"); + } + if let Some(db) = conn.as_sql() { if let Err(e) = load_tag_per_workspace_enabled(db).await { tracing::error!("Error loading default tag per workpsace: {e:#}"); @@ -477,6 +483,21 @@ pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error:: Ok(()) } +pub async fn reload_critical_alerts_on_token_expiry_setting( + conn: &Connection, +) -> error::Result<()> { + if let Ok(Some(serde_json::Value::Bool(t))) = load_value_from_global_settings_with_conn( + conn, + CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, + true, + ) + .await + { + CRITICAL_ALERTS_ON_TOKEN_EXPIRY.store(t, Ordering::Relaxed); + } + Ok(()) +} + pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await; @@ -845,18 +866,82 @@ struct LogFile { hostname: String, } +struct TokenRow { + token_prefix: Option, + label: Option, + email: Option, + workspace_id: Option, +} + +fn is_user_token(label: Option<&str>) -> bool { + match label { + None => true, + Some(l) => l != "session" && !l.starts_with("ephemeral") && !l.starts_with("Ephemeral"), + } +} + +async fn report_token_expiration(db: &DB, token: &TokenRow, expired: bool) { + if !is_user_token(token.label.as_deref()) { + return; + } + let prefix = token.token_prefix.as_deref().unwrap_or("??????????"); + let email_addr = token.email.as_deref().unwrap_or("unknown"); + let token_desc = match token.label.as_deref() { + Some(l) if !l.is_empty() => format!("'{l}' ({prefix}****)"), + _ => format!("{prefix}****"), + }; + + let (alert_message, email_subject, email_body) = if expired { + ( + format!( + "API token {token_desc} of '{email_addr}' has expired and been deleted" + ), + "Windmill: Your API token has expired", + format!( + "Your API token {token_desc} has expired and been deleted.\n\nPlease create a new token if you still need API access." + ), + ) + } else { + ( + format!("API token {token_desc} of '{email_addr}' is expiring soon"), + "Windmill: Your API token is expiring soon", + format!( + "Your API token {token_desc} is expiring soon.\n\nPlease rotate or renew your token to avoid service disruption." + ), + ) + }; + + tracing::info!("{}", alert_message); + if CRITICAL_ALERTS_ON_TOKEN_EXPIRY.load(Ordering::Relaxed) { + report_critical_error( + alert_message, + db.clone(), + token.workspace_id.as_deref(), + None, + ) + .await; + } + if let Some(email) = &token.email { + send_email_if_possible(email_subject, &email_body, email); + } +} + pub async fn delete_expired_items(db: &DB) -> () { - let tokens_deleted_r: std::result::Result, _> = sqlx::query_scalar( + let expired_tokens_r = sqlx::query_as!( + TokenRow, "DELETE FROM token WHERE expiration <= now() - RETURNING concat(substring(token for 10), '*****')", + RETURNING substring(token for 10) as token_prefix, label, email, workspace_id", ) .fetch_all(db) .await; - match tokens_deleted_r { + match expired_tokens_r { Ok(tokens) => { - if tokens.len() > 0 { - tracing::info!("deleted {} tokens: {:?}", tokens.len(), tokens) + if !tokens.is_empty() { + tracing::info!("deleted {} expired tokens", tokens.len()); + for t in &tokens { + report_token_expiration(db, t, true).await; + } } } Err(e) => tracing::error!("Error deleting token: {}", e.to_string()), @@ -1065,6 +1150,41 @@ pub async fn delete_expired_items(db: &DB) -> () { } } +pub async fn check_expiring_tokens(db: &DB) { + // Find tokens expiring within 7 days that still have a pending notification row + let expiring_tokens_r = sqlx::query_as!( + TokenRow, + "DELETE FROM token_expiry_notification n + USING token t + WHERE n.token = t.token + AND n.expiration > now() + AND n.expiration <= now() + interval '7 days' + RETURNING substring(t.token for 10) as token_prefix, t.label, t.email, t.workspace_id", + ) + .fetch_all(db) + .await; + + match expiring_tokens_r { + Ok(tokens) => { + for t in &tokens { + report_token_expiration(db, t, false).await; + } + if !tokens.is_empty() { + tracing::info!("Sent expiration warnings for {} token(s)", tokens.len()); + } + } + Err(e) => tracing::error!("Error checking expiring tokens: {}", e), + } + + // Clean up notification rows whose expiration has passed + if let Err(e) = sqlx::query!("DELETE FROM token_expiry_notification WHERE expiration <= now()") + .execute(db) + .await + { + tracing::error!("Error cleaning up expired token notifications: {}", e); + } +} + /// Delete a batch of expired jobs with LIMIT and SKIP LOCKED for high-scale environments. /// Uses a single transaction per batch to minimize lock duration. /// Returns the number of jobs deleted in this batch. @@ -2052,6 +2172,16 @@ pub async fn monitor_db( } }; + // Run every hour (10 iterations * 30s = 5 minutes) + // Check for tokens expiring within 7 days and send alerts + let check_expiring_tokens_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(10) { + if let Some(db) = conn.as_sql() { + check_expiring_tokens(&db).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2073,6 +2203,7 @@ pub async fn monitor_db( cleanup_worker_group_stats_f, native_triggers_sync_f, cleanup_notify_events_f, + check_expiring_tokens_f, ); } diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 01db7590f5..3ece1ce7ef 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -151,6 +151,8 @@ sqs_trigger: path(char), queue_url(char), aws_resource_path(char), message_attri FK: (workspace_id) -> workspace(id) token: token(char), label(char), expiration(ts), workspace_id(char), owner(char), email(char), super_admin(bool), created_at(ts), last_used_at(ts), scopes(text[]), job(uuid) FK: (workspace_id) -> workspace(id) +token_expiry_notification: token(char), expiration(ts) + INDEX: idx_token_expiry_notification_expiration (expiration) tutorial_progress: email(char), progress(bit64), skipped_all(bool) unique_ext_jwt_token: jwt_hash(bigint), last_used_at(ts) usage: id(char), is_workspace(bool), month_(int), usage(int) diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs index 5acb696bd4..d5ec56d00e 100644 --- a/backend/windmill-api-auth/src/lib.rs +++ b/backend/windmill-api-auth/src/lib.rs @@ -557,6 +557,14 @@ pub async fn create_token_internal( )); } + register_token_expiry_notification( + &mut *tx, + &token, + token_config.label.as_deref(), + token_config.expiration, + ) + .await; + audit_log( &mut *tx, authed, @@ -572,6 +580,31 @@ pub async fn create_token_internal( Ok(token) } +/// Insert a pending expiry notification row for user tokens that have an expiration. +pub async fn register_token_expiry_notification( + tx: &mut sqlx::PgConnection, + token: &str, + label: Option<&str>, + expiration: Option>, +) { + let Some(expiration) = expiration else { return }; + if label == Some("session") + || label.is_some_and(|l| l.starts_with("ephemeral") || l.starts_with("Ephemeral")) + { + return; + } + if let Err(e) = sqlx::query!( + "INSERT INTO token_expiry_notification (token, expiration) VALUES ($1, $2) ON CONFLICT DO NOTHING", + token, + expiration, + ) + .execute(&mut *tx) + .await + { + tracing::error!("Failed to register token expiry notification: {}", e); + } +} + // ------------ Permission helpers ------------ pub fn get_perm_in_extra_perms_for_authed( diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 17ad6cd7fb..ecd1c5fbd3 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1850,6 +1850,14 @@ async fn impersonate( .execute(&mut *tx) .await?; + windmill_api_auth::register_token_expiry_notification( + &mut *tx, + &token, + new_token.label.as_deref(), + new_token.expiration, + ) + .await; + audit_log( &mut *tx, &authed, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index d4d8163ff2..b2d961e173 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -48,6 +48,7 @@ pub const DISABLE_HUB_SETTING: &str = "disable_hub"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; +pub const CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING: &str = "critical_alerts_on_token_expiry"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 643e08fc82..c4e0a47cd8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -193,6 +193,7 @@ lazy_static::lazy_static! { pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); + pub static ref CRITICAL_ALERTS_ON_TOKEN_EXPIRY: AtomicBool = AtomicBool::new(false); pub static ref BASE_URL: Arc> = Arc::new(RwLock::new("".to_string())); pub static ref IS_READY: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); diff --git a/backend/windmill-common/src/webhook.rs b/backend/windmill-common/src/webhook.rs index f1f8508f74..4677084fbe 100644 --- a/backend/windmill-common/src/webhook.rs +++ b/backend/windmill-common/src/webhook.rs @@ -39,29 +39,115 @@ pub enum WebhookPayload { #[serde(tag = "type")] pub enum WebhookMessage { // See https://serde.rs/enum-representations.html#internally-tagged for how this looks in JSON - CreateApp { workspace: String, path: String }, - DeleteApp { workspace: String, path: String }, - UpdateApp { workspace: String, old_path: String, new_path: String }, - CreateFlow { workspace: String, path: String }, - UpdateFlow { workspace: String, old_path: String, new_path: String }, - ArchiveFlow { workspace: String, path: String }, - DeleteFlow { workspace: String, path: String }, - CreateFolder { workspace: String, name: String }, - UpdateFolder { workspace: String, name: String }, - DeleteFolder { workspace: String, name: String }, - DeleteResource { workspace: String, path: String }, - CreateResource { workspace: String, path: String }, - UpdateResource { workspace: String, old_path: String, new_path: String }, - CreateResourceType { name: String }, - DeleteResourceType { name: String }, - UpdateResourceType { name: String }, - CreateScript { workspace: String, path: String, hash: String }, - UpdateScript { workspace: String, path: String, hash: String }, - DeleteScript { workspace: String, hash: String }, - DeleteScriptPath { workspace: String, path: String }, - CreateVariable { workspace: String, path: String }, - UpdateVariable { workspace: String, old_path: String, new_path: String }, - DeleteVariable { workspace: String, path: String }, + CreateApp { + workspace: String, + path: String, + }, + DeleteApp { + workspace: String, + path: String, + }, + UpdateApp { + workspace: String, + old_path: String, + new_path: String, + }, + CreateFlow { + workspace: String, + path: String, + }, + UpdateFlow { + workspace: String, + old_path: String, + new_path: String, + }, + ArchiveFlow { + workspace: String, + path: String, + }, + DeleteFlow { + workspace: String, + path: String, + }, + CreateFolder { + workspace: String, + name: String, + }, + UpdateFolder { + workspace: String, + name: String, + }, + DeleteFolder { + workspace: String, + name: String, + }, + DeleteResource { + workspace: String, + path: String, + }, + CreateResource { + workspace: String, + path: String, + }, + UpdateResource { + workspace: String, + old_path: String, + new_path: String, + }, + CreateResourceType { + name: String, + }, + DeleteResourceType { + name: String, + }, + UpdateResourceType { + name: String, + }, + CreateScript { + workspace: String, + path: String, + hash: String, + }, + UpdateScript { + workspace: String, + path: String, + hash: String, + }, + DeleteScript { + workspace: String, + hash: String, + }, + DeleteScriptPath { + workspace: String, + path: String, + }, + CreateVariable { + workspace: String, + path: String, + }, + UpdateVariable { + workspace: String, + old_path: String, + new_path: String, + }, + DeleteVariable { + workspace: String, + path: String, + }, + TokenExpiringSoon { + workspace: String, + token_prefix: String, + label: String, + owner: String, + expires_at: String, + days_remaining: i64, + }, + TokenExpired { + workspace: String, + token_prefix: String, + label: String, + owner: String, + }, } #[derive(Clone)] @@ -267,6 +353,20 @@ mod tests { new_path: "n".into(), }, WebhookMessage::DeleteVariable { workspace: "w".into(), path: "p".into() }, + WebhookMessage::TokenExpiringSoon { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + expires_at: "2026-03-10T00:00:00Z".into(), + days_remaining: 7, + }, + WebhookMessage::TokenExpired { + workspace: "w".into(), + token_prefix: "abc1234567".into(), + label: "my-token".into(), + owner: "user@example.com".into(), + }, ]; for msg in &messages { diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 25955b74db..80017582f5 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -571,6 +571,15 @@ export const settings: Record = { requiresReloadOnChange: true, ee_only: 'Critical alerts in UI are only available in the EE version' }, + { + label: 'Alert on token expiry', + description: + 'Send critical alerts when API tokens are about to expire (within 7 days) or have expired', + key: 'critical_alerts_on_token_expiry', + fieldType: 'boolean', + storage: 'setting', + ee_only: '' + }, { label: 'Slack', key: 'slack', diff --git a/frontend/src/lib/components/settings/TokensTable.svelte b/frontend/src/lib/components/settings/TokensTable.svelte index 8076b9a17a..af615baa15 100644 --- a/frontend/src/lib/components/settings/TokensTable.svelte +++ b/frontend/src/lib/components/settings/TokensTable.svelte @@ -5,6 +5,8 @@ import { sendUserToast } from '$lib/toast' import CreateToken from './CreateToken.svelte' import Button from '../common/button/Button.svelte' + import Badge from '../common/badge/Badge.svelte' + import Alert from '../common/alert/Alert.svelte' import { Trash } from 'lucide-svelte' // --- Props --- @@ -35,6 +37,46 @@ listTokens() }) + function isUserToken(label: string | undefined): boolean { + if (!label) return true + return label !== 'session' && !label.toLowerCase().startsWith('ephemeral') + } + + function daysUntilExpiration(expiration: string | undefined): number | null { + if (!expiration) return null + const today = new Date() + today.setHours(0, 0, 0, 0) + const exp = new Date(expiration) + exp.setHours(0, 0, 0, 0) + return Math.round((exp.getTime() - today.getTime()) / 86400000) + } + + function expirationBadge( + expiration: string | undefined, + label: string | undefined + ): { + color: 'red' | 'orange' | 'yellow' | 'gray' + text: string + } | null { + if (!isUserToken(label)) return null + const days = daysUntilExpiration(expiration) + if (days === null) return null + if (days < 0) return { color: 'red', text: 'Expired' } + if (days === 0) return { color: 'red', text: 'Expires today' } + if (days === 1) return { color: 'orange', text: 'Expires tomorrow' } + if (days <= 7) return { color: 'orange', text: `Expires in ${days}d` } + if (days <= 30) return { color: 'yellow', text: `Expires in ${days}d` } + return null + } + + let expiringSoonCount = $derived( + tokens.filter((t) => { + if (!isUserToken(t.label)) return false + const days = daysUntilExpiration(t.expiration) + return days !== null && days >= 0 && days <= 7 + }).length + ) + function handleTokenCreated(token: string) { onTokenCreated(token) listTokens() @@ -70,6 +112,11 @@
Authenticate to the Windmill API with access tokens.
+ {#if expiringSoonCount > 0} +
+ +
+ {/if} {#if tokens && tokens.length > 0} - {#each tokens as { token_prefix, expiration, label, scopes }} + {#each tokens as { token_prefix, expiration, label, scopes } (token_prefix)} + {@const badge = expirationBadge(expiration, label)} {token_prefix}**** {label ?? ''} - {displayDate(expiration ?? '')} + +
+ {displayDate(expiration ?? '')} + {#if badge} + {badge.text} + {/if} +
+ {scopes?.join(', ') ?? ''} Date: Thu, 5 Mar 2026 06:29:05 +0100 Subject: [PATCH 08/57] chore(main): release 1.650.0 (#8218) * chore(main): release 1.650.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 21 +++ backend/Cargo.lock | 144 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 108 insertions(+), 87 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae23b08efa..e8f1df09f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## [1.650.0](https://github.com/windmill-labs/windmill/compare/v1.649.0...v1.650.0) (2026-03-05) + + +### Features + +* add move, delete, and duplicate to flow node context menu ([#8050](https://github.com/windmill-labs/windmill/issues/8050)) ([c0c9388](https://github.com/windmill-labs/windmill/commit/c0c9388415716ce77d841bd08a46f94e0a529685)) +* add variable and resource types to flow env variables ([#8214](https://github.com/windmill-labs/windmill/issues/8214)) ([164e499](https://github.com/windmill-labs/windmill/commit/164e499c64dc5eb76fcfb0f8cefbad2df244f610)) +* Ducklake typechecker ([#8118](https://github.com/windmill-labs/windmill/issues/8118)) ([53caecf](https://github.com/windmill-labs/windmill/commit/53caecf1da8d76e246178dfb9b86d330f0ec52fd)) +* make WINDMILL_DIR configurable via environment variable ([#8215](https://github.com/windmill-labs/windmill/issues/8215)) ([424ca59](https://github.com/windmill-labs/windmill/commit/424ca59dfe3e730f5388d9cac4ea7e69773614d3)) +* make WM_END_USER_EMAIL display users from different workspaces ([#8208](https://github.com/windmill-labs/windmill/issues/8208)) ([baf2bcf](https://github.com/windmill-labs/windmill/commit/baf2bcf14da0c8c95bdbbf511fcaee48be33948b)) +* persistent Db manager state in URI ([#8134](https://github.com/windmill-labs/windmill/issues/8134)) ([4bf827b](https://github.com/windmill-labs/windmill/commit/4bf827bea4d44aca8c5ff7aa67ad449dbcf00673)) +* replace hub error toasts with warning alerts and add disable hub setting ([#8225](https://github.com/windmill-labs/windmill/issues/8225)) ([63ebae8](https://github.com/windmill-labs/windmill/commit/63ebae8829a6dc47a4e23c8670b514f042c9d4be)) +* token expiration notifications ([#8190](https://github.com/windmill-labs/windmill/issues/8190)) ([e56ccd2](https://github.com/windmill-labs/windmill/commit/e56ccd200be29e6ac8ea2b04a341b1ce78a307f6)) + + +### Bug Fixes + +* handle multipart stream errors gracefully instead of panicking ([#8226](https://github.com/windmill-labs/windmill/issues/8226)) ([19c065b](https://github.com/windmill-labs/windmill/commit/19c065bed5468c484c8e7a50a6b79ab90153cc0e)) +* improve windows compatibility ([077779e](https://github.com/windmill-labs/windmill/commit/077779ec52f7d3e5fcc93951544bf47bd6dc30b6)) +* wrap set_encryption_key in a single database transaction ([#8212](https://github.com/windmill-labs/windmill/issues/8212)) ([62382fd](https://github.com/windmill-labs/windmill/commit/62382fd2869ea0190dd0c0b714f9cbd35ceddd7a)) + ## [1.649.0](https://github.com/windmill-labs/windmill/compare/v1.648.0...v1.649.0) (2026-03-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d853faa2a0..c7bba49b17 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10709,9 +10709,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.44" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ "proc-macro2", ] @@ -15741,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-nats", @@ -15805,7 +15805,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15818,7 +15818,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "argon2", @@ -15956,7 +15956,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15979,7 +15979,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15992,7 +15992,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16018,7 +16018,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.649.0" +version = "1.650.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16028,7 +16028,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16045,7 +16045,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16068,7 +16068,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16091,7 +16091,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16107,7 +16107,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16127,7 +16127,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16147,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16161,7 +16161,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-nats", @@ -16188,7 +16188,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16213,7 +16213,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16231,7 +16231,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16252,7 +16252,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16272,7 +16272,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16302,7 +16302,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16329,7 +16329,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.649.0" +version = "1.650.0" dependencies = [ "lazy_static", "serde", @@ -16341,7 +16341,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.649.0" +version = "1.650.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16364,7 +16364,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16378,7 +16378,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.649.0" +version = "1.650.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16409,7 +16409,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.649.0" +version = "1.650.0" dependencies = [ "chrono", "lazy_static", @@ -16423,7 +16423,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16442,7 +16442,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.649.0" +version = "1.650.0" dependencies = [ "aes-gcm", "anyhow", @@ -16541,7 +16541,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.649.0" +version = "1.650.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16560,7 +16560,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.649.0" +version = "1.650.0" dependencies = [ "regex", "serde", @@ -16575,7 +16575,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16599,7 +16599,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "futures", @@ -16616,7 +16616,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.649.0" +version = "1.650.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16632,7 +16632,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -16653,7 +16653,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -16684,7 +16684,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-oauth2", @@ -16708,7 +16708,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-stream", @@ -16742,7 +16742,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "futures", @@ -16760,7 +16760,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.649.0" +version = "1.650.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16769,7 +16769,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "lazy_static", @@ -16781,7 +16781,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "serde_json", @@ -16793,7 +16793,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "gosyn", @@ -16805,7 +16805,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "lazy_static", @@ -16817,7 +16817,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "serde_json", @@ -16829,7 +16829,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "nu-parser", @@ -16840,7 +16840,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16851,7 +16851,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16864,7 +16864,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-recursion", @@ -16888,7 +16888,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "lazy_static", @@ -16902,7 +16902,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16919,7 +16919,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "lazy_static", @@ -16934,7 +16934,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "lazy_static", @@ -16953,7 +16953,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "serde", @@ -16964,7 +16964,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-recursion", @@ -17001,7 +17001,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "const_format", @@ -17039,7 +17039,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.649.0" +version = "1.650.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17050,7 +17050,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-recursion", @@ -17079,7 +17079,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17102,7 +17102,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17135,7 +17135,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17155,7 +17155,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17189,7 +17189,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17224,7 +17224,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17247,7 +17247,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17271,7 +17271,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-nats", @@ -17295,7 +17295,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17330,7 +17330,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17358,7 +17358,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-trait", @@ -17381,7 +17381,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17399,7 +17399,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.649.0" +version = "1.650.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index bc1ddf9829..9e8694fbd0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.649.0" +version = "1.650.0" authors.workspace = true edition.workspace = true @@ -76,7 +76,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.649.0" +version = "1.650.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 163217862e..4e24fd6a4b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.649.0 + version: 1.650.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index cc1417c4b3..b326fa80a7 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.649.0"; +export const VERSION = "v1.650.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index b584c540d8..00300458cd 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.649.0"; +export const VERSION = "1.650.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c24e48e02d..e4f18ac398 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.649.0", + "version": "1.650.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.649.0", + "version": "1.650.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index d6c6554459..71c037990d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.649.0", + "version": "1.650.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 37aecedea0..040d21fc67 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.649.0" +wmill = ">=1.650.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 5031c7e889..efb208b861 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.649.0 + version: 1.650.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index ec7cea297d..b26a90786a 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.649.0' + ModuleVersion = '1.650.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 368b0de11b..4490cfb938 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.649.0" +version = "1.650.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 8b07a94942..15c2a5848d 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.649.0", + "version": "1.650.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index a5cd2f4395..4651fcb98b 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.649.0", + "version": "1.650.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index c34844244a..296188f528 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.649.0 +1.650.0 From 0c2ba89c1d676d1b8560808c10e52009a0c21b55 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 5 Mar 2026 07:19:51 +0100 Subject: [PATCH 09/57] feat: add sandbox annotations, volume mounts, for AI sandbox starting with claude (#8058) --- Dockerfile | 6 + ...d55e0379cc84188954037165cbe2d198ef71f.json | 16 + ...5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json} | 7 +- ...c6e9b159a54c179cecb108068597536835f7e.json | 22 + ...6c595401dd2e1f7699a28bf3b79db5e3841f4.json | 23 + ...28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json | 23 + ...11a1dfb1422187200119268b2342b47a960c6.json | 25 + ...ef31e6d647377d37747f7bdc834748a59419e.json | 15 + ...6fc8ce983f857339e6fccf799dc6587964aab.json | 17 + ...485957990eb92fa3ce515895eab0d3f28bfdc.json | 25 + ...580b4c7ba25f279644e3233b63f4f6db0ad98.json | 16 + ...2eb169329116cba57780fa90ecf2bdb910f34.json | 18 + ...367b27171894016c714e41497e69115be1468.json | 76 +++ ...0be7046cf8789d842717b6c793c22a2a05daa.json | 23 + ...67cd897dca4722e4f2076308afdb7ee9fc147.json | 29 + ...8bca8450d09982e582266d215dff521256fa6.json | 22 + ...e919cd25538c0b433bc29bb052c7a7b8568ca.json | 18 + ...e142006825a3911addacdf1a026660b5e2b7f.json | 16 + ...03b566fc75e16e0b7a81204816fd50b3346a5.json | 24 + ...9ba8973a2ef1defa36c6d46d9c1c6406a7c33.json | 16 + ...eea76a8a3079ce80c035087f797cdc410f35b.json | 17 + ...d75ca7014b431239ac1b681f2b26380c719c4.json | 23 + ...050ba7b7e4d9eabba03be251ae9a8017b317d.json | 16 + ...53fd6e9c7606231ad3c522e98cb1fcc14361a.json | 23 + ...425e5f7600e28c728a06235f7ff430a4bd77a.json | 23 + ...9fc2c73e855790e17abf5461b96ea30fbbdb7.json | 23 + ...03b2601630f2d31458bdaf70c2702b2998d89.json | 16 + ...156ada29ae320465d9790dce7e1e8a436d4de.json | 15 + ...2550919fcac56deaf1b3cb36b3e15117936e7.json | 29 + ...263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json | 24 + ...6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json | 17 + ...e78697414b35618c50f8693f4e804bf1d7dbb.json | 29 + ...070c9f117da3dd00ce3247c54a61052a6809c.json | 47 ++ ...8d6affad03a1f388024806a5de3f9cc939c04.json | 23 + ...f33554794444f0dc4e2d2fec158eca5ebe865.json | 28 + ...511b9d3183cb91b740a86944a77a2a964b57d.json | 17 + ...706d78a6f24cb0e614d7d81ba1b643805bf06.json | 22 + ...359b064baf73b890efdc25426261d4eadfee0.json | 23 + ...c5a30037abef63efb5b44dc535c5f45d62a06.json | 41 ++ backend/Cargo.lock | 22 + backend/Cargo.toml | 5 + backend/ee-repo-ref.txt | 2 +- .../20260226000000_add_volumes.down.sql | 1 + .../20260226000000_add_volumes.up.sql | 22 + .../windmill-parser/src/asset_parser.rs | 2 + backend/tests/agent_workers.rs | 327 ++++++++- .../tests/scripts/test_volume_with_claude.ts | 102 +++ backend/tests/volume_tests.rs | 637 ++++++++++++++++++ backend/windmill-api-agent-workers/src/lib.rs | 8 + .../windmill-api-groups/src/granular_acls.rs | 45 +- .../windmill-api-workspaces/src/workspaces.rs | 2 + backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 121 +++- backend/windmill-api/src/health.rs | 38 +- backend/windmill-api/src/job_helpers_oss.rs | 10 +- backend/windmill-api/src/lib.rs | 80 ++- .../windmill-api/src/triggers/http/handler.rs | 6 +- backend/windmill-api/src/volumes_oss.rs | 17 + backend/windmill-common/src/assets.rs | 1 + backend/windmill-common/src/auth.rs | 45 ++ backend/windmill-common/src/worker.rs | 99 +++ backend/windmill-native-triggers/src/lib.rs | 2 +- backend/windmill-test-utils/src/lib.rs | 11 +- backend/windmill-types/src/assets.rs | 1 + backend/windmill-worker-volumes/Cargo.toml | 29 + backend/windmill-worker-volumes/src/lib.rs | 544 +++++++++++++++ .../windmill-worker-volumes/src/volume_oss.rs | 116 ++++ backend/windmill-worker/Cargo.toml | 5 +- .../nsjail/run.bun.config.proto | 12 + backend/windmill-worker/src/bun_executor.rs | 16 +- backend/windmill-worker/src/common.rs | 8 +- backend/windmill-worker/src/deno_executor.rs | 10 +- backend/windmill-worker/src/lib.rs | 3 + .../windmill-worker/src/python_executor.rs | 34 +- backend/windmill-worker/src/volume_oss.rs | 112 +++ backend/windmill-worker/src/worker.rs | 161 ++++- docker/DockerfileSlim | 5 + docker/DockerfileSlimEe | 5 + .../src/lib/components/ApiConnectForm.svelte | 7 + frontend/src/lib/components/EditorBar.svelte | 2 +- .../lib/components/ExploreAssetButton.svelte | 15 +- .../src/lib/components/FilesetEditor.svelte | 27 + .../src/lib/components/ResourceEditor.svelte | 22 +- .../src/lib/components/S3FilePicker.svelte | 3 + .../lib/components/S3FilePickerInner.svelte | 26 +- .../src/lib/components/ScriptBuilder.svelte | 26 +- .../src/lib/components/ScriptEditor.svelte | 2 +- frontend/src/lib/components/ShareModal.svelte | 9 +- .../components/assets/JobAssetsViewer.svelte | 12 +- .../assets/VolumeDetailDrawer.svelte | 107 +++ .../components/assets/VolumesDrawer.svelte | 193 ++++++ frontend/src/lib/components/assets/lib.ts | 4 + .../common/languageIcons/LanguageIcon.svelte | 25 +- .../lib/components/copilot/ResourceGen.svelte | 186 +++++ .../flows/content/FlowInputs.svelte | 18 + .../flows/content/FlowInputsQuick.svelte | 21 + .../components/flows/flowStateUtils.svelte.ts | 2 +- .../flows/map/InsertModuleInner.svelte | 64 +- .../flows/pickers/FlowScriptPicker.svelte | 5 +- .../pickers/FlowScriptPickerQuick.svelte | 5 +- .../flows/pickers/TopLevelNode.svelte | 2 + .../components/graph/graphBuilder.svelte.ts | 2 +- .../components/icons/AssetGenericIcon.svelte | 4 +- .../lib/components/icons/ClaudeIcon.svelte | 25 + .../lib/components/raw_apps/fileTreeUtils.ts | 6 +- frontend/src/lib/components/script_builder.ts | 2 +- .../workspaceSettings/StorageSettings.svelte | 60 +- frontend/src/lib/infer.ts | 67 +- frontend/src/lib/script_helpers.ts | 11 +- .../lib/templates/claude_sandbox.ts.template | 85 +++ frontend/src/lib/workspace_settings.ts | 35 +- .../(root)/(logged)/assets/+page.svelte | 52 +- .../(logged)/workspace_settings/+page.svelte | 6 +- openflow.openapi.yaml | 1 + 114 files changed, 4537 insertions(+), 162 deletions(-) create mode 100644 backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json rename backend/.sqlx/{query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json => query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json} (51%) create mode 100644 backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json create mode 100644 backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json create mode 100644 backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json create mode 100644 backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json create mode 100644 backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json create mode 100644 backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json create mode 100644 backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json create mode 100644 backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json create mode 100644 backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json create mode 100644 backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json create mode 100644 backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json create mode 100644 backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json create mode 100644 backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json create mode 100644 backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json create mode 100644 backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json create mode 100644 backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json create mode 100644 backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json create mode 100644 backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json create mode 100644 backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json create mode 100644 backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json create mode 100644 backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json create mode 100644 backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json create mode 100644 backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json create mode 100644 backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json create mode 100644 backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json create mode 100644 backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json create mode 100644 backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json create mode 100644 backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json create mode 100644 backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json create mode 100644 backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json create mode 100644 backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json create mode 100644 backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json create mode 100644 backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json create mode 100644 backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json create mode 100644 backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json create mode 100644 backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json create mode 100644 backend/migrations/20260226000000_add_volumes.down.sql create mode 100644 backend/migrations/20260226000000_add_volumes.up.sql create mode 100644 backend/tests/scripts/test_volume_with_claude.ts create mode 100644 backend/tests/volume_tests.rs create mode 100644 backend/windmill-api/src/volumes_oss.rs create mode 100644 backend/windmill-worker-volumes/Cargo.toml create mode 100644 backend/windmill-worker-volumes/src/lib.rs create mode 100644 backend/windmill-worker-volumes/src/volume_oss.rs create mode 100644 backend/windmill-worker/src/volume_oss.rs create mode 100644 frontend/src/lib/components/assets/VolumeDetailDrawer.svelte create mode 100644 frontend/src/lib/components/assets/VolumesDrawer.svelte create mode 100644 frontend/src/lib/components/copilot/ResourceGen.svelte create mode 100644 frontend/src/lib/components/icons/ClaudeIcon.svelte create mode 100644 frontend/src/lib/templates/claude_sandbox.ts.template diff --git a/Dockerfile b/Dockerfile index 7cca6ab329..0cc19801d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -262,6 +262,12 @@ COPY --from=oven/bun:1.3.10 /usr/local/bin/bun /usr/bin/bun RUN bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# The installer puts the binary in ~/.local/bin/claude (symlink to ~/.local/share/claude/versions/*) +# Copy it to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json new file mode 100644 index 0000000000..d9d9793cd1 --- /dev/null +++ b/backend/.sqlx/query-00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "00bf3dbd9d3f51dd7fdefcbd654d55e0379cc84188954037165cbe2d198ef71f" +} diff --git a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json similarity index 51% rename from backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json rename to backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json index 84b10ccba6..409faa032f 100644 --- a/backend/.sqlx/query-90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492.json +++ b/backend/.sqlx/query-015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11.json @@ -1,16 +1,17 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT token\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "query": "SELECT group_ FROM usr_to_group WHERE usr = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "token", + "name": "group_", "type_info": "Varchar" } ], "parameters": { "Left": [ + "Text", "Text" ] }, @@ -18,5 +19,5 @@ false ] }, - "hash": "90092c0b3f7612373fcc8fb7a966200118ab308430d4a0cbb5cb16c397246492" + "hash": "015a8551c646f9b027fc23752c5c5c81e520e3ca97dd1cd1e4ebfe3e46c4ad11" } diff --git a/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json new file mode 100644 index 0000000000..52fb375962 --- /dev/null +++ b/backend/.sqlx/query-083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT large_file_storage->>'volume_storage' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "083d69abc8a662bb364cf43b8ffc6e9b159a54c179cecb108068597536835f7e" +} diff --git a/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json new file mode 100644 index 0000000000..6a6b77e650 --- /dev/null +++ b/backend/.sqlx/query-0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0afd4ae50ff7e1b0dcca4b483816c595401dd2e1f7699a28bf3b79db5e3841f4" +} diff --git a/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json new file mode 100644 index 0000000000..0140324406 --- /dev/null +++ b/backend/.sqlx/query-0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0eb54f04a8185085b3f80772f5c28e666f6fbd1ec5ee9d30ee0cdb5e30a68750" +} diff --git a/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json new file mode 100644 index 0000000000..9a6ae60a49 --- /dev/null +++ b/backend/.sqlx/query-14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "14004a7c1641a3157eddd571fea11a1dfb1422187200119268b2342b47a960c6" +} diff --git a/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json new file mode 100644 index 0000000000..9514010409 --- /dev/null +++ b/backend/.sqlx/query-1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "1d2f765c2a71e1154ca5d9f5e52ef31e6d647377d37747f7bdc834748a59419e" +} diff --git a/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json new file mode 100644 index 0000000000..fa67a0797d --- /dev/null +++ b/backend/.sqlx/query-1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at)\n VALUES ($1, $2, $3, $4, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET size_bytes = $3, last_used_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "1e9b9a02f45e6200f4d101bd5336fc8ce983f857339e6fccf799dc6587964aab" +} diff --git a/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json new file mode 100644 index 0000000000..8897e8a7de --- /dev/null +++ b/backend/.sqlx/query-23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by, lease_until, leased_by)\n VALUES ($1, $2, 0, $3, now() + interval '60 seconds', $4)\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET lease_until = now() + interval '60 seconds', leased_by = $4\n WHERE volume.lease_until IS NULL OR volume.lease_until < now()\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "23f47f5207abe0cfaede197aeee485957990eb92fa3ce515895eab0d3f28bfdc" +} diff --git a/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json new file mode 100644 index 0000000000..2010b40667 --- /dev/null +++ b/backend/.sqlx/query-28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "28df7bbe1f54f69640bc76def9e580b4c7ba25f279644e3233b63f4f6db0ad98" +} diff --git a/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json new file mode 100644 index 0000000000..2fdb1ae80d --- /dev/null +++ b/backend/.sqlx/query-3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), updated_by = $5, last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3955e57e216d169c30b1548a2252eb169329116cba57780fa90ecf2bdb910f34" +} diff --git a/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json new file mode 100644 index 0000000000..c73c00c2aa --- /dev/null +++ b/backend/.sqlx/query-40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468.json @@ -0,0 +1,76 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n name as \"name!\",\n size_bytes as \"size_bytes!\",\n file_count as \"file_count!\",\n created_at as \"created_at!\",\n created_by as \"created_by!\",\n updated_at,\n updated_by,\n description as \"description!\",\n last_used_at,\n extra_perms as \"extra_perms!\"\n FROM (\n SELECT\n COALESCE(v.name, a.path) as name,\n COALESCE(v.size_bytes, 0) as size_bytes,\n COALESCE(v.file_count, 0) as file_count,\n COALESCE(v.created_at, a.min_created_at) as created_at,\n COALESCE(v.created_by, 'unknown') as created_by,\n v.updated_at,\n v.updated_by,\n COALESCE(v.description, '') as description,\n v.last_used_at,\n COALESCE(v.extra_perms, '{}'::jsonb) as extra_perms\n FROM (\n SELECT path, MIN(created_at) as min_created_at\n FROM asset\n WHERE workspace_id = $1 AND kind = 'volume'\n GROUP BY path\n ) a\n FULL OUTER JOIN volume v ON v.workspace_id = $1 AND v.name = a.path\n WHERE v.workspace_id = $1 OR a.path IS NOT NULL\n ) combined\n ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "file_count!", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "created_at!", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "updated_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "updated_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "description!", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "last_used_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms!", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + true, + true, + null, + true, + null + ] + }, + "hash": "40d0f6dca30456514cb85e36c6e367b27171894016c714e41497e69115be1468" +} diff --git a/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json new file mode 100644 index 0000000000..2eda021880 --- /dev/null +++ b/backend/.sqlx/query-5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2\n AND (lease_until IS NULL OR lease_until < now())\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af44b46a2e2f1a9adeb39013790be7046cf8789d842717b6c793c22a2a05daa" +} diff --git a/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json new file mode 100644 index 0000000000..dd4a011ee1 --- /dev/null +++ b/backend/.sqlx/query-6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by, extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "extra_perms", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6086849bb08e1b37d6693d2808767cd897dca4722e4f2076308afdb7ee9fc147" +} diff --git a/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json new file mode 100644 index 0000000000..746c306c8f --- /dev/null +++ b/backend/.sqlx/query-712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "712092e5033bc6894025a55ebc58bca8450d09982e582266d215dff521256fa6" +} diff --git a/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json new file mode 100644 index 0000000000..3df9c6c195 --- /dev/null +++ b/backend/.sqlx/query-75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume\n SET size_bytes = $3, file_count = $4,\n updated_at = now(), last_used_at = now(),\n lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "75a03e9e4cba350a104e2e3a95de919cd25538c0b433bc29bb052c7a7b8568ca" +} diff --git a/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json new file mode 100644 index 0000000000..0bf454028b --- /dev/null +++ b/backend/.sqlx/query-769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = now() + interval '60 seconds'\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3 AND lease_until > now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "769035629df5a5034f64bf38992e142006825a3911addacdf1a026660b5e2b7f" +} diff --git a/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json new file mode 100644 index 0000000000..7b85bd9315 --- /dev/null +++ b/backend/.sqlx/query-78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now() AND leased_by = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "78af8bdb6a3ee6396c54f87ff6403b566fc75e16e0b7a81204816fd50b3346a5" +} diff --git a/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json new file mode 100644 index 0000000000..8cbe7146b6 --- /dev/null +++ b/backend/.sqlx/query-7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7ce06d4f623932fce12352be3a09ba8973a2ef1defa36c6d46d9c1c6406a7c33" +} diff --git a/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json new file mode 100644 index 0000000000..594fcf2960 --- /dev/null +++ b/backend/.sqlx/query-7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "7e8e79a7d140be511cedbfe9ff8eea76a8a3079ce80c035087f797cdc410f35b" +} diff --git a/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json new file mode 100644 index 0000000000..c60239fbce --- /dev/null +++ b/backend/.sqlx/query-803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "803abdcd3614437b26c5d2e4f1ad75ca7014b431239ac1b681f2b26380c719c4" +} diff --git a/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json new file mode 100644 index 0000000000..b364fcd4ac --- /dev/null +++ b/backend/.sqlx/query-82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = extra_perms - $1\n WHERE workspace_id = $2 AND name = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "82b3bd95e5d28c4cd4eedcae8cf050ba7b7e4d9eabba03be251ae9a8017b317d" +} diff --git a/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json new file mode 100644 index 0000000000..728141923a --- /dev/null +++ b/backend/.sqlx/query-88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2 AND lease_until > now()", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "88e25dc24bb06237b3677c947ee53fd6e9c7606231ad3c522e98cb1fcc14361a" +} diff --git a/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json new file mode 100644 index 0000000000..26c254eb0b --- /dev/null +++ b/backend/.sqlx/query-907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "907241c195fea227e4a945ee472425e5f7600e28c728a06235f7ff430a4bd77a" +} diff --git a/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json new file mode 100644 index 0000000000..ee64e97d12 --- /dev/null +++ b/backend/.sqlx/query-94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "94d6f598076ad67d68e6f01926c9fc2c73e855790e17abf5461b96ea30fbbdb7" +} diff --git a/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json new file mode 100644 index 0000000000..fc33fd7373 --- /dev/null +++ b/backend/.sqlx/query-9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET lease_until = NULL, leased_by = NULL\n WHERE workspace_id = $1 AND name = $2 AND leased_by = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9662f1e304124fa52db4aa1e80e03b2601630f2d31458bdaf70c2702b2998d89" +} diff --git a/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json new file mode 100644 index 0000000000..86a99d2eaf --- /dev/null +++ b/backend/.sqlx/query-9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9e30b5545a51453205a713a6276156ada29ae320465d9790dce7e1e8a436d4de" +} diff --git a/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json new file mode 100644 index 0000000000..9051a88a50 --- /dev/null +++ b/backend/.sqlx/query-9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT extra_perms, created_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "9f64d6ed0adb609ced1551563062550919fcac56deaf1b3cb36b3e15117936e7" +} diff --git a/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json new file mode 100644 index 0000000000..1f28a0a5a7 --- /dev/null +++ b/backend/.sqlx/query-a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, 0, $3)\n ON CONFLICT (workspace_id, name) DO NOTHING\n RETURNING name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a3970c15271a124307301c0dafa263e7168fa325c5ceb44e9dd1595bdb7e7ce6" +} diff --git a/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json new file mode 100644 index 0000000000..3c7c1ad52a --- /dev/null +++ b/backend/.sqlx/query-ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO volume (workspace_id, name, size_bytes, created_by)\n VALUES ($1, $2, $3, $4)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "ab8daa93bc66d0142b9e9e8d7fa6719fc41b2ca5cb0b7ac5ad73ab01b650c935" +} diff --git a/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json new file mode 100644 index 0000000000..442d55ed25 --- /dev/null +++ b/backend/.sqlx/query-bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "bc61ca62d8f71880facb5d701a6e78697414b35618c50f8693f4e804bf1d7dbb" +} diff --git a/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json new file mode 100644 index 0000000000..0400e6992d --- /dev/null +++ b/backend/.sqlx/query-d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, name, size_bytes, created_by, last_used_at\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "last_used_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true + ] + }, + "hash": "d0869a340c8f34ca7a560d3b4c0070c9f117da3dd00ce3247c54a61052a6809c" +} diff --git a/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json new file mode 100644 index 0000000000..16fc965f6a --- /dev/null +++ b/backend/.sqlx/query-d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "leased_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "d1ad2baf5e3a6f45f1f079d494e8d6affad03a1f388024806a5de3f9cc939c04" +} diff --git a/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json new file mode 100644 index 0000000000..91d89df9cd --- /dev/null +++ b/backend/.sqlx/query-dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "size_bytes", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "dc18db954239c4ebdd3b46cfd34f33554794444f0dc4e2d2fec158eca5ebe865" +} diff --git a/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json new file mode 100644 index 0000000000..c4a1d4cd07 --- /dev/null +++ b/backend/.sqlx/query-eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true)\n WHERE workspace_id = $3 AND name = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "eb79db2aeac7bf246ad56a5f116511b9d3183cb91b740a86944a77a2a964b57d" +} diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json new file mode 100644 index 0000000000..c96961eac4 --- /dev/null +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT token as \"token!\"\n FROM token\n WHERE token LIKE concat($1::text, '%')\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "token!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" +} diff --git a/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json new file mode 100644 index 0000000000..649ee4c387 --- /dev/null +++ b/backend/.sqlx/query-f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT permissioned_as FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0ac12b66c5d3cca680541aed04359b064baf73b890efdc25426261d4eadfee0" +} diff --git a/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json new file mode 100644 index 0000000000..e1ca416938 --- /dev/null +++ b/backend/.sqlx/query-f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT size_bytes, file_count, leased_by, lease_until\n FROM volume WHERE workspace_id = $1 AND name = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "file_count", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "leased_by", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "lease_until", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "f7ba87d5804b9bc05e7156c7c18c5a30037abef63efb5b44dc535c5f45d62a06" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c7bba49b17..7dbc9ee58a 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15798,7 +15798,9 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-test-utils", + "windmill-types", "windmill-worker", + "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", ] @@ -15952,6 +15954,7 @@ dependencies = [ "windmill-trigger-websocket", "windmill-types", "windmill-worker", + "windmill-worker-volumes", ] [[package]] @@ -17499,9 +17502,28 @@ dependencies = [ "windmill-queue", "windmill-runtime-nativets", "windmill-types", + "windmill-worker-volumes", "yaml-rust", ] +[[package]] +name = "windmill-worker-volumes" +version = "1.650.0" +dependencies = [ + "bytes", + "futures", + "lazy_static", + "md-5 0.10.6", + "object_store", + "regex", + "serde", + "serde_json", + "tempfile", + "tokio", + "tracing", + "windmill-common", +] + [[package]] name = "windows" version = "0.56.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9e8694fbd0..a417b66266 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -70,6 +70,7 @@ members = [ "./parsers/windmill-parser-py-imports", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", + "./windmill-worker-volumes", "./windmill-test-utils", "./windmill-api-integration-tests", ] @@ -250,6 +251,8 @@ reqwest.workspace = true windmill-queue = { workspace = true, features = ["failpoints"] } windmill-dep-map.workspace = true windmill-test-utils.workspace = true +windmill-worker-volumes.workspace = true +windmill-types.workspace = true axum.workspace = true serde.workspace = true windmill-api-client.workspace = true @@ -267,6 +270,7 @@ aws-credential-types.workspace = true windmill-api = { path = "./windmill-api", default-features = false } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } +windmill-worker-volumes = { path = "./windmill-worker-volumes" } windmill-dep-map = { path = "./windmill-dep-map" } windmill-types = { path = "./windmill-types" } windmill-common = { path = "./windmill-common", default-features = false } @@ -439,6 +443,7 @@ base64 = "^0.22.1" base32 = "^0" hmac = "0.12.1" sha2 = "0.10.6" +md-5 = "0.10.6" sha1 = "0.10.6" sqlx = { version = "0.8.0", features = [ "macros", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a71cab586d..0e4b8a1791 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6fd5a2ce908235a17975ad4dbdf0051cd89334f3 +151bc3edfe23c160f4f9b0cfaa708beb36c212f4 \ No newline at end of file diff --git a/backend/migrations/20260226000000_add_volumes.down.sql b/backend/migrations/20260226000000_add_volumes.down.sql new file mode 100644 index 0000000000..33dc3804be --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS volume; diff --git a/backend/migrations/20260226000000_add_volumes.up.sql b/backend/migrations/20260226000000_add_volumes.up.sql new file mode 100644 index 0000000000..00f40c9768 --- /dev/null +++ b/backend/migrations/20260226000000_add_volumes.up.sql @@ -0,0 +1,22 @@ +-- Add 'volume' to the asset_kind enum +ALTER TYPE asset_kind ADD VALUE IF NOT EXISTS 'volume'; + +-- Volume metadata table +CREATE TABLE volume ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + size_bytes BIGINT NOT NULL DEFAULT 0, + file_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_by VARCHAR(255) NOT NULL, + updated_at TIMESTAMPTZ, + updated_by VARCHAR(255), + description TEXT NOT NULL DEFAULT '', + lease_until TIMESTAMPTZ, + leased_by VARCHAR(255), + last_used_at TIMESTAMPTZ, + extra_perms JSONB NOT NULL DEFAULT '{}', + PRIMARY KEY (workspace_id, name) +); + +CREATE INDEX idx_volume_last_used ON volume(workspace_id, last_used_at); diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 2463995606..44388690e4 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -18,6 +18,7 @@ pub enum AssetKind { Resource, Ducklake, DataTable, + Volume, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -148,4 +149,5 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("$res:", AssetKind::Resource), ("ducklake://", AssetKind::Ducklake), ("datatable://", AssetKind::DataTable), + ("volume://", AssetKind::Volume), ]; diff --git a/backend/tests/agent_workers.rs b/backend/tests/agent_workers.rs index b6414dc24e..4f12d5f42c 100644 --- a/backend/tests/agent_workers.rs +++ b/backend/tests/agent_workers.rs @@ -1,12 +1,12 @@ #![cfg(all(feature = "private", feature = "agent_worker_server"))] -use windmill_test_utils::*; use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ jobs::{JobPayload, RawCode}, scripts::ScriptLang, }; +use windmill_test_utils::*; fn bun_code(code: &str) -> RawCode { RawCode { @@ -18,8 +18,8 @@ fn bun_code(code: &str) -> RawCode { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - concurrency_settings: - windmill_common::runnable_settings::ConcurrencySettings::default().into(), + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), } } @@ -223,7 +223,10 @@ async fn test_agent_worker_token_and_ping(db: Pool) -> anyhow::Result< .fetch_one(&db) .await?; - assert!(worker_count > 0, "worker ping should be recorded in database"); + assert!( + worker_count > 0, + "worker ping should be recorded in database" + ); // MainLoop ping updates the existing record let resp = http_client @@ -265,3 +268,319 @@ async fn test_agent_worker_multiple_jobs_sequential(db: Pool) -> anyho Ok(()) } + +/// Test the volume HTTP proxy endpoints that agent workers use. +/// +/// Exercises the full volume lifecycle via HTTP: +/// 1. Configure workspace S3 storage (FilesystemStorage) +/// 2. Pre-populate a volume with a file +/// 3. POST /begin — acquire lease, get manifest +/// 4. GET /file/* — download existing file +/// 5. PUT /file/* — upload a new file +/// 6. POST /commit — finalize with stats, release lease +/// 7. Verify DB state and storage +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_e2e(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // 3. POST /begin — acquire lease, get manifest + permissions + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-1", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "begin should succeed, got: {}", + resp.status() + ); + + let begin_body: serde_json::Value = resp.json().await?; + assert!( + begin_body["writable"].as_bool().unwrap(), + "should be writable" + ); + let manifest = begin_body["manifest"].as_object().unwrap(); + assert!( + manifest.contains_key("hello.txt"), + "manifest should contain hello.txt, got: {manifest:?}" + ); + + // 4. GET /file/* — download the existing file + let resp = http + .get(format!("{vol_base}/file/hello.txt")) + .send() + .await?; + assert!( + resp.status().is_success(), + "file download should succeed, got: {}", + resp.status() + ); + let file_bytes = resp.bytes().await?; + assert_eq!( + file_bytes.as_ref(), + b"hello from volume", + "downloaded file content should match" + ); + + // 5. PUT /file/* — upload a new file + let resp = http + .put(format!("{vol_base}/file/output.txt")) + .body(b"written by agent worker".to_vec()) + .send() + .await?; + assert!( + resp.status().is_success(), + "file upload should succeed, got: {}", + resp.status() + ); + + // 6. POST /commit — finalize: report stats, release lease + let resp = http + .post(format!("{vol_base}/commit")) + .json(&json!({ + "worker_name": "test-worker-1", + "deleted_keys": [], + "symlinks": {}, + "file_count": 2, + "size_bytes": 39 + })) + .send() + .await?; + assert!( + resp.status().is_success(), + "commit should succeed, got: {}", + resp.status() + ); + + // 7. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert_eq!(vol_row.file_count, 2, "file_count should be 2"); + assert_eq!(vol_row.size_bytes, 39, "size_bytes should match"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 8. Verify the uploaded file was persisted in storage + let output_path = vol_dir.join("output.txt"); + assert!(output_path.exists(), "output.txt should be in storage"); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Full E2E test: agent worker in HTTP mode runs a Bun script with a volume mount. +/// +/// The worker pulls the job via HTTP, downloads volume files via the server-side +/// volume proxy endpoints, executes the script, and syncs changes back. +#[cfg(all(feature = "parquet", feature = "enterprise"))] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_http_worker_e2e(db: Pool) -> anyhow::Result<()> { + let (_client, port, _server) = init_client_agent_mode(db.clone()).await; + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file + let vol_dir = storage_dir.path().join("volumes").join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job, then run worker with HTTP connection (bun tag) + let code = r#"// volume: test-vol /tmp/data +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by agent worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let uuid = RunJob::from(JobPayload::Code(bun_code(code))) + .push(&db) + .await; + let listener = listen_for_completed_jobs(&db).await; + + let conn = testing_http_connection_with_tags( + port, + vec!["bun".into(), "flow".into(), "dependency".into()], + ) + .await; + + in_test_worker(conn, listener.find(&uuid), port).await; + + let result = completed_job(uuid, &db).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + assert!( + vol_row.lease_until.is_none() || vol_row.lease_until.unwrap() < chrono::Utc::now(), + "lease_until should be cleared or in the past" + ); + + // 5. Verify the new file was written back to the storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by agent worker"); + + Ok(()) +} + +/// Test the volume release endpoint (error/cancel path). +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_agent_worker_volume_release(db: Pool) -> anyhow::Result<()> { + let (client, _port, _server) = init_client_agent_mode(db.clone()).await; + + // Set up filesystem storage + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null + }); + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + let base = client.baseurl(); + let http = client.client(); + let vol_base = format!("{base}/w/test-workspace/volumes/test-vol"); + + // Begin (acquire lease) + let resp = http + .post(format!("{vol_base}/begin")) + .json(&json!({ + "worker_name": "test-worker-2", + "permissioned_as": "u/test-user" + })) + .send() + .await?; + assert!(resp.status().is_success(), "begin should succeed"); + + // Verify lease is held + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert_eq!(leased.as_deref(), Some("test-worker-2")); + + // Release without commit (simulating error path) + let resp = http + .post(format!("{vol_base}/release")) + .json(&json!({ "worker_name": "test-worker-2" })) + .send() + .await?; + assert!(resp.status().is_success(), "release should succeed"); + + // Verify lease is cleared + let leased = sqlx::query_scalar!( + "SELECT leased_by FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await? + .flatten(); + assert!(leased.is_none(), "lease should be released"); + + Ok(()) +} diff --git a/backend/tests/scripts/test_volume_with_claude.ts b/backend/tests/scripts/test_volume_with_claude.ts new file mode 100644 index 0000000000..b6230a19ee --- /dev/null +++ b/backend/tests/scripts/test_volume_with_claude.ts @@ -0,0 +1,102 @@ +// volume: agent-memory .claude +// sandbox + +import Anthropic from "@anthropic-ai/sdk"; +import * as fs from "fs"; +import * as path from "path"; + +type Anthropic = { + api_key: string; + model?: string; +}; + +export async function main(anthropic_resource: Anthropic) { + const claudeDir = ".claude"; + const results: Record = {}; + + // --- Step 1: Verify volume is mounted at the relative path --- + results["volume_exists"] = fs.existsSync(claudeDir); + if (!results["volume_exists"]) { + fs.mkdirSync(claudeDir, { recursive: true }); + } + + const testFile = path.join(claudeDir, "mount-check.txt"); + fs.writeFileSync(testFile, "volume mount verified"); + results["volume_writable"] = fs.readFileSync(testFile, "utf-8") === "volume mount verified"; + + // --- Step 2: Create memory directory structure --- + const memoryDir = path.join(claudeDir, "memory"); + fs.mkdirSync(memoryDir, { recursive: true }); + + const memoryFile = path.join(memoryDir, "MEMORY.md"); + fs.writeFileSync(memoryFile, "# Agent Memory\n\nThis file persists across runs.\n"); + results["memory_file_created"] = fs.existsSync(memoryFile); + + // --- Step 3: Call Claude to generate structured content --- + const client = new Anthropic({ apiKey: anthropic_resource.api_key }); + const model = anthropic_resource.model ?? "claude-sonnet-4-20250514"; + + const response = await client.messages.create({ + model, + max_tokens: 256, + messages: [ + { + role: "user", + content: + 'Return a JSON object with exactly these keys: "greeting" (a short hello), "timestamp" (current ISO date you estimate), "items" (array of 3 random fruit names). Only return the JSON, no markdown.', + }, + ], + }); + + const assistantText = + response.content[0].type === "text" ? response.content[0].text : ""; + results["claude_responded"] = assistantText.length > 0; + results["claude_model"] = response.model; + results["claude_stop_reason"] = response.stop_reason; + + let parsed: Record = {}; + try { + parsed = JSON.parse(assistantText); + results["claude_valid_json"] = true; + results["claude_has_greeting"] = "greeting" in parsed; + results["claude_has_items"] = + Array.isArray(parsed.items) && parsed.items.length === 3; + } catch { + results["claude_valid_json"] = false; + } + + // --- Step 4: Write Claude's response to volume --- + const responsePath = path.join(claudeDir, "claude-response.json"); + fs.writeFileSync(responsePath, JSON.stringify(parsed, null, 2)); + results["response_written"] = fs.existsSync(responsePath); + + // --- Step 5: Read back and verify --- + const readBack = fs.readFileSync(responsePath, "utf-8"); + const readParsed = JSON.parse(readBack); + results["readback_matches"] = + JSON.stringify(readParsed) === JSON.stringify(parsed); + + // --- Step 6: List all volume contents --- + const volumeContents = fs.readdirSync(claudeDir); + results["volume_files"] = volumeContents; + results["volume_file_count"] = volumeContents.length; + + // --- Step 7: Verify memory file persists --- + const memoryContent = fs.readFileSync(memoryFile, "utf-8"); + results["memory_persisted"] = memoryContent.includes("Agent Memory"); + + // --- Summary --- + const allChecks = [ + results["volume_exists"] || true, + results["volume_writable"], + results["claude_responded"], + results["claude_valid_json"], + results["response_written"], + results["readback_matches"], + results["memory_file_created"], + results["memory_persisted"], + ]; + results["all_passed"] = allChecks.every(Boolean); + + return results; +} diff --git a/backend/tests/volume_tests.rs b/backend/tests/volume_tests.rs new file mode 100644 index 0000000000..9df30c5615 --- /dev/null +++ b/backend/tests/volume_tests.rs @@ -0,0 +1,637 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{JobPayload, RawCode}; +use windmill_common::scripts::ScriptLang; +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_volume_insert(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "test-volume", + 1024_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT workspace_id, name, size_bytes, created_by, last_used_at + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-volume" + ) + .fetch_one(&db) + .await?; + + assert_eq!(row.workspace_id, "test-workspace"); + assert_eq!(row.name, "test-volume"); + assert_eq!(row.size_bytes, 1024); + assert_eq!(row.created_by, "test-user"); + assert!(row.last_used_at.is_none()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_upsert_size(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 500_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 500); + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by, last_used_at) + VALUES ($1, $2, $3, $4, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET size_bytes = $3, last_used_at = now()", + "test-workspace", + "upsert-vol", + 2048_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT size_bytes, last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "upsert-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.size_bytes, 2048); + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_last_used(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "used-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_none()); + + sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT last_used_at FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "used-vol" + ) + .fetch_one(&db) + .await?; + assert!(row.last_used_at.is_some()); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_update_nonexistent_noop(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "UPDATE volume SET last_used_at = now() WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "nonexistent-vol" + ) + .execute(&db) + .await?; + + assert_eq!(result.rows_affected(), 0); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_list_multiple(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + for i in 0..5 { + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + format!("vol-{}", i), + (i * 100) as i64, + "test-user" + ) + .execute(&db) + .await?; + } + + let rows = sqlx::query!( + "SELECT name, size_bytes FROM volume WHERE workspace_id = $1 ORDER BY name", + "test-workspace" + ) + .fetch_all(&db) + .await?; + + assert_eq!(rows.len(), 5); + assert_eq!(rows[0].name, "vol-0"); + assert_eq!(rows[0].size_bytes, 0); + assert_eq!(rows[4].name, "vol-4"); + assert_eq!(rows[4].size_bytes, 400); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_delete(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "deleteme", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(1)); + + sqlx::query!( + "DELETE FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .execute(&db) + .await?; + + let count = sqlx::query_scalar!( + "SELECT count(*) FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "deleteme" + ) + .fetch_one(&db) + .await?; + assert_eq!(count, Some(0)); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_workspace_fk_constraint(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "nonexistent-workspace", + "vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("foreign key"), + "Expected foreign key violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_primary_key_uniqueness(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + let result = sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "unique-vol", + 200_i64, + "another-user" + ) + .execute(&db) + .await; + + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("duplicate key") || err.contains("unique"), + "Expected unique violation, got: {}", + err + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_volume_extra_perms(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Insert volume with default (empty) extra_perms + sqlx::query!( + "INSERT INTO volume (workspace_id, name, size_bytes, created_by) + VALUES ($1, $2, $3, $4)", + "test-workspace", + "perms-vol", + 100_i64, + "test-user" + ) + .execute(&db) + .await?; + + // Default extra_perms should be empty object + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + // Set extra_perms via jsonb_set (same pattern as granular_acls.rs) + sqlx::query!( + "UPDATE volume SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2::bool), true) + WHERE workspace_id = $3 AND name = $4", + &vec!["u/alice".to_string()], + true, + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + let perms = row.extra_perms.as_object().unwrap(); + assert_eq!(perms.get("u/alice").and_then(|v| v.as_bool()), Some(true)); + + // Remove a permission entry + sqlx::query!( + "UPDATE volume SET extra_perms = extra_perms - $1 + WHERE workspace_id = $2 AND name = $3", + "u/alice", + "test-workspace", + "perms-vol" + ) + .execute(&db) + .await?; + + let row = sqlx::query!( + "SELECT extra_perms FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "perms-vol" + ) + .fetch_one(&db) + .await?; + assert_eq!(row.extra_perms, serde_json::json!({})); + + Ok(()) +} + +#[test] +fn test_parse_volume_annotations_python() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"# sandbox +# volume: training-data /tmp/training +# volume: models /opt/models + +def main(): + pass +"#; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 2); + assert_eq!(volumes[0].name, "training-data"); + assert_eq!(volumes[0].target, "/tmp/training"); + assert_eq!(volumes[1].name, "models"); + assert_eq!(volumes[1].target, "/opt/models"); +} + +#[test] +fn test_parse_volume_annotations_typescript() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = r#"// sandbox +// volume: datasets /tmp/datasets + +export async function main() { + return "hello"; +} +"#; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "datasets"); + assert_eq!(volumes[0].target, "/tmp/datasets"); +} + +#[test] +fn test_parse_volume_annotations_no_prefix_match() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "def main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_parse_volume_annotations_empty_script() { + use windmill_worker_volumes::parse_volume_annotations; + + let volumes = parse_volume_annotations("", "#"); + assert!(volumes.is_empty()); +} + +#[test] +fn test_sandbox_annotation_python() { + use windmill_common::worker::PythonAnnotations; + + let content = "# sandbox\n# volume: data /tmp/data\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_sandbox_annotation_typescript() { + use windmill_common::worker::TypeScriptAnnotations; + + let content = "// sandbox\n// volume: data /tmp/data\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); +} + +#[test] +fn test_volume_comment_prefix_selection() { + use windmill_common::scripts::ScriptLang; + + let get_prefix = |lang: &ScriptLang| -> &str { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + } + }; + + assert_eq!(get_prefix(&ScriptLang::Python3), "#"); + assert_eq!(get_prefix(&ScriptLang::Bash), "#"); + assert_eq!(get_prefix(&ScriptLang::Powershell), "#"); + assert_eq!(get_prefix(&ScriptLang::Ansible), "#"); + assert_eq!(get_prefix(&ScriptLang::Ruby), "#"); + assert_eq!(get_prefix(&ScriptLang::Deno), "//"); + assert_eq!(get_prefix(&ScriptLang::Bun), "//"); + assert_eq!(get_prefix(&ScriptLang::Bunnative), "//"); + assert_eq!(get_prefix(&ScriptLang::Nativets), "//"); + assert_eq!(get_prefix(&ScriptLang::Go), "//"); +} + +#[test] +fn test_volume_mount_struct() { + use windmill_worker_volumes::VolumeMount; + + let mount = VolumeMount { name: "test-vol".to_string(), target: "/mnt/data".to_string() }; + assert_eq!(mount.name, "test-vol"); + assert_eq!(mount.target, "/mnt/data"); +} + +#[test] +fn test_parse_volume_relative_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let volumes = parse_volume_annotations(content, "//"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "agent-memory"); + assert_eq!(volumes[0].target, ".claude"); +} + +#[test] +fn test_parse_volume_relative_nested_path() { + use windmill_worker_volumes::parse_volume_annotations; + + let content = "# volume: data data/models\ndef main():\n pass"; + let volumes = parse_volume_annotations(content, "#"); + assert_eq!(volumes.len(), 1); + assert_eq!(volumes[0].name, "data"); + assert_eq!(volumes[0].target, "data/models"); +} + +#[cfg(feature = "private")] +#[test] +fn test_volume_nsjail_mount() { + use std::path::Path; + use windmill_worker_volumes::volume_nsjail_mount; + + let result = volume_nsjail_mount(Path::new("/tmp/volumes/data"), "/mnt/data"); + assert!(result.contains("src: \"/tmp/volumes/data\"")); + assert!(result.contains("dst: \"/mnt/data\"")); + assert!(result.contains("is_bind: true")); + assert!(result.contains("rw: true")); +} + +#[test] +fn test_sync_stats_default() { + use windmill_worker_volumes::SyncStats; + + let stats = SyncStats { new_size_bytes: 0, file_count: 0, uploaded: 0, skipped: 0 }; + assert_eq!(stats.new_size_bytes, 0); + assert_eq!(stats.file_count, 0); + assert_eq!(stats.uploaded, 0); + assert_eq!(stats.skipped, 0); +} + +#[test] +fn test_asset_kind_volume_variant() { + use windmill_types::assets::AssetKind; + + let kind = AssetKind::Volume; + let serialized = serde_json::to_string(&kind).unwrap(); + assert_eq!(serialized, "\"volume\""); + + let deserialized: AssetKind = serde_json::from_str("\"volume\"").unwrap(); + assert!(matches!(deserialized, AssetKind::Volume)); +} + +/// E2E test: run a bun script with volume mount through a SQL-connected worker. +/// Pre-populates the volume in filesystem storage, verifies the script can read +/// files and write new ones, then checks sync-back to storage and DB state. +#[cfg(feature = "parquet")] +#[sqlx::test(fixtures("base"))] +async fn test_volume_sql_worker_e2e(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // 1. Set up filesystem-based object storage in a temp dir + let storage_dir = tempfile::tempdir()?; + let storage_root = storage_dir.path().to_string_lossy().to_string(); + + let lfs_config = json!({ + "type": "FilesystemStorage", + "root_path": storage_root, + "public_resource": null, + "advanced_permissions": null, + "volume_storage": "primary" + }); + + sqlx::query!( + "UPDATE workspace_settings SET large_file_storage = $1 WHERE workspace_id = $2", + lfs_config, + "test-workspace" + ) + .execute(&db) + .await?; + + // 2. Pre-populate the volume with a file (workspace-namespaced path) + let vol_dir = storage_dir + .path() + .join("volumes") + .join("test-workspace") + .join("test-vol"); + std::fs::create_dir_all(&vol_dir)?; + std::fs::write(vol_dir.join("hello.txt"), b"hello from volume")?; + + // 3. Push the job and run with SQL-connected worker + let code = r#"// volume: test-vol /tmp/data + +import { readFileSync, writeFileSync, existsSync } from "fs"; + +export function main() { + const content = readFileSync("/tmp/data/hello.txt", "utf-8"); + writeFileSync("/tmp/data/output.txt", "written by sql worker"); + return { + read_content: content, + output_exists: existsSync("/tmp/data/output.txt"), + }; +}"#; + + let job = JobPayload::Code(RawCode { + hash: None, + content: code.to_string(), + path: None, + language: ScriptLang::Bun, + lock: None, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + }); + + let result = run_job_in_new_worker_until_complete(&db, false, job, port).await; + + assert!(result.success, "job should succeed: {:?}", result.result); + let json = result.json_result().expect("should have JSON result"); + assert_eq!(json["read_content"], json!("hello from volume")); + assert_eq!(json["output_exists"], json!(true)); + + // 4. Verify volume DB row was updated + let vol_row = sqlx::query!( + "SELECT size_bytes, file_count, leased_by, lease_until + FROM volume WHERE workspace_id = $1 AND name = $2", + "test-workspace", + "test-vol" + ) + .fetch_optional(&db) + .await?; + + let vol_row = vol_row.expect("volume row should exist"); + assert!( + vol_row.file_count >= 2, + "should have at least 2 files (hello.txt + output.txt), got: {}", + vol_row.file_count + ); + assert!(vol_row.size_bytes > 0, "size_bytes should be > 0"); + assert!(vol_row.leased_by.is_none(), "lease should be released"); + + // 5. Verify the new file was written back to storage + let output_path = vol_dir.join("output.txt"); + assert!( + output_path.exists(), + "output.txt should be synced back to storage" + ); + let output_content = std::fs::read_to_string(&output_path)?; + assert_eq!(output_content, "written by sql worker"); + + Ok(()) +} diff --git a/backend/windmill-api-agent-workers/src/lib.rs b/backend/windmill-api-agent-workers/src/lib.rs index d2913a217c..b02a7098b7 100644 --- a/backend/windmill-api-agent-workers/src/lib.rs +++ b/backend/windmill-api-agent-workers/src/lib.rs @@ -51,4 +51,12 @@ impl AgentCache { pub fn new() -> Self { AgentCache {} } + + pub async fn extract_worker_name( + &self, + _token: &str, + _db: &windmill_common::DB, + ) -> Option { + None + } } diff --git a/backend/windmill-api-groups/src/granular_acls.rs b/backend/windmill-api-groups/src/granular_acls.rs index 048a4120f5..da9267419d 100644 --- a/backend/windmill-api-groups/src/granular_acls.rs +++ b/backend/windmill-api-groups/src/granular_acls.rs @@ -24,7 +24,7 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 18] = [ +const KINDS: [&str; 19] = [ "script", "group_", "resource", @@ -43,6 +43,7 @@ const KINDS: [&str; 18] = [ "gcp_trigger", "sqs_trigger", "email_trigger", + "volume", ]; pub fn workspaced_service() -> Router { @@ -77,7 +78,7 @@ async fn add_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -89,6 +90,22 @@ async fn add_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -243,6 +260,22 @@ async fn remove_granular_acl( } else if kind == "group_" { crate::groups::require_is_owner(path, &authed.username, &authed.groups, &w_id, &db) .await?; + } else if kind == "volume" { + let created_by = sqlx::query_scalar!( + "SELECT created_by FROM volume WHERE name = $1 AND workspace_id = $2", + path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("volume '{path}' not found")))?; + // created_by is stored with u/ prefix (from job.permissioned_as) + let owner_username = created_by.strip_prefix("u/").unwrap_or(&created_by); + if owner_username != authed.username { + return Err(Error::NotAuthorized( + "Only the volume owner or an admin can modify permissions".to_string(), + )); + } } else { require_owner_of_path(&authed, path)?; } @@ -250,7 +283,7 @@ async fn remove_granular_acl( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" || kind == "folder" { + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { "name" } else { "path" @@ -380,7 +413,11 @@ async fn get_granular_acls( let mut tx = user_db.begin(&authed).await?; - let identifier = if kind == "group_" { "name" } else { "path" }; + let identifier = if kind == "group_" || kind == "folder" || kind == "volume" { + "name" + } else { + "path" + }; let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!( "SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2" )) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 80e497ba97..a85383e496 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -302,6 +302,8 @@ struct LargeFileStorageWithSecondary { large_file_storage: LargeFileStorage, #[serde(default)] secondary_storage: HashMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + volume_storage: Option, } #[derive(Deserialize, Debug)] struct EditLargeFileStorageConfig { diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 7af8137630..de8935d097 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -70,6 +70,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } windmill-worker = { workspace = true, optional = true } +windmill-worker-volumes.workspace = true windmill-dep-map.workspace = true tokio.workspace = true tokio-stream.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4e24fd6a4b..86d6084980 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -15198,6 +15198,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] responses: "200": @@ -15243,6 +15244,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -15299,6 +15301,7 @@ paths: gcp_trigger, sqs_trigger, email_trigger, + volume, ] requestBody: description: acl to add @@ -17282,7 +17285,90 @@ paths: path: type: string description: The asset path - + + + /w/{workspace}/volumes/list: + get: + summary: List all volumes in the workspace + operationId: listVolumes + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of volumes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Volume" + + /w/{workspace}/volumes/storage: + get: + summary: Get the volume storage name (secondary storage) or null for primary + operationId: getVolumeStorage + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: volume storage name or null + content: + application/json: + schema: + type: string + nullable: true + + /w/{workspace}/volumes/create: + post: + summary: Create a new volume + operationId: createVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - name + properties: + name: + type: string + responses: + "200": + description: volume created + content: + text/plain: + schema: + type: string + + /w/{workspace}/volumes/delete/{name}: + delete: + summary: Delete a volume (admin only) + operationId: deleteVolume + tags: + - volume + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: name + in: path + required: true + schema: + type: string + responses: + "200": + description: volume deleted + content: + text/plain: + schema: + type: string /mcp/w/{workspace}/list_tools: get: @@ -23997,6 +24083,7 @@ components: - resource - ducklake - datatable + - volume Asset: type: object properties: @@ -24005,6 +24092,38 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + Volume: + type: object + required: + - name + - size_bytes + - file_count + - created_at + - created_by + properties: + name: + type: string + size_bytes: + type: integer + format: int64 + file_count: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + nullable: true + last_used_at: + type: string + format: date-time + nullable: true + extra_perms: + type: object + additionalProperties: true ProtectionRuleset: type: object description: A workspace protection rule defining restrictions and bypass permissions diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index d861f338ef..60bd290f66 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -240,11 +240,7 @@ async fn check_database_detailed(db: &DB) -> DatabaseHealth { let check = check_database_with_latency(db).await; let pool = get_pool_stats(db); - DatabaseHealth { - healthy: check.healthy, - latency_ms: check.latency_ms, - pool, - } + DatabaseHealth { healthy: check.healthy, latency_ms: check.latency_ms, pool } } async fn check_worker_count(db: &DB) -> i64 { @@ -295,13 +291,7 @@ async fn check_workers_detailed(db: &DB) -> WorkersHealth { let healthy = active_count > 0; - WorkersHealth { - healthy, - active_count, - worker_groups, - min_version, - versions, - } + WorkersHealth { healthy, active_count, worker_groups, min_version, versions } } async fn check_queue(db: &DB) -> QueueHealth { @@ -333,10 +323,7 @@ fn get_version() -> String { /// Spawn a background task that performs a health check every 10 seconds. /// Updates the cache and prometheus metrics continuously. -pub fn start_health_check_loop( - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { +pub fn start_health_check_loop(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { tokio::spawn(async move { let mut interval = tokio::time::interval(Duration::from_secs(10)); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); @@ -550,10 +537,7 @@ async fn health_status( } /// Detailed health check - requires DB authentication (always fresh, no caching) -async fn health_detailed( - _authed: ApiAuthed, - Extension(db): Extension, -) -> impl IntoResponse { +async fn health_detailed(_authed: ApiAuthed, Extension(db): Extension) -> impl IntoResponse { let checked_at = Utc::now(); let database = check_database_detailed(&db).await; let readiness = check_readiness(); @@ -564,12 +548,7 @@ async fn health_detailed( status: HealthStatus::Unhealthy, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: None, - queue: None, - readiness, - }, + checks: HealthChecks { database, workers: None, queue: None, readiness }, }; return (StatusCode::SERVICE_UNAVAILABLE, Json(response)); } @@ -587,12 +566,7 @@ async fn health_detailed( status, checked_at, version: get_version(), - checks: HealthChecks { - database, - workers: Some(workers), - queue: Some(queue), - readiness, - }, + checks: HealthChecks { database, workers: Some(workers), queue: Some(queue), readiness }, }; let status_code = if status == HealthStatus::Unhealthy { diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 01f5d67bd3..23f20459c3 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -12,15 +12,15 @@ use windmill_types::s3::StorageResourceType; #[cfg(all(feature = "parquet", not(feature = "private")))] use crate::db::{ApiAuthed, OptJobAuthed, DB}; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; -#[cfg(not(feature = "private"))] -use windmill_object_store::ObjectStoreResource; -#[cfg(all(feature = "parquet", not(feature = "private")))] use std::sync::Arc; +#[cfg(all(feature = "parquet", not(feature = "private")))] +use windmill_common::db::UserDB; #[cfg(not(feature = "private"))] use windmill_common::error; #[cfg(all(feature = "parquet", not(feature = "private")))] -use windmill_common::db::UserDB; +use windmill_object_store::object_store_reexports::{ObjectStore, PutMultipartOpts, PutResult}; +#[cfg(not(feature = "private"))] +use windmill_object_store::ObjectStoreResource; #[cfg(all(feature = "parquet", not(feature = "private")))] use bytes::Bytes; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 528e60b02f..073c1a8aa1 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -170,6 +170,9 @@ pub mod users_ee; mod users_oss; mod utils; mod variables; +#[cfg(feature = "private")] +pub mod volumes_ee; +mod volumes_oss; pub mod webhook_util; mod workspaces; #[cfg(feature = "private")] @@ -248,6 +251,74 @@ type IndexReader = windmill_indexer::completed_runs_oss::IndexReader; #[cfg(feature = "tantivy")] type ServiceLogIndexReader = windmill_indexer::service_logs_oss::ServiceLogIndexReader; +/// Worker name derived from the agent JWT token, used to authenticate volume operations. +/// Defined unconditionally so volume endpoint handlers can reference it regardless of +/// whether agent_worker_server is enabled (the extension is only populated on the agent path). +#[derive(Clone)] +pub struct AgentWorkerName(pub String); + +/// Middleware that injects a synthetic `ApiAuthed` and JWT-derived worker name +/// into request extensions. +/// +/// Used for volume proxy endpoints under the agent_workers path, where the +/// agent JWT auth layer has already validated the request. The volume handlers +/// need `ApiAuthed` to resolve the workspace S3 client, but the agent JWT +/// format is incompatible with the standard auth extractor. +/// +/// The worker name is extracted from the JWT claims rather than trusting +/// self-reported values in request bodies/query params. +#[cfg(feature = "agent_worker_server")] +async fn inject_agent_authed( + request: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut request = request; + + // Extract worker name from agent JWT via AgentCache + // (OSS returns None; EE decodes the JWT and returns the worker name) + { + let extracted = { + let token = request + .headers() + .get(axum::http::header::AUTHORIZATION) + .and_then(|v| v.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ").map(|t| t.to_string())); + let cache = request.extensions().get::>().cloned(); + let db = request.extensions().get::().cloned(); + match (token, cache, db) { + (Some(token), Some(cache), Some(db)) => Some((token, cache, db)), + _ => None, + } + }; + + if let Some((token, cache, db)) = extracted { + if let Some(worker_name) = cache.extract_worker_name(&token, &db).await { + request + .extensions_mut() + .insert(AgentWorkerName(worker_name)); + } + } + } + + request + .extensions_mut() + .insert(windmill_api_auth::OptJobAuthed { + authed: ApiAuthed { + email: "agent-worker@windmill.dev".to_string(), + username: "agent-worker".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }, + job_id: None, + }); + next.run(request).await +} + pub async fn run_server( db: DB, job_index_reader: Option, @@ -513,6 +584,7 @@ pub async fn run_server( users::workspaced_service().layer(Extension(argon2.clone())), ) .nest("/variables", variables::workspaced_service()) + .nest("/volumes", volumes_oss::workspaced_service()) .nest("/workers", windmill_api_workers::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) @@ -626,7 +698,13 @@ pub async fn run_server( .nest("/w/:workspace_id/agent_workers", { #[cfg(feature = "agent_worker_server")] { - agent_workers_router.layer(Extension(agent_cache.clone())) + agent_workers_router + .nest( + "/volumes", + volumes_oss::agent_workspaced_service() + .layer(axum::middleware::from_fn(inject_agent_authed)), + ) + .layer(Extension(agent_cache.clone())) } #[cfg(not(feature = "agent_worker_server"))] { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index f3ec05348a..9d95ead840 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -369,7 +369,11 @@ async fn route_job( let s3_object = s3_client.get(&path).await; let s3_object = match s3_object { - Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) if trigger.is_static_website => { + Err( + windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { + .. + }, + ) if trigger.is_static_website => { // fallback to index.html if the file is not found let path = windmill_object_store::object_store_reexports::Path::from(format!( "{}/index.html", diff --git a/backend/windmill-api/src/volumes_oss.rs b/backend/windmill-api/src/volumes_oss.rs new file mode 100644 index 0000000000..26b1c2cd46 --- /dev/null +++ b/backend/windmill-api/src/volumes_oss.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use crate::volumes_ee::*; + +#[cfg(not(feature = "private"))] +use axum::Router; + +#[cfg(not(feature = "private"))] +pub fn workspaced_service() -> Router { + Router::new() +} + +#[cfg(not(feature = "private"))] +#[allow(dead_code)] +pub fn agent_workspaced_service() -> Router { + Router::new() +} diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 669614d64a..2968493ecc 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -72,6 +72,7 @@ pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetK windmill_parser::asset_parser::AssetKind::Resource => AssetKind::Resource, windmill_parser::asset_parser::AssetKind::Ducklake => AssetKind::Ducklake, windmill_parser::asset_parser::AssetKind::DataTable => AssetKind::DataTable, + windmill_parser::asset_parser::AssetKind::Volume => AssetKind::Volume, } } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index ccd9b5d2a0..c0555f5093 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -95,6 +95,51 @@ impl PermsCache { } } +/// Check a user's access level against an `extra_perms` JSONB object. +/// +/// Returns `None` if the user has no matching entry (no access). +/// Returns `Some(true)` if the user (or any of their groups) has write access. +/// Returns `Some(false)` if the user (or any of their groups) has read-only access. +pub fn check_extra_perms( + extra_perms: &serde_json::Map, + username: &str, + groups: &[String], +) -> Option { + // Check direct user permission + let user_key = if username.starts_with("u/") { + username.to_string() + } else { + format!("u/{username}") + }; + if let Some(v) = extra_perms.get(&user_key) { + return Some(v.as_bool().unwrap_or(false)); + } + + // Check group permissions — return highest access level found + let mut found = false; + let mut write = false; + for g in groups { + let key = if g.starts_with("g/") { + g.to_string() + } else { + format!("g/{g}") + }; + if let Some(v) = extra_perms.get(&key) { + found = true; + if v.as_bool().unwrap_or(false) { + write = true; + break; + } + } + } + + if found { + Some(write) + } else { + None + } +} + pub fn has_expired(expiration_time: DateTime, take: Option) -> bool { let now = Utc::now(); diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 3195562368..b3876fbcfa 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -354,6 +354,45 @@ impl HttpClient { ))) } } + + pub async fn get_bytes(&self, url: &str) -> anyhow::Result { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .get(format!("{}{}", base_url, url)) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(response.bytes().await?) + } else { + Err(anyhow::anyhow!( + "HTTP agent request GET {} failed {}", + url, + response.status() + )) + } + } + + pub async fn put_bytes(&self, url: &str, bytes: Bytes) -> anyhow::Result<()> { + let base_url = self.base_internal_url.clone(); + let response = self + .client + .put(format!("{}{}", base_url, url)) + .body(bytes) + .send() + .await + .map_err(|e| anyhow::anyhow!(e))?; + if response.status().is_success() { + Ok(()) + } else { + Err(anyhow::anyhow!( + "HTTP agent request PUT {} failed {}", + url, + response.status() + )) + } + } } #[derive(Clone)] @@ -698,6 +737,7 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub sandbox: bool, } #[annotations("//")] @@ -711,6 +751,7 @@ pub struct TypeScriptAnnotations { pub nodejs: bool, pub native: bool, pub nobundling: bool, + pub sandbox: bool, } #[annotations("--")] @@ -2169,4 +2210,62 @@ mod tests { ); assert_ne!(a, b); } + + #[test] + fn test_python_sandbox_annotation() { + let content = "# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_python_sandbox_annotation_with_other_annotations() { + let content = "# no_cache\n# sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.no_cache); + } + + #[test] + fn test_python_no_sandbox_annotation() { + let content = "# no_cache\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation() { + let content = "// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_annotation_with_other_annotations() { + let content = "// npm\n// sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + assert!(annotations.npm); + } + + #[test] + fn test_typescript_no_sandbox_annotation() { + let content = "// npm\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(!annotations.sandbox); + } + + #[test] + fn test_python_sandbox_no_space() { + let content = "#sandbox\ndef main():\n pass"; + let annotations = PythonAnnotations::parse(content); + assert!(annotations.sandbox); + } + + #[test] + fn test_typescript_sandbox_no_space() { + let content = "//sandbox\nexport function main() {}"; + let annotations = TypeScriptAnnotations::parse(content); + assert!(annotations.sandbox); + } } diff --git a/backend/windmill-native-triggers/src/lib.rs b/backend/windmill-native-triggers/src/lib.rs index 4f27dff2e2..ed22480a9a 100644 --- a/backend/windmill-native-triggers/src/lib.rs +++ b/backend/windmill-native-triggers/src/lib.rs @@ -726,7 +726,7 @@ pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>> ) -> Result> { let token = sqlx::query_scalar!( r#" - SELECT token + SELECT token as "token!" FROM token WHERE token LIKE concat($1::text, '%') LIMIT 1 diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 68da452335..eff7475785 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -830,6 +830,15 @@ pub async fn run_preview_relative_imports( #[cfg(all(feature = "private", feature = "agent_worker_server"))] pub async fn testing_http_connection(port: u16) -> Connection { + testing_http_connection_with_tags( + port, + vec!["flow".into(), "python3".into(), "dependency".into()], + ) + .await +} + +#[cfg(all(feature = "private", feature = "agent_worker_server"))] +pub async fn testing_http_connection_with_tags(port: u16, tags: Vec) -> Connection { let suffix = windmill_common::utils::create_default_worker_suffix("test-agent-worker"); let agent_token = format!( "{}{}", @@ -837,7 +846,7 @@ pub async fn testing_http_connection(port: u16) -> Connection { windmill_common::jwt::encode_with_internal_secret(windmill_api_agent_workers::AgentAuth { worker_group: "testing-agent".to_owned(), suffix: Some(suffix.clone()), - tags: vec!["flow".into(), "python3".into(), "dependency".into()], + tags, exp: Some(usize::MAX), }) .await diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index cf86d83801..be20cfae3e 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -13,6 +13,7 @@ pub enum AssetKind { Variable, // Deprecated Ducklake, DataTable, + Volume, } #[derive( diff --git a/backend/windmill-worker-volumes/Cargo.toml b/backend/windmill-worker-volumes/Cargo.toml new file mode 100644 index 0000000000..08b9a554b2 --- /dev/null +++ b/backend/windmill-worker-volumes/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "windmill-worker-volumes" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_worker_volumes" +path = "src/lib.rs" + +[features] +enterprise = [] +private = [] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +object_store.workspace = true +tokio.workspace = true +tracing.workspace = true +bytes.workspace = true +futures.workspace = true +serde.workspace = true +serde_json.workspace = true +regex.workspace = true +lazy_static.workspace = true +md-5.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs new file mode 100644 index 0000000000..a9003a3712 --- /dev/null +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -0,0 +1,544 @@ +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; +pub use volume_oss::*; + +pub use object_store::ObjectStore as DynObjectStore; + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; + +pub const MAX_VOLUMES_PER_JOB: usize = 10; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FileEntry { + pub size: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub md5: Option, +} + +pub fn compute_md5_hex(data: &[u8]) -> String { + use md5::{Digest, Md5}; + let result = Md5::digest(data); + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hex = String::with_capacity(32); + for &b in result.iter() { + hex.push(HEX[(b >> 4) as usize] as char); + hex.push(HEX[(b & 0x0f) as usize] as char); + } + hex +} + +/// Extract an MD5 hash from an S3 ETag, if it's a simple (non-multipart) ETag. +pub fn etag_to_md5(e_tag: Option<&str>) -> Option { + let tag = e_tag?.trim_matches('"'); + // Multipart ETags contain a '-' (e.g. "abc123-5"), skip those + if tag.contains('-') || tag.is_empty() { + return None; + } + Some(tag.to_string()) +} + +lazy_static::lazy_static! { + static ref ARGS_INTERPOLATION_RE: regex::Regex = + regex::Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap(); + static ref VALID_VOLUME_NAME_RE: regex::Regex = + regex::Regex::new(r"^[a-zA-Z0-9][a-zA-Z0-9._-]{0,253}[a-zA-Z0-9]$").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct VolumeMount { + pub name: String, + pub target: String, +} + +pub struct VolumeState { + pub mount: VolumeMount, + pub local_dir: PathBuf, + pub manifest: HashMap, + pub symlinks: HashMap, +} + +pub struct DownloadStats { + pub total_files: usize, + pub from_cache: usize, + pub downloaded: usize, +} + +pub struct SyncStats { + pub new_size_bytes: u64, + pub file_count: usize, + pub uploaded: usize, + pub skipped: usize, +} + +pub fn validate_volume_name(name: &str) -> Result<(), String> { + if name.contains("..") { + return Err(format!( + "Volume name '{}' contains '..' which is not allowed", + name + )); + } + if !VALID_VOLUME_NAME_RE.is_match(name) { + return Err(format!( + "Volume name '{}' is invalid. Names must be 2-255 characters, \ + start and end with alphanumeric, and contain only alphanumeric, '.', '_', or '-'", + name + )); + } + Ok(()) +} + +const ALLOWED_ABSOLUTE_PREFIXES: &[&str] = &["/tmp/", "/mnt/", "/opt/", "/home/", "/data/"]; + +pub fn validate_volume_target(target: &str) -> Result<(), String> { + if target.split('/').any(|seg| seg == "..") { + return Err(format!( + "Volume target '{target}' contains '..' segments which is not allowed" + )); + } + if target.starts_with('/') + && !ALLOWED_ABSOLUTE_PREFIXES + .iter() + .any(|p| target.starts_with(p)) + { + return Err(format!( + "Volume target '{target}' must be a relative path or start with one of: {}", + ALLOWED_ABSOLUTE_PREFIXES.join(", ") + )); + } + Ok(()) +} + +pub fn validate_volume_mounts(mounts: &[VolumeMount]) -> Result<(), String> { + if mounts.len() > MAX_VOLUMES_PER_JOB { + return Err(format!( + "Too many volume mounts ({}, max {})", + mounts.len(), + MAX_VOLUMES_PER_JOB + )); + } + let mut seen_names = HashSet::new(); + let mut seen_targets = HashSet::new(); + for v in mounts { + if !seen_names.insert(&v.name) { + return Err(format!("Duplicate volume name: '{}'", v.name)); + } + if !seen_targets.insert(&v.target) { + return Err(format!("Duplicate volume target: '{}'", v.target)); + } + } + Ok(()) +} + +pub fn interpolate_volume_name( + name: &str, + args: Option<&HashMap>>, + workspace_id: &str, +) -> String { + let name = name.replace("$workspace", workspace_id); + if !name.contains("$args[") { + return name; + } + let Some(args) = args else { + return name; + }; + let mut result = name.clone(); + for cap in ARGS_INTERPOLATION_RE.captures_iter(&name) { + let full_match = cap.get(0).unwrap().as_str(); + let arg_name = cap.get(1).unwrap().as_str(); + let arg_value = if arg_name.contains('.') { + let parts: Vec<&str> = arg_name.split('.').collect(); + let root = parts[0]; + let mut value = args + .get(root) + .map(|x| x.get().to_string()) + .unwrap_or_default(); + for part in parts.iter().skip(1) { + if let Ok(obj) = serde_json::from_str::(&value) { + value = obj + .get(part) + .map(|v| v.to_string()) + .unwrap_or_default() + .to_string(); + } else { + value = String::new(); + break; + } + } + value.trim_matches('"').to_string() + } else { + args.get(arg_name) + .map(|x| x.get().trim_matches('"').to_string()) + .unwrap_or_default() + }; + result = result.replace(full_match, &arg_value); + } + result +} + +pub fn parse_volume_annotations(content: &str, comment_prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + if !trimmed.starts_with(comment_prefix) { + break; + } + let after_prefix = trimmed[comment_prefix.len()..].trim(); + if let Some(rest) = after_prefix.strip_prefix("volume:") { + let rest = rest.trim(); + let mut parts = rest.splitn(2, char::is_whitespace); + if let (Some(name), Some(target)) = (parts.next(), parts.next()) { + let name = name.trim(); + let target = target.trim(); + if !name.is_empty() && !target.is_empty() { + volumes + .push(VolumeMount { name: name.to_string(), target: target.to_string() }); + } + } + } + } + volumes +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_python_single_volume() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_typescript_single_volume() { + let content = "// volume: mydata /tmp/data\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_multiple_volumes() { + let content = "# volume: data1 /tmp/data1\n# volume: data2 /tmp/data2\n# volume: models /opt/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + VolumeMount { name: "models".to_string(), target: "/opt/models".to_string() }, + ] + ); + } + + #[test] + fn parse_mixed_annotations_and_volumes() { + let content = "# sandbox\n# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_no_volumes() { + let content = "# sandbox\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_empty_content() { + let result = parse_volume_annotations("", "#"); + assert!(result.is_empty()); + } + + #[test] + fn parse_stops_at_non_comment_line() { + let content = + "# volume: data1 /tmp/data1\ndef main():\n # volume: data2 /tmp/data2\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }] + ); + } + + #[test] + fn parse_skips_blank_lines_in_header() { + let content = + "# volume: data1 /tmp/data1\n\n# volume: data2 /tmp/data2\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![ + VolumeMount { name: "data1".to_string(), target: "/tmp/data1".to_string() }, + VolumeMount { name: "data2".to_string(), target: "/tmp/data2".to_string() }, + ] + ); + } + + #[test] + fn parse_ignores_malformed_volume_lines() { + let content = + "# volume:\n# volume: onlyname\n# volume: good /tmp/good\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "good".to_string(), target: "/tmp/good".to_string() }] + ); + } + + #[test] + fn parse_extra_whitespace() { + let content = "# volume: mydata /tmp/data \ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "mydata".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn parse_target_with_spaces_in_path() { + let content = "// volume: mydata /tmp/my data dir\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { + name: "mydata".to_string(), + target: "/tmp/my data dir".to_string(), + }] + ); + } + + #[test] + fn parse_volume_with_dashes_and_underscores() { + let content = "# volume: my-data_v2 /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "my-data_v2".to_string(), target: "/tmp/data".to_string() }] + ); + } + + #[test] + fn interpolate_workspace() { + let name = "$workspace-data"; + let result = interpolate_volume_name(name, None, "my_ws"); + assert_eq!(result, "my_ws-data"); + } + + #[test] + fn interpolate_args_simple() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"prod\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("data-$args[env]", Some(&args), "ws"); + assert_eq!(result, "data-prod"); + } + + #[test] + fn interpolate_args_and_workspace() { + let mut args = HashMap::new(); + args.insert( + "env".to_string(), + serde_json::value::RawValue::from_string("\"staging\"".to_string()).unwrap(), + ); + let result = interpolate_volume_name("$workspace-$args[env]-cache", Some(&args), "acme"); + assert_eq!(result, "acme-staging-cache"); + } + + #[test] + fn interpolate_no_placeholders() { + let result = interpolate_volume_name("plain-name", None, "ws"); + assert_eq!(result, "plain-name"); + } + + #[test] + fn interpolate_missing_arg() { + let args = HashMap::new(); + let result = interpolate_volume_name("data-$args[missing]", Some(&args), "ws"); + assert_eq!(result, "data-"); + } + + #[test] + fn interpolate_nested_arg() { + let mut args = HashMap::new(); + args.insert( + "config".to_string(), + serde_json::value::RawValue::from_string( + r#"{"env": "prod", "region": "us-east"}"#.to_string(), + ) + .unwrap(), + ); + let result = interpolate_volume_name( + "data-$args[config.env]-$args[config.region]", + Some(&args), + "ws", + ); + assert_eq!(result, "data-prod-us-east"); + } + + #[test] + fn parse_wrong_prefix_returns_empty() { + let content = "# volume: mydata /tmp/data\ndef main():\n pass"; + let result = parse_volume_annotations(content, "//"); + assert!(result.is_empty()); + } + + #[test] + fn parse_relative_path() { + let content = "// volume: agent-memory .claude\nexport function main() {}"; + let result = parse_volume_annotations(content, "//"); + assert_eq!( + result, + vec![VolumeMount { name: "agent-memory".to_string(), target: ".claude".to_string() }] + ); + } + + #[test] + fn parse_relative_nested_path() { + let content = "# volume: data data/models\ndef main():\n pass"; + let result = parse_volume_annotations(content, "#"); + assert_eq!( + result, + vec![VolumeMount { name: "data".to_string(), target: "data/models".to_string() }] + ); + } + + #[test] + fn validate_valid_names() { + assert!(validate_volume_name("mydata").is_ok()); + assert!(validate_volume_name("my-data_v2").is_ok()); + assert!(validate_volume_name("acme-staging-cache").is_ok()); + assert!(validate_volume_name("a1").is_ok()); + assert!(validate_volume_name("data.v2").is_ok()); + assert!(validate_volume_name("A0").is_ok()); + } + + #[test] + fn validate_rejects_path_traversal() { + assert!(validate_volume_name("../other-workspace").is_err()); + assert!(validate_volume_name("data/../secrets").is_err()); + assert!(validate_volume_name("a..b").is_err()); + } + + #[test] + fn validate_rejects_special_start_end() { + assert!(validate_volume_name("-data").is_err()); + assert!(validate_volume_name("data-").is_err()); + assert!(validate_volume_name(".data").is_err()); + assert!(validate_volume_name("data.").is_err()); + assert!(validate_volume_name("_data").is_err()); + } + + #[test] + fn validate_rejects_path_separators() { + assert!(validate_volume_name("data/secrets").is_err()); + assert!(validate_volume_name("data\\secrets").is_err()); + } + + #[test] + fn validate_rejects_too_short() { + assert!(validate_volume_name("").is_err()); + assert!(validate_volume_name("a").is_err()); + } + + #[test] + fn validate_rejects_too_long() { + let long_name = format!("a{}a", "b".repeat(254)); + assert!(validate_volume_name(&long_name).is_err()); + } + + #[test] + fn validate_rejects_spaces_and_special() { + assert!(validate_volume_name("my data").is_err()); + assert!(validate_volume_name("my@data").is_err()); + assert!(validate_volume_name("my$data").is_err()); + } + + #[test] + fn validate_target_allows_relative() { + assert!(validate_volume_target("data").is_ok()); + assert!(validate_volume_target("data/models").is_ok()); + assert!(validate_volume_target(".claude").is_ok()); + } + + #[test] + fn validate_target_allows_safe_absolute() { + assert!(validate_volume_target("/tmp/data").is_ok()); + assert!(validate_volume_target("/mnt/data").is_ok()); + assert!(validate_volume_target("/opt/models").is_ok()); + assert!(validate_volume_target("/home/user/data").is_ok()); + assert!(validate_volume_target("/data/cache").is_ok()); + } + + #[test] + fn validate_target_rejects_dangerous_absolute() { + assert!(validate_volume_target("/etc/passwd").is_err()); + assert!(validate_volume_target("/proc/self").is_err()); + assert!(validate_volume_target("/sys/fs").is_err()); + assert!(validate_volume_target("/dev/null").is_err()); + assert!(validate_volume_target("/usr/bin").is_err()); + assert!(validate_volume_target("/var/log").is_err()); + } + + #[test] + fn validate_target_rejects_traversal() { + assert!(validate_volume_target("../../etc").is_err()); + assert!(validate_volume_target("data/../../../etc").is_err()); + assert!(validate_volume_target("/tmp/../etc/passwd").is_err()); + } + + #[test] + fn validate_mounts_rejects_too_many() { + let mounts: Vec = (0..11) + .map(|i| VolumeMount { name: format!("v{:02}", i), target: format!("t{}", i) }) + .collect(); + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_name() { + let mounts = vec![ + VolumeMount { name: "data".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "data".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_rejects_duplicate_target() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/data".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/data".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_err()); + } + + #[test] + fn validate_mounts_ok() { + let mounts = vec![ + VolumeMount { name: "v1".to_string(), target: "/tmp/a".to_string() }, + VolumeMount { name: "v2".to_string(), target: "/tmp/b".to_string() }, + ]; + assert!(validate_volume_mounts(&mounts).is_ok()); + } +} diff --git a/backend/windmill-worker-volumes/src/volume_oss.rs b/backend/windmill-worker-volumes/src/volume_oss.rs new file mode 100644 index 0000000000..07cd6906d2 --- /dev/null +++ b/backend/windmill-worker-volumes/src/volume_oss.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "private")] +pub use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +use crate::{DownloadStats, SyncStats, VolumeMount, VolumeState}; +#[cfg(not(feature = "private"))] +use object_store::ObjectStore; +#[cfg(not(feature = "private"))] +use std::path::Path; +#[cfg(not(feature = "private"))] +use std::sync::Arc; +#[cfg(not(feature = "private"))] +use windmill_common::error; + +#[cfg(not(feature = "private"))] +pub async fn download_volume( + _client: Arc, + _volume: &VolumeMount, + _job_dir: &str, + _workspace_id: &str, +) -> error::Result<(VolumeState, DownloadStats)> { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn volume_nsjail_mount(_local_dir: &Path, _target: &str) -> String { + String::new() +} + +#[cfg(not(feature = "private"))] +pub async fn sync_volume_back( + _client: Arc, + _state: &VolumeState, + _workspace_id: &str, +) -> error::Result { + Err(error::Error::internal_err( + "Volumes are not available in this build".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +pub fn walk_dir(dir: &Path) -> std::io::Result> { + let mut result = Vec::new(); + walk_dir_inner(dir, &mut result)?; + Ok(result) +} + +#[cfg(not(feature = "private"))] +fn walk_dir_inner(dir: &Path, result: &mut Vec) -> std::io::Result<()> { + if !dir.is_dir() { + return Ok(()); + } + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.is_dir() { + walk_dir_inner(&path, result)?; + } else if meta.is_file() { + result.push(path); + } + } + Ok(()) +} + +#[cfg(not(feature = "private"))] +pub fn collect_symlinks(dir: &Path) -> std::collections::HashMap { + let mut symlinks = std::collections::HashMap::new(); + collect_symlinks_inner(dir, dir, &mut symlinks); + symlinks +} + +#[cfg(not(feature = "private"))] +fn collect_symlinks_inner( + base: &Path, + dir: &Path, + symlinks: &mut std::collections::HashMap, +) { + let entries = match std::fs::read_dir(dir) { + Ok(e) => e, + Err(_) => return, + }; + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let path = entry.path(); + let meta = match std::fs::symlink_metadata(&path) { + Ok(m) => m, + Err(_) => continue, + }; + if meta.file_type().is_symlink() { + if let Ok(target) = std::fs::read_link(&path) { + let relative = path + .strip_prefix(base) + .unwrap_or(&path) + .to_string_lossy() + .replace('\\', "/"); + symlinks.insert(relative, target.to_string_lossy().to_string()); + } + } else if meta.is_dir() { + collect_symlinks_inner(base, &path, symlinks); + } + } +} + +#[cfg(not(feature = "private"))] +pub fn restore_symlinks(_dir: &Path, _symlinks: &std::collections::HashMap) { + // No-op in OSS build +} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5cf0a4c26e..23927753ee 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -10,10 +10,10 @@ path = "src/lib.rs" [features] default = [] -private = [] +private = ["windmill-worker-volumes/private", "windmill-queue/private"] mcp = ["dep:windmill-mcp"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] -enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] +enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker-volumes/enterprise", "dep:pem", "dep:tokio-util", "dep:opentelemetry-proto", "dep:prost", "dep:hudsucker", "dep:rcgen", "dep:hyper-http-proxy", "dep:hyper-tls", "dep:hyper-util"] mssql = ["dep:tiberius"] mssql-kerberos = ["mssql", "tiberius/integrated-auth-gssapi"] # Linux/Unix integrated auth mssql-winauth = ["mssql", "tiberius/winauth"] # Windows integrated auth @@ -47,6 +47,7 @@ windmill-audit.workspace = true # there isn't really a reason for audit-worth ac windmill-common = { workspace = true, default-features = false } windmill-types.workspace = true windmill-object-store.workspace = true +windmill-worker-volumes.workspace = true windmill-jseval.workspace = true windmill-runtime-nativets = { workspace = true, optional = true } windmill-mcp = { workspace = true, optional = true } diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 8cbee3dec9..3ba8c73257 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -14,6 +14,18 @@ clone_newnet: false clone_newuser: {CLONE_NEWUSER} clone_newcgroup: false +uidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + +gidmap { + inside_id: "1000" + outside_id: "" + count: 1 +} + skip_setsid: true keep_caps: false keep_env: true diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 40f4b9f77e..7939dc77e1 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -22,7 +22,7 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, - NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + NPMRC, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ @@ -990,6 +990,14 @@ pub async fn handle_bun_job( ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); + if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(error::Error::ExecutionErr( + "Script has //sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the //sandbox annotation." + .to_string(), + )); + } + let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if let (Some(lock), true) = ( maybe_lock.get_lock(), !annotation.nobundling && !*DISABLE_BUNDLING && codebase.is_none(), @@ -1161,6 +1169,10 @@ pub async fn handle_bun_job( init_logs = format!("\n{}{}", cache_logs, init_logs); } + if annotation.sandbox { + init_logs.push_str("sandbox mode (nsjail)\n"); + } + let write_wrapper_f = async { if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; @@ -1485,7 +1497,7 @@ try {{ append_logs(&job.id, &job.workspace_id, init_logs, conn).await; //do not cache local dependencies - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotation.sandbox { let _ = write_file( job_dir, "run.config.proto", diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1239aa8d7d..846c421024 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -886,7 +886,7 @@ pub async fn cached_result_path( } #[cfg(feature = "parquet")] -async fn get_workspace_s3_resource_path( +pub(crate) async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, @@ -948,7 +948,11 @@ async fn get_workspace_s3_resource_path( ) } Some(LargeFileStorage::FilesystemStorage(fs)) => { - (StorageResourceType::Filesystem, fs.root_path.clone()) + return Ok(Some( + windmill_object_store::ObjectStoreResource::Filesystem( + windmill_object_store::FilesystemSettings { root_path: fs.root_path.clone() }, + ), + )); } None => { return Ok(None); diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 19831eadd0..6cbf85c47b 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -17,6 +17,7 @@ use crate::{ NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use windmill_common::client::AuthedClient; +use windmill_common::worker::TypeScriptAnnotations; use tokio::{fs::File, io::AsyncReadExt, process::Command}; use windmill_common::{error::Result, scripts::ScriptLang, worker::write_file, BASE_URL}; @@ -231,8 +232,13 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + let annotations = TypeScriptAnnotations::parse(inner_content); + // let mut start = Instant::now(); - let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + let mut logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); + if annotations.sandbox { + logs1.push_str("sandbox mode (nsjail)\n"); + } append_logs(&job.id, &job.workspace_id, logs1, conn).await; let main_override = job.script_entrypoint_override.as_deref(); @@ -451,7 +457,7 @@ try {{ for flag in deno_flags { args.push(flag); } - } else if is_sandboxing_enabled() { + } else if is_sandboxing_enabled() || annotations.sandbox { args.push("--allow-net"); args.push("--allow-sys"); args.push(allow_read.as_str()); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 741458c0e0..f6d752558d 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -70,6 +70,9 @@ mod sanitized_sql_params; mod schema; pub mod sql_utils; mod universal_pkg_installer; +#[cfg(feature = "private")] +mod volume_ee; +mod volume_oss; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b289f05380..619087659d 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -134,8 +134,8 @@ use crate::{ handle_child::handle_child, is_sandboxing_enabled, read_ee_registry, worker_utils::ping_job_status, - PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, - PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, + PyV, DISABLE_NUSER, HOME_ENV, NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PIP_EXTRA_INDEX_URL, + PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, UV_CACHE_DIR, UV_INDEX_STRATEGY, }; use windmill_common::client::AuthedClient; @@ -567,6 +567,14 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { + return Err(Error::ExecutionErr( + "Script has #sandbox annotation but nsjail is not available on this worker. \ + Please ensure nsjail is installed or remove the #sandbox annotation." + .to_string(), + )); + } + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, @@ -605,16 +613,14 @@ pub async fn handle_python_job( } { - append_logs( - &job.id, - &job.workspace_id, - format!( - "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.clone().to_string() - ), - conn, - ) - .await; + let mut logs = format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.clone().to_string() + ); + if annotations.sandbox { + logs.push_str("sandbox mode (nsjail)\n"); + } + append_logs(&job.id, &job.workspace_id, logs, conn).await; } let ( import_loader, @@ -784,7 +790,7 @@ except BaseException as e: #[cfg(windows)] let additional_python_paths_folders = additional_python_paths_folders.replace(":", ";"); - if is_sandboxing_enabled() { + if is_sandboxing_enabled() || annotations.sandbox { let shared_deps = additional_python_paths .into_iter() .map(|pp| { @@ -828,7 +834,7 @@ mount {{ job.id ); - let child = if is_sandboxing_enabled() { + let child = if is_sandboxing_enabled() || annotations.sandbox { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd .current_dir(job_dir) diff --git a/backend/windmill-worker/src/volume_oss.rs b/backend/windmill-worker/src/volume_oss.rs new file mode 100644 index 0000000000..3736e83485 --- /dev/null +++ b/backend/windmill-worker/src/volume_oss.rs @@ -0,0 +1,112 @@ +#[cfg(feature = "private")] +pub(crate) use crate::volume_ee::*; + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct LeaseRenewalGuard(pub Option>); + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +impl Drop for LeaseRenewalGuard { + fn drop(&mut self) { + if let Some(handle) = self.0.take() { + handle.abort(); + } + } +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) struct VolumeSetupResult { + pub states: Vec, + pub writable: Vec, + pub client: Option>, + pub lease_renewal: LeaseRenewalGuard, +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +#[allow(dead_code)] +pub(crate) fn setup_volume_mount_paths( + _volume: &windmill_worker_volumes::VolumeMount, + _state: &windmill_worker_volumes::VolumeState, + _job_dir: &str, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result<()> { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_sql_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _worker_name: &str, + _job_dir: &str, + _client: &windmill_common::client::AuthedClient, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn setup_volumes_http_worker( + _volume_mounts: &[windmill_worker_volumes::VolumeMount], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _permissioned_as: &str, + _canceled_by: &Option, + _worker_name: &str, + _job_dir: &str, + _conn: &windmill_common::worker::Connection, + _language: windmill_common::scripts::ScriptLang, + _envs: &mut std::collections::HashMap, + _shared_mount: &mut String, +) -> windmill_common::error::Result { + Err(windmill_common::error::Error::internal_err( + "Volumes are not available in OSS".to_string(), + )) +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_sql_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _vol_client: &std::sync::Arc, + _db: &windmill_common::DB, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} + +#[cfg(not(feature = "private"))] +#[cfg(feature = "parquet")] +pub(crate) async fn sync_volumes_http_worker( + _volume_states: &[windmill_worker_volumes::VolumeState], + _volume_writable: &[bool], + _http: &windmill_common::worker::HttpClient, + _workspace_id: &str, + _job_id: uuid::Uuid, + _worker_name: &str, + _conn: &windmill_common::worker::Connection, + _job_succeeded: bool, +) { +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1497d3ebb8..350af4dab6 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4161,7 +4161,8 @@ pub async fn run_language_executor( job.id ); - let shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { + #[allow(unused_mut)] + let mut shared_mount = if job.same_worker && job.script_lang != Some(ScriptLang::Deno) { let folder = if job.script_lang == Some(ScriptLang::Go) { "/go" } else { @@ -4183,7 +4184,8 @@ mount {{ // println!("handle lang job {:?}", SystemTime::now()); - let envs = build_envs(envs.as_ref())?; + #[allow(unused_mut)] + let mut envs = build_envs(envs.as_ref())?; let Some(language) = language else { return Err(Error::ExecutionErr( @@ -4219,6 +4221,106 @@ mount {{ } } + // Volume mount setup (requires workspace S3 storage; CE has file count/size limits) + #[cfg(feature = "parquet")] + let volume_mounts = { + let comment_prefix = match language { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby => "#", + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => "//", + _ => "", + }; + let raw_mounts = windmill_worker_volumes::parse_volume_annotations(&code, comment_prefix); + let args_ref = job.args.as_ref().map(|a| &**a); + let mut interpolated = Vec::new(); + for mut v in raw_mounts { + v.name = windmill_worker_volumes::interpolate_volume_name( + &v.name, + args_ref, + &job.workspace_id, + ); + if let Err(e) = windmill_worker_volumes::validate_volume_name(&v.name) { + return Err(Error::ExecutionErr(e)); + } + if let Err(e) = windmill_worker_volumes::validate_volume_target(&v.target) { + return Err(Error::ExecutionErr(e)); + } + interpolated.push(v); + } + if let Err(e) = windmill_worker_volumes::validate_volume_mounts(&interpolated) { + return Err(Error::ExecutionErr(e)); + } + interpolated + }; + + #[cfg(feature = "parquet")] + let mut volume_setup = crate::volume_oss::VolumeSetupResult { + states: Vec::new(), + writable: Vec::new(), + client: None, + lease_renewal: crate::volume_oss::LeaseRenewalGuard(None), + }; + + #[cfg(feature = "parquet")] + if !volume_mounts.is_empty() { + let vol_summary: Vec = volume_mounts + .iter() + .map(|v| format!("'{}' -> {}", v.name, v.target)) + .collect(); + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n--- VOLUME MOUNTS ---\nPulling {} volume(s): {}\n", + volume_mounts.len(), + vol_summary.join(", "), + ), + conn, + ) + .await; + + if let Connection::Sql(db) = conn { + volume_setup = crate::volume_oss::setup_volumes_sql_worker( + &volume_mounts, + db, + &job.workspace_id, + job.id, + &job.permissioned_as, + worker_name, + job_dir, + client, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } else if let Connection::Http(http) = conn { + volume_setup = crate::volume_oss::setup_volumes_http_worker( + &volume_mounts, + http, + &job.workspace_id, + job.id, + &job.permissioned_as, + &job.canceled_by, + worker_name, + job_dir, + conn, + language, + &mut envs, + &mut shared_mount, + ) + .await?; + } + } + // Box::pin all language handlers to prevent large match enum on stack let result: error::Result> = match language { ScriptLang::Python3 => { @@ -4630,6 +4732,61 @@ mount {{ // for related places search: ADD_NEW_LANG _ => panic!("unreachable, language is not supported: {language:#?}"), }; + // Volume sync-back and lease release + #[cfg(feature = "parquet")] + if !volume_setup.states.is_empty() { + // Stop lease renewal before sync-back + volume_setup.lease_renewal.0.take().map(|h| h.abort()); + + if let Some(ref vol_client) = volume_setup.client { + if let Connection::Sql(db) = conn { + crate::volume_oss::sync_volumes_sql_worker( + &volume_setup.states, + &volume_setup.writable, + vol_client, + db, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + } + + if let Connection::Http(http) = conn { + crate::volume_oss::sync_volumes_http_worker( + &volume_setup.states, + &volume_setup.writable, + http, + &job.workspace_id, + job.id, + worker_name, + conn, + result.is_ok(), + ) + .await; + } + + // Clean up absolute-path symlinks created by setup_volume_mount_paths + if !is_sandboxing_enabled() { + for state in &volume_setup.states { + #[cfg(unix)] + if state.mount.target.starts_with('/') { + let target_path = std::path::Path::new(&state.mount.target); + if target_path + .symlink_metadata() + .map(|m| m.file_type().is_symlink()) + .unwrap_or(false) + { + std::fs::remove_file(target_path).ok(); + } + } + } + } + } + tracing::info!( workspace_id = %job.workspace_id, is_ok = result.is_ok(), diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 450dd68399..279c79b780 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7cc4dafa05..88c16aaac0 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -61,6 +61,11 @@ RUN ln -s /usr/bin/bun /usr/bin/node \ && bun install -g windmill-cli \ && ln -s $(bun pm bin -g)/wmill /usr/bin/wmill +# Install Claude Code CLI (used by claude sandbox scripts) +# Copy to /usr/bin/claude so it's accessible inside nsjail sandbox (which mounts /usr but not /root) +RUN curl -fsSL https://claude.ai/install.sh | bash \ + && cp /root/.local/share/claude/versions/* /usr/bin/claude + # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index d8429a5ebf..8a79bd7109 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -15,6 +15,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import BedrockCredentialsCheck from './BedrockCredentialsCheck.svelte' import { isCloudHosted } from '$lib/cloud' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { resourceType: string @@ -149,6 +150,12 @@ }} class="as-json-toggle" /> + {#if resourceType == 'postgresql'} - {#if asset.kind === 's3object'} + {#if asset.kind === 's3object' || asset.kind === 'volume'} Explore {:else if asset.kind === 'resource' || asset.kind === 'ducklake' || asset.kind === 'datatable'} Manage diff --git a/frontend/src/lib/components/FilesetEditor.svelte b/frontend/src/lib/components/FilesetEditor.svelte index 41559d0d8a..39a67f4ad6 100644 --- a/frontend/src/lib/components/FilesetEditor.svelte +++ b/frontend/src/lib/components/FilesetEditor.svelte @@ -46,6 +46,9 @@ } } + // Track the last args we wrote so we can detect external changes. + let lastWrittenArgs: Record = $state(args ?? {}) + // Sync files → args, overlaying current editContent for the active file. // This avoids spreading a new files object on every keystroke. $effect(() => { @@ -58,9 +61,33 @@ newArgs[argKey] = key === currentKey ? currentContent : value } } + lastWrittenArgs = newArgs args = newArgs }) + // Sync args → files when args changes externally (e.g. from AI generation). + $effect(() => { + const currentArgs = args + if (currentArgs === lastWrittenArgs) return + // Check if the args object is actually different + const currentKeys = Object.keys(currentArgs ?? {}).sort().join('\0') + const lastKeys = Object.keys(lastWrittenArgs ?? {}).sort().join('\0') + if (currentKeys === lastKeys) { + const allSame = Object.entries(currentArgs ?? {}).every( + ([k, v]) => lastWrittenArgs[k] === v + ) + if (allSame) return + } + const newFiles = Object.fromEntries( + Object.entries(currentArgs ?? {}).map(([k, v]) => ['/' + k, String(v ?? '')]) + ) + files = newFiles + lastWrittenArgs = currentArgs + const firstFile = Object.keys(newFiles).find((k) => !k.endsWith('/')) + selectedPath = firstFile ?? '/' + editContent = firstFile ? (newFiles[firstFile] ?? '') : '' + }) + function inferLang(filePath: string): string { const ext = filePath.split('.').pop()?.toLowerCase() if (!ext) return 'plaintext' diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 5a5943390e..0ffed949d1 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -24,6 +24,7 @@ import GitHubAppIntegration from './GitHubAppIntegration.svelte' import Button from './common/button/Button.svelte' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' + import ResourceGen from './copilot/ResourceGen.svelte' interface Props { canSave?: boolean @@ -270,6 +271,13 @@ right: 'As JSON' }} /> + {#if resourceToEdit?.resource_type === 'nats' || resourceToEdit?.resource_type === 'kafka'} {:else} @@ -296,9 +304,17 @@ {#if loadingSchema} {:else if !viewJsonSchema && resourceTypeInfo?.is_fileset} -
- Fileset -
+
+
Fileset
+ +
{:else if !viewJsonSchema && resourceSchema && resourceSchema?.properties} {#if resourceTypeInfo?.format_extension} diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index a07ea85d25..55bb25c5ef 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -13,6 +13,7 @@ interface Props { fromWorkspaceSettings?: boolean readOnlyMode: boolean + allowDelete?: boolean initialFileKey?: { s3: string; storage?: string } | undefined selectedFileKey?: { s3: string; storage?: string } | undefined folderOnly?: boolean @@ -24,6 +25,7 @@ let { fromWorkspaceSettings = false, readOnlyMode, + allowDelete = false, initialFileKey = $bindable(undefined), selectedFileKey = $bindable(undefined), folderOnly = false, @@ -94,6 +96,7 @@ }} {fromWorkspaceSettings} {readOnlyMode} + {allowDelete} bind:initialFileKey bind:selectedFileKey bind:workspaceSettingsInitialized diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 380e3d9a35..74b85bd327 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -81,6 +81,7 @@ count: number } > + allowDelete?: boolean replaceUnauthorizedWarning?: Snippet listStoredFilesRequest?: (d: ListStoredFilesData) => CancelablePromise loadFilePreviewRequest?: (d: LoadFilePreviewData) => CancelablePromise @@ -102,11 +103,12 @@ folderOnly = false, regexFilter = undefined, hideS3SpecificDetails = false, - rootPath = '', + rootPath: initialRootPath = '', workspaceSettingsInitialized = $bindable(true), storage = $bindable(undefined), uploadModalOpen = $bindable(false), allFilesByKey = $bindable({}), + allowDelete = false, replaceUnauthorizedWarning, listStoredFilesRequest = HelpersService.listStoredFiles, loadFilePreviewRequest = HelpersService.loadFilePreview, @@ -116,6 +118,7 @@ testConnectionRequest = HelpersService.datasetStorageTestConnection }: Props = $props() + let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) let csvSeparatorChar: string = $state(',') @@ -263,7 +266,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() fileListLoading = false fileInfoLoading = false } @@ -381,7 +384,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } async function clearAndLoadFiles({ keepFilter }: { keepFilter?: boolean } = {}) { @@ -424,9 +427,16 @@ export async function open(_preSelectedFileKey: S3Object | undefined = undefined) { const preSelectedFileKey = _preSelectedFileKey && parseS3Object(_preSelectedFileKey) storage = preSelectedFileKey?.storage - if (preSelectedFileKey !== undefined) { + if (preSelectedFileKey !== undefined && preSelectedFileKey.s3.endsWith('/')) { + rootPath = preSelectedFileKey.s3 + filter = '' + selectedFileKey = undefined + } else if (preSelectedFileKey !== undefined) { + rootPath = '' initialFileKey = { ...preSelectedFileKey } selectedFileKey = { ...preSelectedFileKey } + } else { + rootPath = '' } reloadContent() } @@ -461,7 +471,7 @@ if (selectedFileKey !== undefined) { if (allFilesByKey[selectedFileKey.s3] === undefined) { selectedFileKey = { s3: '', storage } - } else { + } else if (allFilesByKey[selectedFileKey.s3].type !== 'folder') { loadFileMetadataPlusPreviewAsync(selectedFileKey.s3) } } @@ -518,7 +528,7 @@ } } } - displayedFileKeys = displayedFileKeys.sort() + displayedFileKeys = [...new Set(displayedFileKeys)].sort() } else { selectedFileKey = { s3: item_key, @@ -719,8 +729,10 @@ startIcon={{ icon: MoveRight }} iconOnly={true} /> + {/if} + {#if !readOnlyMode || allowDelete} +
{#if customUi?.settingsPanel?.metadata?.disableScriptKind !== true}
{#snippet header()} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 5fe03b03fd..0f1e1d8a6e 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -100,7 +100,7 @@ path: string | undefined lang: Preview['language'] kind?: string | undefined - template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' + template?: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox' tag: string | undefined initialArgs?: Record fixedOverflowWidgets?: boolean diff --git a/frontend/src/lib/components/ShareModal.svelte b/frontend/src/lib/components/ShareModal.svelte index e5ad54f88b..c8c69106ed 100644 --- a/frontend/src/lib/components/ShareModal.svelte +++ b/frontend/src/lib/components/ShareModal.svelte @@ -35,6 +35,7 @@ | 'postgres_trigger' | 'gcp_trigger' | 'email_trigger' + | 'volume' let kind: Kind let path: string = $state('') @@ -53,13 +54,17 @@ let drawer: Drawer | undefined = $state() let own = $state(false) - export async function openDrawer(newPath: string, kind_l: Kind) { + export async function openDrawer(newPath: string, kind_l: Kind, isOwnerOverride?: boolean) { path = newPath kind = kind_l loadAcls() loadGroups() loadUsernames() - loadOwner() + if (isOwnerOverride !== undefined) { + own = isOwnerOverride + } else { + loadOwner() + } drawer?.openDrawer() } diff --git a/frontend/src/lib/components/assets/JobAssetsViewer.svelte b/frontend/src/lib/components/assets/JobAssetsViewer.svelte index 86b9c7eca7..10e2f011f9 100644 --- a/frontend/src/lib/components/assets/JobAssetsViewer.svelte +++ b/frontend/src/lib/components/assets/JobAssetsViewer.svelte @@ -1,5 +1,5 @@ + + (open = false)}> + (open = false)}> + {#if loading} +
+ +
+ {:else if volume} +
+
+
+ Files + {volume.file_count} +
+
+ Size + {displaySize(volume.size_bytes) ?? '0 B'} +
+
+ Created at + {displayDate(volume.created_at)} +
+
+ Created by + {volume.created_by} +
+ {#if volume.last_used_at} +
+ Last used + {displayDate(volume.last_used_at)} +
+ {/if} +
+ +
+ {#if s3FilePicker} + + {/if} + {#if $userStore?.is_admin} + + {/if} +
+
+ {:else} +
+ Volume '{volumeName}' not found. +
+ {/if} + + diff --git a/frontend/src/lib/components/assets/VolumesDrawer.svelte b/frontend/src/lib/components/assets/VolumesDrawer.svelte new file mode 100644 index 0000000000..3d6d861b15 --- /dev/null +++ b/frontend/src/lib/components/assets/VolumesDrawer.svelte @@ -0,0 +1,193 @@ + + + (open = false)}> + (open = false)}> + {#snippet actions()} + + {#snippet trigger()} + + {/snippet} + {#snippet content({ close })} +
+ { + if (e.key === 'Enter' && newVolumeName.trim()) { + createVolume(newVolumeName.trim(), close) + } + } + }} + bind:value={newVolumeName} + /> + +
+ {/snippet} + + {/snippet} + {#if volumes.loading} +
+ +
+ {:else if !volumes.current?.length} +
+ No volumes yet. Create one above or they are auto-created when a job declares a volume annotation. +
+ {:else} +
+ {#each volumes.current as vol (vol.name)} + {@const readable = canReadVolume(vol.created_by, vol.extra_perms)} + {@const writable = canWriteVolume(vol.created_by, vol.extra_perms)} +
+ +
+
+ {vol.name} + } + canWrite={writable} + /> +
+ + {vol.file_count} {vol.file_count === 1 ? 'file' : 'files'} + · {displaySize(vol.size_bytes) ?? '0 B'} + · owner: {vol.created_by.replace(/^u\//, '')} + +
+ {#if vol.last_used_at} + + Used {displayDate(vol.last_used_at)} + + {/if} + {#if writable} + + {/if} + {#if writable} +
+ {/each} +
+ {/if} + + + + refreshKey++} /> diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 2dacedbae3..4dae4c20b1 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -26,6 +26,8 @@ export function formatAsset(asset: Asset): string { return `ducklake://${asset.path}` case 'datatable': return `datatable://${asset.path}` + case 'volume': + return `volume://${asset.path}` } return 'unknown' } @@ -89,6 +91,8 @@ export function formatAssetKind(asset: { return 'Ducklake' case 'datatable': return 'Data table' + case 'volume': + return 'Volume' } } diff --git a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte index a380a5d753..6c7a052301 100644 --- a/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte +++ b/frontend/src/lib/components/common/languageIcons/LanguageIcon.svelte @@ -24,6 +24,7 @@ import JavaIcon from '$lib/components/icons/JavaIcon.svelte' import DuckDbIcon from '$lib/components/icons/DuckDbIcon.svelte' import RubyIcon from '$lib/components/icons/RubyIcon.svelte' + import ClaudeIcon from '$lib/components/icons/ClaudeIcon.svelte' interface Props { lang: @@ -36,6 +37,7 @@ | 'docker' | 'powershell' | 'bunnative' + | 'claudesandbox' width?: number height?: number scale?: number @@ -45,7 +47,7 @@ let { lang, width = 30, height = 30, scale = 1, size = undefined, ...rest }: Props = $props() - const languageLabel: Record = { + const languageLabel: Record = { python3: 'Python', deno: 'TypeScript', go: 'Go', @@ -68,12 +70,13 @@ csharp: 'C#', nu: 'Nu', java: 'Java', - ruby: 'Ruby' + ruby: 'Ruby', + claudesandbox: 'Claude Sandbox' // for related places search: ADD_NEW_LANG } const langToComponent: Record< - SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative', + SupportedLanguage | 'pgsql' | 'javascript' | 'fetch' | 'docker' | 'powershell' | 'bunnative' | 'claudesandbox', any > = { go: GoIcon, @@ -103,7 +106,8 @@ nu: NuIcon, java: JavaIcon, ruby: RubyIcon, - duckdb: DuckDbIcon + duckdb: DuckDbIcon, + claudesandbox: TypeScriptIcon // for related places search: ADD_NEW_LANG } @@ -142,4 +146,17 @@ />
{/if} + {#if lang === 'claudesandbox'} +
+ +
+ {/if} diff --git a/frontend/src/lib/components/copilot/ResourceGen.svelte b/frontend/src/lib/components/copilot/ResourceGen.svelte new file mode 100644 index 0000000000..aa18ca0546 --- /dev/null +++ b/frontend/src/lib/components/copilot/ResourceGen.svelte @@ -0,0 +1,186 @@ + + + + {#snippet trigger()} + {/if} - + {@render children?.()} diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 5fe25dbba2..a04923e156 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -79,7 +79,7 @@ const Icon = $derived(theme[type].Icon ?? AlertTriangle) - + {#if open}
import ConfirmationModal from './ConfirmationModal.svelte' - import { createEventDispatcher } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import type { Trigger } from '$lib/components/triggers/utils' import DataTable from '$lib/components/table/DataTable.svelte' import { twMerge } from 'tailwind-merge' @@ -20,7 +20,7 @@ let { open = $bindable(false), draftTriggers = [], isFlow = false }: Props = $props() - let selectedTriggers: Trigger[] = $state(draftTriggers) + let selectedTriggers: Trigger[] = $state(untrack(() => draftTriggers)) const dispatch = createEventDispatcher<{ canceled: void diff --git a/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte index 56d965be75..20ebe04c85 100644 --- a/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte @@ -10,7 +10,7 @@ replaceFalseWithUndefined, type Value } from '$lib/utils' - import { page } from '$app/stores' + import { page } from '$app/state' import type { GetInitialAndModifiedValues } from './unsavedTypes' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' @@ -43,9 +43,9 @@ !bypassBeforeNavigate && getInitialAndModifiedValues && newNavigationState.to && - ((newNavigationState.to.url != $page.url && + ((newNavigationState.to.url != page.url && newNavigationState.to.url.pathname !== newNavigationState.from?.url.pathname) || - (triggerOnSearchParamsChange && newNavigationState.to.url.search != $page.url.search)) + (triggerOnSearchParamsChange && newNavigationState.to.url.search != page.url.search)) ) { goingTo = newNavigationState.to.url diff --git a/frontend/src/lib/components/common/drawer/Disposable.svelte b/frontend/src/lib/components/common/drawer/Disposable.svelte index 05e469cfaf..68f079dac0 100644 --- a/frontend/src/lib/components/common/drawer/Disposable.svelte +++ b/frontend/src/lib/components/common/drawer/Disposable.svelte @@ -26,7 +26,7 @@ onClose }: Props = $props() - let offset = $state(initialOffset) + let offset = $state(untrack(() => initialOffset)) let zIndex = $derived(zIndexes.disposables + offset) export function toggleDrawer() { @@ -87,8 +87,8 @@ } if (open) { - openedDrawers.val.push(id) - offset = initialOffset + openedDrawers.val.length + openedDrawers.val.push(untrack(() => id)) + offset = untrack(() => initialOffset) + openedDrawers.val.length } let wasEverOpen = false diff --git a/frontend/src/lib/components/common/fileInput/FileInput.svelte b/frontend/src/lib/components/common/fileInput/FileInput.svelte index 37a8b3801e..11f7a0caa9 100644 --- a/frontend/src/lib/components/common/fileInput/FileInput.svelte +++ b/frontend/src/lib/components/common/fileInput/FileInput.svelte @@ -8,27 +8,49 @@ type ConvertedFile = string | ArrayBuffer | null - let c = '' - export { c as class } - export let style = '' - export let accept = '*' - export let multiple = false - export let convertTo: ReadFileAs | undefined = undefined - export let hideIcon = false - export let iconSize = 24 - export let returnFileNames = false - export let submittedText: string | undefined = undefined - export let defaultFile: string | string[] | undefined = undefined - export let disabled: boolean | undefined = undefined - export let folderOnly = false - const dispatch = createEventDispatcher() - let input: HTMLInputElement + let input: HTMLInputElement | undefined = $state() type FileWithPath = File & { path?: string } - export let files: FileWithPath[] | undefined = undefined + interface Props { + class?: string + style?: string + accept?: string + multiple?: boolean + convertTo?: ReadFileAs | undefined + hideIcon?: boolean + iconSize?: number + returnFileNames?: boolean + submittedText?: string | undefined + defaultFile?: string | string[] | undefined + disabled?: boolean | undefined + folderOnly?: boolean + files?: FileWithPath[] | undefined + selectedTitle?: import('svelte').Snippet + children?: import('svelte').Snippet + [key: string]: any + } - let pointerStartX = 0 - let pointerStartY = 0 + let { + class: c = '', + style = '', + accept = '*', + multiple = false, + convertTo = undefined, + hideIcon = false, + iconSize = 24, + returnFileNames = false, + submittedText = undefined, + defaultFile = undefined, + disabled = undefined, + folderOnly = false, + files = $bindable(undefined), + selectedTitle, + children, + ...rest + }: Props = $props() + + let pointerStartX = $state(0) + let pointerStartY = $state(0) function handlePointerDown(e: PointerEvent) { pointerStartX = e.clientX @@ -50,7 +72,7 @@ // Needs to be reset so the same file can be selected // multiple times in a row - input.value = '' + if (input) input.value = '' dispatchChange() } @@ -194,10 +216,10 @@ duration-200 px-1 py-8`, c )} - on:dragover={handleDragOver} - on:drop={handleDrop} - on:pointerdown={handlePointerDown} - on:click={(e) => { + ondragover={handleDragOver} + ondrop={handleDrop} + onpointerdown={handlePointerDown} + onclick={(e) => { const deltaX = Math.abs(e.clientX - pointerStartX) const deltaY = Math.abs(e.clientY - pointerStartY) if (deltaX > 5 || deltaY > 5) { @@ -214,11 +236,11 @@ {/if} {#if files}
- + {#if selectedTitle}{@render selectedTitle()}{:else}
{submittedText ? submittedText : `Selected file${files.length > 1 ? 's' : ''}`}:
- + {/if}
    {#each files as { name }, i}
- {:else} - - Drag and drop {folderOnly ? 'a folder' : multiple ? 'files' : 'a file'} - + {:else if children}{@render children()}{:else} + Drag and drop {folderOnly ? 'a folder' : multiple ? 'files' : 'a file'} {/if} 1 ? 's' : ''} chosen` : 'No file chosen'} bind:this={input} - on:change={({ currentTarget }) => { + onchange={({ currentTarget }) => { onChange(currentTarget.files ? Array.from(currentTarget.files) : null) }} {accept} {multiple} - {...$$restProps} + {...rest} /> {#if defaultFile && (!Array.isArray(defaultFile) || defaultFile.length > 0)}
diff --git a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte index eaf3adcb20..2a7de694aa 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte @@ -6,12 +6,16 @@ import { X } from 'lucide-svelte' import FileUpload from './FileUpload.svelte' - export let title: string - export let open: boolean = false - export let fileKey: string | undefined = undefined + interface Props { + title: string; + open?: boolean; + fileKey?: string | undefined; + } - let s3Folder: string = '' + let { title, open = false, fileKey = $bindable(undefined) }: Props = $props(); + + let s3Folder: string = $state('') const dispatch = createEventDispatcher() function fadeFast(node: HTMLElement) { diff --git a/frontend/src/lib/components/common/kbd/Kbd.svelte b/frontend/src/lib/components/common/kbd/Kbd.svelte index 002f4e8c16..5cfed5353c 100644 --- a/frontend/src/lib/components/common/kbd/Kbd.svelte +++ b/frontend/src/lib/components/common/kbd/Kbd.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/common/layout/ListElement.svelte b/frontend/src/lib/components/common/layout/ListElement.svelte index 891079ce42..f865305c9e 100644 --- a/frontend/src/lib/components/common/layout/ListElement.svelte +++ b/frontend/src/lib/components/common/layout/ListElement.svelte @@ -1,3 +1,11 @@ + +
- + {@render children?.()}
diff --git a/frontend/src/lib/components/common/menu/MenuItem.svelte b/frontend/src/lib/components/common/menu/MenuItem.svelte index 4c3d50643c..99d14826e1 100644 --- a/frontend/src/lib/components/common/menu/MenuItem.svelte +++ b/frontend/src/lib/components/common/menu/MenuItem.svelte @@ -1,11 +1,22 @@ - - -
+ + + + +
- + {@render children?.()}
diff --git a/frontend/src/lib/components/common/menu/ResolveOpen.svelte b/frontend/src/lib/components/common/menu/ResolveOpen.svelte index ec32904a3e..5f6d1a0666 100644 --- a/frontend/src/lib/components/common/menu/ResolveOpen.svelte +++ b/frontend/src/lib/components/common/menu/ResolveOpen.svelte @@ -1,10 +1,18 @@ diff --git a/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte b/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte index c4049802fc..d80a3a14bb 100644 --- a/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte +++ b/frontend/src/lib/components/common/modal/AlwaysMountedModal.svelte @@ -1,4 +1,6 @@ - + {#if isOpen} @@ -72,10 +91,7 @@ css?.popup?.class, 'wm-modal-form-popup' )} - use:clickOutside - on:click_outside={() => { - close() - }} + use:clickOutside={{ onClickOutside: () => close() }} >
@@ -84,15 +100,15 @@
- + {@render headerLeft?.()}
- + {@render headerRight?.()}
- - + +
{}} + onclick={stopPropagation(() => {})} > - + {@render children?.()}
diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte index 55bc4312b5..42257ff53b 100644 --- a/frontend/src/lib/components/common/popup/PopupV2.svelte +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -1,4 +1,5 @@ {#if href} diff --git a/frontend/src/lib/components/common/table/RowIcon.svelte b/frontend/src/lib/components/common/table/RowIcon.svelte index e5769b9816..7599fa6426 100644 --- a/frontend/src/lib/components/common/table/RowIcon.svelte +++ b/frontend/src/lib/components/common/table/RowIcon.svelte @@ -18,7 +18,10 @@ Unplug } from 'lucide-svelte' - export let kind: + + + interface Props { + kind: | 'script' | 'flow' | 'app' @@ -38,13 +41,15 @@ | 'mqtt' | 'sqs' | 'gcp' - | 'emails' + | 'emails'; + /** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */ + triggerKind?: string | undefined; + } - /** For 'trigger' kind, specifies the specific trigger type (routes, schedules, etc.) */ - export let triggerKind: string | undefined = undefined + let { kind, triggerKind = undefined }: Props = $props(); // Use triggerKind if kind is 'trigger' and triggerKind is provided - $: effectiveKind = kind === 'trigger' && triggerKind ? triggerKind : kind + let effectiveKind = $derived(kind === 'trigger' && triggerKind ? triggerKind : kind)
diff --git a/frontend/src/lib/components/common/table/Table.svelte b/frontend/src/lib/components/common/table/Table.svelte index b0e99bc9e1..0e6b72658a 100644 --- a/frontend/src/lib/components/common/table/Table.svelte +++ b/frontend/src/lib/components/common/table/Table.svelte @@ -1,8 +1,16 @@ + +
- + {@render children?.()}
diff --git a/frontend/src/lib/components/common/tabs/Tabs.svelte b/frontend/src/lib/components/common/tabs/Tabs.svelte index 642340b01b..e8a98a21c2 100644 --- a/frontend/src/lib/components/common/tabs/Tabs.svelte +++ b/frontend/src/lib/components/common/tabs/Tabs.svelte @@ -1,5 +1,5 @@ @@ -27,16 +31,18 @@ {/each} - + {#snippet actions()} + + {/snippet} diff --git a/frontend/src/lib/components/copilot/MetadataGen.svelte b/frontend/src/lib/components/copilot/MetadataGen.svelte index 502b38e848..bb18b3dc1e 100644 --- a/frontend/src/lib/components/copilot/MetadataGen.svelte +++ b/frontend/src/lib/components/copilot/MetadataGen.svelte @@ -4,7 +4,7 @@ import { Check, Loader2, Wand2 } from 'lucide-svelte' import { metadataCompletionEnabled } from '$lib/stores' import { copilotInfo } from '$lib/aiStore' - import { onDestroy } from 'svelte' + import { onDestroy, untrack } from 'svelte' import { sendUserToast } from '$lib/toast' import { twMerge } from 'tailwind-merge' import autosize from '$lib/autosize' @@ -143,7 +143,7 @@ Generate a tool name for the script below: let genHeight = $state(0) let focused = $state(false) - let config: PromptConfig = promptConfigs[promptConfigName] + let config: PromptConfig = promptConfigs[untrack(() => promptConfigName)] async function generateContent(automatic = false) { abortController = new AbortController() @@ -187,10 +187,10 @@ Generate a tool name for the script below: if ( $copilotInfo.enabled && $metadataCompletionEnabled && - generateOnAppear && + untrack(() => generateOnAppear) && !content && - code && - !isInitialCode(code) + untrack(() => code) && + !isInitialCode(untrack(() => code) ?? '') ) { setTimeout(() => { el?.focus() diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index faf6406c2a..b716e48385 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,4 +1,6 @@ { + onkeydown={(e) => { if ((e.ctrlKey || e.metaKey) && e.key === 'l') { e.preventDefault() aiChatManager.toggleOpen() diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 9b609b9095..140f20f140 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -121,55 +121,59 @@
- - - {/each} -
- {/if} -
- + > +
+ {chat.title} +
+ + {/each} +
+ {/if} +
+ + {/snippet} + {#snippet trigger()} + +
+ + {aiChatManager.mode.charAt(0).toUpperCase() + aiChatManager.mode.slice(1)} mode + + {#if Object.keys(aiChatManager.allowedModes).filter((k) => aiChatManager.allowedModes[k]).length > 1} +
+ +
{/if} - {/each} -
- +
+ + {/snippet} + {#snippet content({ close })} + +
+ {#each Object.values(AIMode) as possibleMode} + {#if aiChatManager.allowedModes[possibleMode]} + + {/if} + {/each} +
+ + {/snippet}
diff --git a/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte b/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte index f30bdbe48f..5ef41b38e0 100644 --- a/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte @@ -1,4 +1,5 @@ -
- - AI Changes Will Be Lost -

You have pending AI changes that will be rejected when saving. Do you want to continue?

-
-
- + {#snippet actions()} +
+
+ +
-
+ {/snippet} diff --git a/frontend/src/lib/components/details/ClipboardPanel.svelte b/frontend/src/lib/components/details/ClipboardPanel.svelte index 289c7da416..4b6c5312cd 100644 --- a/frontend/src/lib/components/details/ClipboardPanel.svelte +++ b/frontend/src/lib/components/details/ClipboardPanel.svelte @@ -4,27 +4,40 @@ import { twMerge } from 'tailwind-merge' import { inputSizeClasses } from '../text_input/TextInput.svelte' - export let content: string - export let title: string | undefined = undefined - export let size: 'sm' | 'md' = 'md' - export let disabled = false + + interface Props { + content: string; + title?: string | undefined; + size?: 'sm' | 'md'; + disabled?: boolean; + class?: string; + } + + let { + content, + title = undefined, + size = 'md', + disabled = false, + class: className = '' + }: Props = $props(); + {#if title !== undefined}
{title}
{/if} - - + +
{ + onclick={(e) => { if (disabled) { return } diff --git a/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte b/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte index bb9aee5987..9c0b964c9f 100644 --- a/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte +++ b/frontend/src/lib/components/details/ErrorHandlerToggleButton.svelte @@ -7,10 +7,19 @@ import { workspaceStore } from '$lib/stores' import Tooltip from '../Tooltip.svelte' - export let kind: 'script' | 'flow' - export let scriptOrFlowPath: string - export let errorHandlerMuted: boolean | undefined - export let iconOnly: boolean = true + interface Props { + kind: 'script' | 'flow'; + scriptOrFlowPath: string; + errorHandlerMuted: boolean | undefined; + iconOnly?: boolean; + } + + let { + kind, + scriptOrFlowPath, + errorHandlerMuted = $bindable(), + iconOnly = true + }: Props = $props(); async function toggleErrorHandler(): Promise { if ($workspaceStore !== undefined) { diff --git a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte index ea1d610918..51cb389616 100644 --- a/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte +++ b/frontend/src/lib/components/details/ErrorHandlerToggleButtonV2.svelte @@ -5,13 +5,23 @@ import { sendUserToast } from '$lib/toast' import { workspaceStore } from '$lib/stores' - export let kind: 'script' | 'flow' - export let scriptOrFlowPath: string - export let errorHandlerMuted: boolean | undefined - export let textDisabled = false - let toggleState = errorHandlerMuted + interface Props { + kind: 'script' | 'flow' + scriptOrFlowPath: string + errorHandlerMuted: boolean | undefined + textDisabled?: boolean + color?: 'nord' | 'red' | 'blue' | undefined + } - export let color: 'nord' | 'red' | 'blue' | undefined = undefined + let { + kind, + scriptOrFlowPath, + errorHandlerMuted = $bindable(), + textDisabled = false, + color = undefined + }: Props = $props() + + let toggleState = $state(errorHandlerMuted) async function toggleErrorHandler(): Promise { toggleState = !toggleState diff --git a/frontend/src/lib/components/flows/FlowModuleIcon.svelte b/frontend/src/lib/components/flows/FlowModuleIcon.svelte index 97a7fd4340..9da2812ec0 100644 --- a/frontend/src/lib/components/flows/FlowModuleIcon.svelte +++ b/frontend/src/lib/components/flows/FlowModuleIcon.svelte @@ -1,4 +1,5 @@ {#if module?.value?.type === 'aiagent'} diff --git a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte index 87e073ca94..401bf232b1 100644 --- a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte +++ b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte @@ -55,8 +55,9 @@ const key = getCachedKey(path) latestHash = cachedValues[key]?.latestHash } - if (flowModuleValue?.type === 'script' && flowModuleValue.path) { - getCachedValues(flowModuleValue.path) + const untrackedFlowModuleValue = untrack(() => flowModuleValue) + if (untrackedFlowModuleValue?.type === 'script' && untrackedFlowModuleValue.path) { + getCachedValues(untrackedFlowModuleValue.path) } async function loadLatestHash(value: PathScript) { diff --git a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte index 4d9b003d81..d624c2132f 100644 --- a/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte +++ b/frontend/src/lib/components/flows/content/AgentToolWrapper.svelte @@ -1,11 +1,6 @@
diff --git a/frontend/src/lib/components/flows/content/FlowFailureModule.svelte b/frontend/src/lib/components/flows/content/FlowFailureModule.svelte index d8745aa8d9..69a30dbe84 100644 --- a/frontend/src/lib/components/flows/content/FlowFailureModule.svelte +++ b/frontend/src/lib/components/flows/content/FlowFailureModule.svelte @@ -4,10 +4,14 @@ import FlowModuleWrapper from './FlowModuleWrapper.svelte' import type { FlowModule } from '$lib/gen' - export let noEditor = false - export let savedModule: FlowModule | undefined = undefined + interface Props { + noEditor?: boolean; + savedModule?: FlowModule | undefined; + } - const { flowStore } = getContext('FlowEditorContext') + let { noEditor = false, savedModule = undefined }: Props = $props(); + + const { flowStore } = $state(getContext('FlowEditorContext')) {#if flowStore.val.value.failure_module} diff --git a/frontend/src/lib/components/flows/content/FlowInputs.svelte b/frontend/src/lib/components/flows/content/FlowInputs.svelte index 297cbf2ba5..730dd5985d 100644 --- a/frontend/src/lib/components/flows/content/FlowInputs.svelte +++ b/frontend/src/lib/components/flows/content/FlowInputs.svelte @@ -3,7 +3,7 @@ import ToggleHubWorkspace from '$lib/components/ToggleHubWorkspace.svelte' import Tooltip from '$lib/components/Tooltip.svelte' - import { createEventDispatcher, getContext } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import FlowScriptPicker from '../pickers/FlowScriptPicker.svelte' import PickHubScript from '../pickers/PickHubScript.svelte' import WorkspaceScriptPicker from '../pickers/WorkspaceScriptPicker.svelte' @@ -38,7 +38,7 @@ const dispatch = createEventDispatcher() let kind: 'script' | 'failure' | 'approval' | 'trigger' = $state( - failureModule + untrack(() => failureModule) ? 'failure' : summary == 'Trigger' ? 'trigger' diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 48edcc43d9..4a43c2c38d 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -152,7 +152,7 @@ shellcheck: false }) - let selected = $state(preprocessorModule ? 'test' : 'inputs') + let selected = $state(untrack(() => preprocessorModule) ? 'test' : 'inputs') let advancedSelected = $state('retries') let advancedRuntimeSelected = $state('concurrency') let s3Kind = $state('s3_client') @@ -237,7 +237,7 @@ } let forceReload = $state(0) - let editorPanelSize = $state(noEditor ? 0 : flowModule.value.type == 'script' ? 30 : 50) + let editorPanelSize = $state(untrack(() => noEditor) ? 0 : flowModule.value.type == 'script' ? 30 : 50) let editorSettingsPanelSize = $state(100 - untrack(() => editorPanelSize)) let stepHistoryLoader = getStepHistoryLoaderContext() diff --git a/frontend/src/lib/components/flows/content/FlowModuleMockTransitionMessage.svelte b/frontend/src/lib/components/flows/content/FlowModuleMockTransitionMessage.svelte index 08d08a48c0..34384cc506 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleMockTransitionMessage.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleMockTransitionMessage.svelte @@ -4,7 +4,7 @@ import DarkModeObserver from '$lib/components/DarkModeObserver.svelte' import { Info } from 'lucide-svelte' - let darkMode = true + let darkMode = $state(true) function openFullscreen(event: MouseEvent) { const img = event.target as HTMLImageElement @@ -59,13 +59,13 @@
- - + + History picker
@@ -82,13 +82,13 @@
- - + + Pin action
@@ -105,13 +105,13 @@
- - + + Recover pins
@@ -129,13 +129,13 @@
- - + + Flow view pinning
diff --git a/frontend/src/lib/components/flows/content/FlowModuleScript.svelte b/frontend/src/lib/components/flows/content/FlowModuleScript.svelte index cde3371f38..7b63910010 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleScript.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleScript.svelte @@ -64,7 +64,7 @@ notFound = cachedValues[key]?.notFound ?? false } - getCachedValues(path, hash) + getCachedValues(untrack(() => path), untrack(() => hash)) async function loadPreviousCode(previousHash: string) { try { diff --git a/frontend/src/lib/components/flows/content/FlowPathViewer.svelte b/frontend/src/lib/components/flows/content/FlowPathViewer.svelte index 31593fbf43..c1c4b5c9de 100644 --- a/frontend/src/lib/components/flows/content/FlowPathViewer.svelte +++ b/frontend/src/lib/components/flows/content/FlowPathViewer.svelte @@ -1,4 +1,6 @@
diff --git a/frontend/src/lib/components/flows/content/FlowPreprocessorModule.svelte b/frontend/src/lib/components/flows/content/FlowPreprocessorModule.svelte index b074b90685..24534b53db 100644 --- a/frontend/src/lib/components/flows/content/FlowPreprocessorModule.svelte +++ b/frontend/src/lib/components/flows/content/FlowPreprocessorModule.svelte @@ -4,10 +4,14 @@ import FlowModuleWrapper from './FlowModuleWrapper.svelte' import type { FlowModule } from '$lib/gen' - export let noEditor = false - export let savedModule: FlowModule | undefined = undefined + interface Props { + noEditor?: boolean; + savedModule?: FlowModule | undefined; + } - const { flowStore } = getContext('FlowEditorContext') + let { noEditor = false, savedModule = undefined }: Props = $props(); + + const { flowStore } = $state(getContext('FlowEditorContext')) {#if flowStore.val.value.preprocessor_module} diff --git a/frontend/src/lib/components/flows/content/FlowSettings.svelte b/frontend/src/lib/components/flows/content/FlowSettings.svelte index 98a247f792..3047996a9f 100644 --- a/frontend/src/lib/components/flows/content/FlowSettings.svelte +++ b/frontend/src/lib/components/flows/content/FlowSettings.svelte @@ -83,7 +83,10 @@ { name: 'Dedicated Worker', active: Boolean(flowStore.val.dedicated_worker) }, { name: 'Concurrent Limit', active: Boolean(flowStore.val.value.concurrent_limit) }, { name: 'Debouncing', active: Boolean(flowStore.val.value.debounce_delay_s) }, - { name: `Run on Behalf of ${flowStore.val.on_behalf_of_email ?? 'Last Editor'}`, active: Boolean(flowStore.val.on_behalf_of_email) }, + { + name: `Run on Behalf of ${flowStore.val.on_behalf_of_email ?? 'Last Editor'}`, + active: Boolean(flowStore.val.on_behalf_of_email) + }, { name: 'Worker Tag', active: displayWorkerTagPicker } ]) @@ -431,8 +434,10 @@ /> {:else if flowStore.val.on_behalf_of_email && !canPreserve} - Currently: {$savedOnBehalfOfEmail ?? flowStore.val.on_behalf_of_email}. - Will be set to {$userStore?.email} on deploy (requires admin or wm_deployers group to override) + Currently: {$savedOnBehalfOfEmail ?? flowStore.val.on_behalf_of_email}. Will be set to {$userStore?.email} on deploy (requires + admin or wm_deployers group to override) {/if} diff --git a/frontend/src/lib/components/flows/content/GenAiQuick.svelte b/frontend/src/lib/components/flows/content/GenAiQuick.svelte index 39ae79d0cf..e0e681d568 100644 --- a/frontend/src/lib/components/flows/content/GenAiQuick.svelte +++ b/frontend/src/lib/components/flows/content/GenAiQuick.svelte @@ -1,10 +1,17 @@ - + - +
To add a form, go to the Form tab, inside the Advanced {'->'} Suspend tab, and add a form. diff --git a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte index 39e232b7b9..c945454e52 100644 --- a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte +++ b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte @@ -4,7 +4,7 @@ import Popover from '$lib/components/Popover.svelte' import { twMerge } from 'tailwind-merge' import VirtualList from '@tutorlatin/svelte-tiny-virtual-list' - import type { onSelectedIteration } from '$lib/components/graph/graphBuilder.svelte' + import type { OnSelectedIteration } from '$lib/components/graph/graphBuilder.svelte' import { untrack } from 'svelte' interface Props { @@ -14,7 +14,7 @@ flowJobsSuccess: (boolean | undefined)[] | undefined selected: number selectedManually: boolean | undefined - onSelectedIteration?: onSelectedIteration + onSelectedIteration?: OnSelectedIteration showIcon?: boolean } diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 28c66c05ca..8e0250cec8 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -21,7 +21,7 @@ Timer, Maximize2 } from 'lucide-svelte' - import { createEventDispatcher, getContext } from 'svelte' + import { createEventDispatcher, getContext, untrack } from 'svelte' import { fade } from 'svelte/transition' import type { FlowEditorContext } from '../types' import { twMerge } from 'tailwind-merge' @@ -157,7 +157,7 @@ let editId = $state(false) - let newId: string = $state(id ?? '') + let newId: string = $state(untrack(() => id) ?? '') let moduleTest: ModuleTest | undefined = $state(undefined) let testIsLoading = $state(false) diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 3bc8d0c82a..ba19244f31 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -3,7 +3,7 @@ - + diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte index c08cfcc056..62eadd5d90 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggerCount.svelte @@ -1,7 +1,11 @@ {#if count && count > 0} diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte index 01485fa519..4bf556078d 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte @@ -189,11 +189,7 @@ type === 'email' || type === 'cli' || (triggersGrouped[type] && triggersGrouped[type].length === 1)} - e.stopPropagation()} - > + {#snippet text()} {camelCaseToWords(type)} {/snippet} @@ -279,7 +275,7 @@ isSelected ? 'bg-surface-accent-selected text-accent border-border-selected' : '', small ? 'w-[23px] h-[23px]' : 'p-2' )} - on:click={(e) => { + onClick={(e) => { e.stopPropagation() e.preventDefault() if (singleItem) { diff --git a/frontend/src/lib/components/home/FlowIcon.svelte b/frontend/src/lib/components/home/FlowIcon.svelte index f9822b063f..70eb02aaea 100644 --- a/frontend/src/lib/components/home/FlowIcon.svelte +++ b/frontend/src/lib/components/home/FlowIcon.svelte @@ -2,9 +2,16 @@ import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' import { twMerge } from 'tailwind-merge' - export let color: string = 'black' + + interface Props { + color?: string; + class?: string; + } + + let { color = 'black', class: className = '' }: Props = $props(); + -
+
diff --git a/frontend/src/lib/components/home/ItemsList.svelte b/frontend/src/lib/components/home/ItemsList.svelte index f66e096ecc..cae0144169 100644 --- a/frontend/src/lib/components/home/ItemsList.svelte +++ b/frontend/src/lib/components/home/ItemsList.svelte @@ -31,7 +31,7 @@ import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import FlowIcon from './FlowIcon.svelte' import { canWrite, getLocalSetting, storeLocalSetting } from '$lib/utils' - import { page } from '$app/stores' + import { page } from '$app/state' import { setQuery } from '$lib/navigation' import Drawer from '../common/drawer/Drawer.svelte' import HighlightCode from '../HighlightCode.svelte' @@ -48,7 +48,11 @@ showEditButtons?: boolean } - let { filter = $bindable(''), subtab = $bindable('script'), showEditButtons = true }: Props = $props() + let { + filter = $bindable(''), + subtab = $bindable('script'), + showEditButtons = true + }: Props = $props() type TableItem = T & { canWrite: boolean @@ -73,7 +77,7 @@ let filteredItems: (TableScript | TableFlow | TableApp | TableRawApp)[] = $state([]) let itemKind = $state( - ($page.url.searchParams.get('kind') as 'script' | 'flow' | 'app' | 'all') ?? 'all' + (page.url.searchParams.get('kind') as 'script' | 'flow' | 'app' | 'all') ?? 'all' ) let loading = $state(true) @@ -367,7 +371,7 @@ if (itemKind != 'all') { subtab = v } - setQuery($page.url, 'kind', v) + setQuery(page.url, 'kind', v) }} > {#snippet children({ item })} diff --git a/frontend/src/lib/components/home/ListFiltersQuick.svelte b/frontend/src/lib/components/home/ListFiltersQuick.svelte index 6a36338094..ed80815700 100644 --- a/frontend/src/lib/components/home/ListFiltersQuick.svelte +++ b/frontend/src/lib/components/home/ListFiltersQuick.svelte @@ -4,13 +4,16 @@ import { createEventDispatcher } from 'svelte' import { Button } from '../common' - export let filters: string[] - export let selectedFilter: - | { kind: 'owner' | 'integrations'; name: string | undefined } - | undefined = undefined - $: selectedAppFilter = selectedFilter?.kind === 'integrations' ? selectedFilter?.name : undefined - export let resourceType = false + interface Props { + filters: string[]; + selectedFilter?: + | { kind: 'owner' | 'integrations'; name: string | undefined } + | undefined; + resourceType?: boolean; + } + + let { filters, selectedFilter = $bindable(undefined), resourceType = false }: Props = $props(); function getIconComponent(name: string, resourceType: boolean) { if (resourceType) { @@ -29,6 +32,7 @@ } const dispatch = createEventDispatcher() + let selectedAppFilter = $derived(selectedFilter?.kind === 'integrations' ? selectedFilter?.name : undefined) {#if Array.isArray(filters) && filters.length > 0} diff --git a/frontend/src/lib/components/icons/RubyIcon.svelte b/frontend/src/lib/components/icons/RubyIcon.svelte index 307fde1a3d..5f2623dddf 100644 --- a/frontend/src/lib/components/icons/RubyIcon.svelte +++ b/frontend/src/lib/components/icons/RubyIcon.svelte @@ -24,7 +24,7 @@ image/svg+xml - + > diff --git a/frontend/src/lib/components/meltComponents/Menu.svelte b/frontend/src/lib/components/meltComponents/Menu.svelte index 47f79262e5..e18e520141 100644 --- a/frontend/src/lib/components/meltComponents/Menu.svelte +++ b/frontend/src/lib/components/meltComponents/Menu.svelte @@ -1,4 +1,5 @@ -
- +
+ {@render children?.({ createMenu, })}
diff --git a/frontend/src/lib/components/meltComponents/Popover.svelte b/frontend/src/lib/components/meltComponents/Popover.svelte index 4d9e9f83ed..96a0934611 100644 --- a/frontend/src/lib/components/meltComponents/Popover.svelte +++ b/frontend/src/lib/components/meltComponents/Popover.svelte @@ -1,4 +1,4 @@ - isOpen && onKeyDown(e)} /> {#if fullScreen && isOpen && !disablePopup} @@ -206,14 +249,14 @@ {#if isOpen && !disablePopup}
{ + onmouseenter={() => { if (openOnHover) { open() clearDebounceClose() } }} - on:mouseleave={debounceClose} - use:melt={$content} + onmouseleave={debounceClose} + use:melt={$_content} transition:fly={{ duration: enableFlyTransition ? 100 : 0, y: -16 }} class={twMerge( 'relative dark:border rounded-md bg-surface-tertiary shadow-lg', @@ -253,7 +296,7 @@ {/if} - + {@render content?.({ open, close })}
{/if} diff --git a/frontend/src/lib/components/meltComponents/SideBarTab.svelte b/frontend/src/lib/components/meltComponents/SideBarTab.svelte index de6bceb0c1..b654422c01 100644 --- a/frontend/src/lib/components/meltComponents/SideBarTab.svelte +++ b/frontend/src/lib/components/meltComponents/SideBarTab.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/meltComponents/Tooltip.svelte b/frontend/src/lib/components/meltComponents/Tooltip.svelte index 3bf7482fbd..86b0d948fd 100644 --- a/frontend/src/lib/components/meltComponents/Tooltip.svelte +++ b/frontend/src/lib/components/meltComponents/Tooltip.svelte @@ -1,4 +1,5 @@ - - + + {@render children?.()} -{#if !$$slots.default} +{#if !children}
@@ -49,7 +72,7 @@
{#snippet children()} - + {@render text?.()} {/snippet}
diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte index 75dc1573d3..f024b73eda 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewer.svelte @@ -67,7 +67,7 @@ editKey }: Props = $props() - let jsonFiltered = $state(json) + let jsonFiltered = $state(untrack(() => json)) let search = $state('') let searchOpen = $state(false) diff --git a/frontend/src/lib/components/propertyPicker/ObjectViewerWrapper.svelte b/frontend/src/lib/components/propertyPicker/ObjectViewerWrapper.svelte index 69b476279f..ccf272e9d5 100644 --- a/frontend/src/lib/components/propertyPicker/ObjectViewerWrapper.svelte +++ b/frontend/src/lib/components/propertyPicker/ObjectViewerWrapper.svelte @@ -1,33 +1,50 @@
- +
diff --git a/frontend/src/lib/components/propertyPicker/PropPicker.svelte b/frontend/src/lib/components/propertyPicker/PropPicker.svelte index 6910750480..e48afe0bc9 100644 --- a/frontend/src/lib/components/propertyPicker/PropPicker.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPicker.svelte @@ -61,9 +61,9 @@ $pickablePropertiesFiltered = pickableProperties }) - let flowInputsFiltered: any = $state(pickableProperties.flow_input) - let resultByIdFiltered: any = $state(pickableProperties.priorIds) - let flowEnvFiltered: any = $state(pickableProperties.flow_env) + let flowInputsFiltered: any = $state(untrack(() => pickableProperties).flow_input) + let resultByIdFiltered: any = $state(untrack(() => pickableProperties).priorIds) + let flowEnvFiltered: any = $state(untrack(() => pickableProperties).flow_env) let timeout: number | undefined function onSearch(search: string) { diff --git a/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte b/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte index 7d56abeb05..b4ed9c11c9 100644 --- a/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte +++ b/frontend/src/lib/components/propertyPicker/PropPickerResult.svelte @@ -1,11 +1,21 @@
diff --git a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte index 9a5e4cba4d..9e91ef55d0 100644 --- a/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte +++ b/frontend/src/lib/components/raw_apps/DefaultDatabaseSelector.svelte @@ -48,44 +48,48 @@ - - - - -
-
Default Datatable & Schema
+ {#snippet trigger()} + + + + {/snippet} + {#snippet content()} + +
+
Default Datatable & Schema
-

- {description} -

+

+ {description} +

-
- Database - datatable, (v) => onChange?.(v, schema)} + placeholder="Select database" + size="sm" + /> +
+ +
+ Schema + schema ?? '', (v) => onChange?.(datatable, v || undefined)} - placeholder="public" - size="sm" - /> -
-
- + + {/snippet} diff --git a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte index c9be2662e9..9819ed8796 100644 --- a/frontend/src/lib/components/raw_apps/FileTreeNode.svelte +++ b/frontend/src/lib/components/raw_apps/FileTreeNode.svelte @@ -27,7 +27,7 @@ import HtmlIcon from '../icons/HtmlIcon.svelte' import MarkdownIcon from '../icons/MarkdownIcon.svelte' import YamlIcon from '../icons/YamlIcon.svelte' - import { tick } from 'svelte' + import { tick, untrack } from 'svelte' interface TreeNode { name: string @@ -68,7 +68,7 @@ let userExpanded = $state(null) // null = not set by user let isHovered = $state(false) - let editValue = $state(node.name) + let editValue = $state(untrack(() => node).name) let textInputElement: TextInput | undefined = $state() let dropdownOpen = $state(false) diff --git a/frontend/src/lib/components/raw_apps/RunnableRow.svelte b/frontend/src/lib/components/raw_apps/RunnableRow.svelte index 0b901b365b..0ed67aab22 100644 --- a/frontend/src/lib/components/raw_apps/RunnableRow.svelte +++ b/frontend/src/lib/components/raw_apps/RunnableRow.svelte @@ -5,7 +5,7 @@ import { twMerge } from 'tailwind-merge' import type { Runnable } from '../apps/inputType' import TextInput from '../text_input/TextInput.svelte' - import { tick } from 'svelte' + import { tick, untrack } from 'svelte' interface Props { id: string @@ -32,7 +32,7 @@ }: Props = $props() let dropdownOpen = $state(false) - let editValue = $state(id) + let editValue = $state(untrack(() => id)) let textInputElement: TextInput | undefined = $state() function finishEdit() { diff --git a/frontend/src/lib/components/recording/FlowRecordingReplay.svelte b/frontend/src/lib/components/recording/FlowRecordingReplay.svelte index cf6c3b2bdf..c5d94a7b95 100644 --- a/frontend/src/lib/components/recording/FlowRecordingReplay.svelte +++ b/frontend/src/lib/components/recording/FlowRecordingReplay.svelte @@ -180,10 +180,12 @@

{recording.flow_path}

- - Recorded {new Date(recording.recorded_at).toLocaleString()} — - {(recording.total_duration_ms / 1000).toFixed(1)}s - + {#snippet text()} + + Recorded {new Date(recording.recorded_at).toLocaleString()} — + {(recording.total_duration_ms / 1000).toFixed(1)}s + + {/snippet}
- + {#if done && job}

Result

{#if job.type === 'CompletedJob' && job.result !== undefined} - + {:else}
No result available
{/if} diff --git a/frontend/src/lib/components/runs/MobileFilters.svelte b/frontend/src/lib/components/runs/MobileFilters.svelte index 8271897001..74bbfdacc5 100644 --- a/frontend/src/lib/components/runs/MobileFilters.svelte +++ b/frontend/src/lib/components/runs/MobileFilters.svelte @@ -4,17 +4,26 @@ import { Filter } from 'lucide-svelte' import { Button } from '../common' import Popover from '$lib/components/meltComponents/Popover.svelte' + interface Props { + filters?: import('svelte').Snippet; + } + + let { filters }: Props = $props(); - - - - -
- -
- + {#snippet trigger()} + + + + {/snippet} + {#snippet content()} + +
+ {@render filters?.()} +
+ + {/snippet} diff --git a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte index 19d373beae..7dea3dcbca 100644 --- a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte +++ b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte @@ -7,16 +7,21 @@ // import Button from '$lib/components/common/button/Button.svelte' // import { json } from 'svelte-highlight/languages' // import { copyToClipboard } from '$lib/utils' - // import { deepEqual } from 'fast-equals' + - export let preprocessed: boolean | undefined + interface Props { + // import { deepEqual } from 'fast-equals' + preprocessed: boolean | undefined; + } + + let { preprocessed }: Props = $props(); // $: args = // '_metadata' in flowStatus && 'original_args' in flowStatus['_metadata'] // ? flowStatus['_metadata']['original_args'] // : undefined - $: hasPreprocessedArgs = preprocessed === true + let hasPreprocessedArgs = $derived(preprocessed === true) // $: argsStr = args !== undefined ? JSON.stringify(args, null, 4) : undefined diff --git a/frontend/src/lib/components/runs/QueuePopover.svelte b/frontend/src/lib/components/runs/QueuePopover.svelte index bede71a09a..936fae1041 100644 --- a/frontend/src/lib/components/runs/QueuePopover.svelte +++ b/frontend/src/lib/components/runs/QueuePopover.svelte @@ -5,8 +5,12 @@ import Skeleton from '../common/skeleton/Skeleton.svelte' import { displayDate } from '$lib/utils' - let jobs: QueuedJob[] | undefined = undefined - export let allWorkspaces: boolean = false + let jobs: QueuedJob[] | undefined = $state(undefined) + interface Props { + allWorkspaces?: boolean; + } + + let { allWorkspaces = false }: Props = $props(); getQueuedJobs() async function getQueuedJobs() { diff --git a/frontend/src/lib/components/runs/RunBadges.svelte b/frontend/src/lib/components/runs/RunBadges.svelte index 6497f108fd..7f32cbe588 100644 --- a/frontend/src/lib/components/runs/RunBadges.svelte +++ b/frontend/src/lib/components/runs/RunBadges.svelte @@ -89,7 +89,7 @@ {/if} {#if concurrencyKey}
- + {#snippet text()} This job has concurrency limits enabled with the key: {#if onFilterByConcurrencyKey} diff --git a/frontend/src/lib/components/schema/AddProperty.svelte b/frontend/src/lib/components/schema/AddProperty.svelte index f5a7b2b8b7..81270b6fd8 100644 --- a/frontend/src/lib/components/schema/AddProperty.svelte +++ b/frontend/src/lib/components/schema/AddProperty.svelte @@ -5,7 +5,11 @@ import SimpleEditor from '../SimpleEditor.svelte' import AddPropertyForm from './AddPropertyForm.svelte' - export let schema: Schema | any = emptySchema() + interface Props { + schema?: Schema | any; + } + + let { schema = $bindable(emptySchema()) }: Props = $props(); export const DEFAULT_PROPERTY: ModalSchemaProperty = { selectedType: 'string', diff --git a/frontend/src/lib/components/schema/AddPropertyForm.svelte b/frontend/src/lib/components/schema/AddPropertyForm.svelte index 7e745a706b..a9c1d7f27f 100644 --- a/frontend/src/lib/components/schema/AddPropertyForm.svelte +++ b/frontend/src/lib/components/schema/AddPropertyForm.svelte @@ -4,7 +4,7 @@ import { Plus } from 'lucide-svelte' import TextInput from '../text_input/TextInput.svelte' - let name: string = '' + let name: string = $state('') const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/schema/EditableSchemaSdkWrapper.svelte b/frontend/src/lib/components/schema/EditableSchemaSdkWrapper.svelte index 124779635e..e546b7b30f 100644 --- a/frontend/src/lib/components/schema/EditableSchemaSdkWrapper.svelte +++ b/frontend/src/lib/components/schema/EditableSchemaSdkWrapper.svelte @@ -1,4 +1,5 @@ +
+
(open = false) }} diff --git a/frontend/src/lib/components/select/SelectDropdown.svelte b/frontend/src/lib/components/select/SelectDropdown.svelte index efed21b988..62a990bb46 100644 --- a/frontend/src/lib/components/select/SelectDropdown.svelte +++ b/frontend/src/lib/components/select/SelectDropdown.svelte @@ -77,7 +77,7 @@ { + onkeydown={(e) => { if (!isVisible || !processedItems?.length) return if (e.key === 'ArrowUp' && keyArrowPos !== undefined && processedItems.length > 0) { keyArrowPos = keyArrowPos <= 0 ? undefined : keyArrowPos - 1 diff --git a/frontend/src/lib/components/settings/ChangeWorkspaceId.svelte b/frontend/src/lib/components/settings/ChangeWorkspaceId.svelte index 3c46007c8b..8194f0f440 100644 --- a/frontend/src/lib/components/settings/ChangeWorkspaceId.svelte +++ b/frontend/src/lib/components/settings/ChangeWorkspaceId.svelte @@ -1,4 +1,6 @@
diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index ccbf6b167d..fbd9e41081 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -1,4 +1,5 @@ @@ -140,17 +148,19 @@
- - - + {#snippet actions()} + + + + {/snippet}
@@ -497,7 +507,7 @@ {#if premiumInfo?.premium && plan === 'team'}
- - + {/snippet} + {#snippet content()}
{#if $superadmin} @@ -226,7 +239,7 @@ />
- + {/snippet} {/if} @@ -236,14 +249,14 @@ portal="#settings-button" contentClasses="p-4" > - + {#snippet trigger()}
- - + {/snippet} + {#snippet content()}
- + {/snippet} {:else} - - {#if !isDeployed} - Deploy the runnable to enable trigger creation - {:else if cloudDisabled} - This trigger is disabled in the multi-tenant cloud - {:else} - Enter a valid config to {trigger?.isDraft ? 'deploy' : 'update'} the trigger - {/if} - + {#snippet text()} + + {#if !isDeployed} + Deploy the runnable to enable trigger creation + {:else if cloudDisabled} + This trigger is disabled in the multi-tenant cloud + {:else} + Enter a valid config to {trigger?.isDraft ? 'deploy' : 'update'} the trigger + {/if} + + {/snippet} {/if}
diff --git a/frontend/src/lib/components/triggers/TriggerTokens.svelte b/frontend/src/lib/components/triggers/TriggerTokens.svelte index b546a467a1..4cf1a5d027 100644 --- a/frontend/src/lib/components/triggers/TriggerTokens.svelte +++ b/frontend/src/lib/components/triggers/TriggerTokens.svelte @@ -1,4 +1,6 @@
@@ -62,7 +70,7 @@
{#if token.email == $userStore?.email} {:else} diff --git a/frontend/src/lib/components/triggers/email/EmailCapture.svelte b/frontend/src/lib/components/triggers/email/EmailCapture.svelte index 828a32b0d0..f77c15b5bd 100644 --- a/frontend/src/lib/components/triggers/email/EmailCapture.svelte +++ b/frontend/src/lib/components/triggers/email/EmailCapture.svelte @@ -1,7 +1,7 @@ {#if postgres_resource_path} diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte index b29508782d..cd176ac51c 100644 --- a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte @@ -156,7 +156,7 @@ ) const postgresConfig = $derived.by(getSaveCfg) - const captureConfig = $derived.by(isEditor ? getCaptureConfig : () => ({})) + const captureConfig = $derived.by(untrack(() => isEditor) ? getCaptureConfig : () => ({})) const saveDisabled = $derived( pathError !== '' || @@ -642,7 +642,7 @@ resourceType={'postgresql'} datatableAsPgResource /> - +
{#if loadingPostgres}
@@ -769,7 +769,7 @@
{:else} + import { run } from 'svelte/legacy'; + import { Button } from '$lib/components/common' import Select from '$lib/components/select/Select.svelte' import { safeSelectItems } from '$lib/components/select/utils.svelte' @@ -9,17 +11,29 @@ import { emptyString } from '$lib/utils' import { RefreshCw } from 'lucide-svelte' - export let items: string[] = [] - export let can_write: boolean = true - export let publication_name: string = '' - export let postgres_resource_path: string = '' - export let relations: Relations[] | undefined = undefined - export let transaction_to_track: string[] = [] - export let disabled: boolean = false + interface Props { + items?: string[]; + can_write?: boolean; + publication_name?: string; + postgres_resource_path?: string; + relations?: Relations[] | undefined; + transaction_to_track?: string[]; + disabled?: boolean; + } - let loadingPublication: boolean = false - let deletingPublication: boolean = false - let updatingPublication: boolean = false + let { + items = $bindable([]), + can_write = true, + publication_name = $bindable(''), + postgres_resource_path = '', + relations = $bindable(undefined), + transaction_to_track = $bindable([]), + disabled = false + }: Props = $props(); + + let loadingPublication: boolean = $state(false) + let deletingPublication: boolean = $state(false) + let updatingPublication: boolean = $state(false) async function listDatabasePublication() { try { loadingPublication = true @@ -94,7 +108,9 @@ } listDatabasePublication() - $: publication_name && getAllRelations() + run(() => { + publication_name && getAllRelations() + });
diff --git a/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte b/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte index 677eff5abf..25f0863330 100644 --- a/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte +++ b/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte @@ -8,14 +8,23 @@ import { emptyString } from '$lib/utils' import { RefreshCw } from 'lucide-svelte' - export let edit: boolean - export let replication_slot_name: string = '' - export let postgres_resource_path: string = '' - export let disabled: boolean = false + interface Props { + edit: boolean; + replication_slot_name?: string; + postgres_resource_path?: string; + disabled?: boolean; + } - let deletingSlot: boolean = false - let loadingSlot: boolean = false - let items: (string | undefined)[] = [] + let { + edit, + replication_slot_name = $bindable(''), + postgres_resource_path = '', + disabled = false + }: Props = $props(); + + let deletingSlot: boolean = $state(false) + let loadingSlot: boolean = $state(false) + let items: (string | undefined)[] = $state([]) async function listDatabaseSlot() { try { loadingSlot = true diff --git a/frontend/src/lib/components/triggers/testingBadge.svelte b/frontend/src/lib/components/triggers/testingBadge.svelte index 2deb844f21..182c8cd05d 100644 --- a/frontend/src/lib/components/triggers/testingBadge.svelte +++ b/frontend/src/lib/components/triggers/testingBadge.svelte @@ -5,5 +5,7 @@ - Config used for creating a testing endpoint + {#snippet text()} + Config used for creating a testing endpoint + {/snippet} diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte index b359a76625..9bdf67ab94 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte @@ -15,7 +15,7 @@ import { typescript } from 'svelte-highlight/languages' import ClipboardPanel from '../../details/ClipboardPanel.svelte' import { copyToClipboard, isObject, readFieldsRecursively } from '$lib/utils' - // import { page } from '$app/stores' + // import { page } from '$app/state' import { base } from '$lib/base' import TriggerTokens from '../TriggerTokens.svelte' import { workspaceStore, userStore } from '$lib/stores' diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketCapture.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketCapture.svelte index ba64404e12..7fb8307c33 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketCapture.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketCapture.svelte @@ -34,22 +34,24 @@ {isFlow} {captureLoading} > - - {#if captureInfo.active} - {#if captureInfo.connectionInfo?.connected} -

- Listenning to websocket events... -

+ {#snippet description()} + + {#if captureInfo.active} + {#if captureInfo.connectionInfo?.connected} +

+ Listenning to websocket events... +

+ {:else} +

+ Connecting to websocket... +

+ {/if} {:else}

- Connecting to websocket... + Start capturing to listen to websocket events.

{/if} - {:else} -

- Start capturing to listen to websocket events. -

- {/if} - + + {/snippet} {/if} diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index c3a50fa820..01cb49b802 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -117,7 +117,7 @@ let hasChanged = $derived(!deepEqual(getSaveCfg(), originalConfig ?? {})) const websocketCfg = $derived.by(getSaveCfg) - const captureConfig = $derived.by(isEditor ? getCaptureConfig : () => ({})) + const captureConfig = $derived.by(untrack(() => isEditor) ? getCaptureConfig : () => ({})) const saveDisabled = $derived.by(() => { const invalidInitialMessages = initial_messages.some((v) => { if ('runnable_result' in v) { diff --git a/frontend/src/lib/components/tutorials/Tutorial.svelte b/frontend/src/lib/components/tutorials/Tutorial.svelte index 1ef9e2b53c..fc95141f5c 100644 --- a/frontend/src/lib/components/tutorials/Tutorial.svelte +++ b/frontend/src/lib/components/tutorials/Tutorial.svelte @@ -8,20 +8,30 @@ import TutorialInner from './TutorialInner.svelte' import { isCurrentlyInTutorial } from '$lib/stores' - export let index: number = 0 - export let name: string = 'action' - export let tainted: boolean = false - export let onDestroyed: (() => void) | undefined = undefined type Options = { indexToInsertAt?: number skipStepsCount?: number } - export let getSteps: (driver: Driver, options?: Options | undefined) => DriveStep[] = () => [] + interface Props { + index?: number; + name?: string; + tainted?: boolean; + onDestroyed?: (() => void) | undefined; + getSteps?: (driver: Driver, options?: Options | undefined) => DriveStep[]; + } + + let { + index = 0, + name = 'action', + tainted = false, + onDestroyed = undefined, + getSteps = () => [] + }: Props = $props(); let totalSteps = 0 - let tutorial: Driver | undefined = undefined + let tutorial: Driver | undefined = $state(undefined) const dispatch = createEventDispatcher() // Render controls needs to be exposed so steps that have a custom render can call it diff --git a/frontend/src/lib/components/tutorials/TutorialControls.svelte b/frontend/src/lib/components/tutorials/TutorialControls.svelte index 96c0b7385b..826e66d151 100644 --- a/frontend/src/lib/components/tutorials/TutorialControls.svelte +++ b/frontend/src/lib/components/tutorials/TutorialControls.svelte @@ -4,8 +4,12 @@ import { createEventDispatcher } from 'svelte' import Alert from '../common/alert/Alert.svelte' - export let activeIndex: number | undefined = undefined - export let totalSteps: number | undefined = undefined + interface Props { + activeIndex?: number | undefined; + totalSteps?: number | undefined; + } + + let { activeIndex = undefined, totalSteps = undefined }: Props = $props(); const dispatch = createEventDispatcher() diff --git a/frontend/src/lib/components/tutorials/TutorialWrapper.svelte b/frontend/src/lib/components/tutorials/TutorialWrapper.svelte index 34eaad50a0..32b6fa212f 100644 --- a/frontend/src/lib/components/tutorials/TutorialWrapper.svelte +++ b/frontend/src/lib/components/tutorials/TutorialWrapper.svelte @@ -1,4 +1,5 @@ - + @@ -49,14 +51,11 @@
- - + + {#snippet trigger()} - - + {/snippet} + {#snippet content({ close })}
- + {/snippet}
@@ -80,43 +79,47 @@ {#if instanceGroups && instanceGroups.length > 0}
- - Name - Summary - Members - - - - {#each instanceGroups as { name, summary, emails }} - - - { - editGroupName = name - groupDrawer.openDrawer() - }} - >{name} - - - - {summary ? summary.slice(0, 50) + (summary.length > 50 ? '...' : '') : '-'} - - {emails?.length ?? 0} members - - - {/each} - + {#snippet headerRow()} + + Name + Summary + Members + + + {/snippet} + {#snippet body()} + + {#each instanceGroups as { name, summary, emails }} + + + { + editGroupName = name + groupDrawer?.openDrawer() + }} + >{name} + + + + {summary ? summary.slice(0, 50) + (summary.length > 50 ? '...' : '') : '-'} + + {emails?.length ?? 0} members + + + {/each} + + {/snippet}
{/if} diff --git a/frontend/src/routes/(root)/(logged)/oauth/mcp_authorize/+page.svelte b/frontend/src/routes/(root)/(logged)/oauth/mcp_authorize/+page.svelte index 01fec29ff0..99b7691aa5 100644 --- a/frontend/src/routes/(root)/(logged)/oauth/mcp_authorize/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/oauth/mcp_authorize/+page.svelte @@ -1,19 +1,19 @@ diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index 30f8351330..2ad7ec0089 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -1,4 +1,6 @@
Selected Workspace: - - + + {#snippet children({ createMenu })} + + {/snippet}
userSettings.openDrawer()} + on:click={() => userSettings?.openDrawer()} > Generate webhook-specific Token diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte index c7fa2b8e95..e14a2113cb 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte @@ -1,4 +1,6 @@ @@ -73,7 +77,7 @@ {#if !automateUsernameCreation} +{/if} diff --git a/frontend/src/lib/components/CustomOauth.svelte b/frontend/src/lib/components/CustomOauth.svelte index bbc020d794..432a02152b 100644 --- a/frontend/src/lib/components/CustomOauth.svelte +++ b/frontend/src/lib/components/CustomOauth.svelte @@ -6,14 +6,7 @@ import Toggle from './Toggle.svelte' import Tooltip from './Tooltip.svelte' - let { connect_config = $bindable({ - scopes: [], - auth_url: '', - token_url: '', - req_body_auth: false, - extra_params: {}, - extra_params_callback: {} - }) } = $props(); + let { connect_config = $bindable() } = $props(); run(() => { if (!connect_config) { diff --git a/frontend/src/lib/components/CustomSso.svelte b/frontend/src/lib/components/CustomSso.svelte index 5c36e1c57e..4de203357a 100644 --- a/frontend/src/lib/components/CustomSso.svelte +++ b/frontend/src/lib/components/CustomSso.svelte @@ -6,15 +6,7 @@ import Toggle from './Toggle.svelte' import Tooltip from './Tooltip.svelte' - let { login_config = $bindable({ - scopes: [], - auth_url: '', - token_url: '', - userinfo_url: '', - req_body_auth: false, - extra_params: {}, - extra_params_callback: {} - }) } = $props(); + let { login_config = $bindable() } = $props(); run(() => { if (!login_config) { diff --git a/frontend/src/lib/components/OauthExtraParams.svelte b/frontend/src/lib/components/OauthExtraParams.svelte index eec9cd47b8..a2a5ae9749 100644 --- a/frontend/src/lib/components/OauthExtraParams.svelte +++ b/frontend/src/lib/components/OauthExtraParams.svelte @@ -6,9 +6,15 @@ extra_params?: Record; } - let { extra_params = $bindable({}) }: Props = $props(); + let { extra_params = $bindable() }: Props = $props(); - let extra_params_vec: [string, string][] = $state(Object.entries(extra_params)) + $effect.pre(() => { + if (!extra_params) { + extra_params = {} + } + }) + + let extra_params_vec: [string, string][] = $state(Object.entries(extra_params ?? {})) function sync() { extra_params = Object.fromEntries(extra_params_vec) diff --git a/frontend/src/lib/components/OauthScopes.svelte b/frontend/src/lib/components/OauthScopes.svelte index 545471d0dc..4ec17c6e2a 100644 --- a/frontend/src/lib/components/OauthScopes.svelte +++ b/frontend/src/lib/components/OauthScopes.svelte @@ -6,7 +6,13 @@ scopes?: string[] } - let { scopes = $bindable([]) }: Props = $props() + let { scopes = $bindable() }: Props = $props() + + $effect.pre(() => { + if (!scopes) { + scopes = [] + } + }) {#if scopes && Array.isArray(scopes)} @@ -18,7 +24,7 @@ size="xs" btnClasses="mx-6" on:click={() => { - scopes = scopes.filter((el) => el != v) + scopes = scopes?.filter((el) => el != v) }} startIcon={{ icon: Minus }} iconOnly From 2f0b22443765d013351f0b344aa3aa8979790fce Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 18:55:40 +0000 Subject: [PATCH 29/57] docs: ban $bindable(default_value) on optional props in CLAUDE.md (#8267) Add a "Banned Patterns" section documenting that $bindable(default_value) on props that can be undefined is banned. The correct alternatives are using $derived(my_prop ?? default_value) or creating a useMyPropState() helper higher in the component tree. Closes #8266 Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] --- CLAUDE.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 4e7afeba8a..0acc541c92 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,27 @@ Open-source platform for internal tools, workflows, API integrations, background - **Login**: `admin@windmill.dev` / `changeme` - **Instance settings**: navigate to `/#superadmin-settings` +## Banned Patterns + +### `$bindable(default_value)` on optional props + +Using `$bindable(default_value)` on props that can be `undefined` is **banned**. This pattern causes subtle bugs because the default value masks the `undefined` state. + +**Bad:** +```svelte +let { my_prop = $bindable(default_value) }: { my_prop?: string } = $props() +``` + +**Correct alternatives:** + +1. **Use `$derived` with nullish coalescing** — handle the potential `undefined` at the usage site: + ```svelte + let { my_prop = $bindable() }: { my_prop?: string } = $props() + let effective_value = $derived(my_prop ?? default_value) + ``` + +2. **Create a `useMyPropState()` helper** — encapsulate the undefined-handling logic in a reusable function and call it higher in the component tree, so the child component always receives a defined value. + ## Core Principles - Search for existing code to reuse before writing new code From 1c5dea8c3e863024e85c6906039740dcec0069d0 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Sat, 7 Mar 2026 22:41:14 +0100 Subject: [PATCH 30/57] fix: sql input horizontal scroll missing after switching flow steps (#8249) Co-authored-by: Claude Opus 4.6 --- frontend/src/lib/components/TemplateEditor.svelte | 5 ----- 1 file changed, 5 deletions(-) diff --git a/frontend/src/lib/components/TemplateEditor.svelte b/frontend/src/lib/components/TemplateEditor.svelte index 8046e6ec97..98f11c3076 100644 --- a/frontend/src/lib/components/TemplateEditor.svelte +++ b/frontend/src/lib/components/TemplateEditor.svelte @@ -438,7 +438,6 @@ let cip let extraModel - let width = $state(0) // let widgets: HTMLElement | undefined = document.getElementById('monaco-widgets-root') ?? undefined let initialized = $state(false) @@ -545,9 +544,6 @@ if (divEl) { divEl.style.height = `${contentHeight}px` } - try { - editor?.layout({ width, height: contentHeight }) - } catch {} } editor.onDidContentSizeChange(updateHeight) updateHeight() @@ -718,7 +714,6 @@ bind:this={divEl} style="height: 18px;" class="template nonmain-editor rounded-md overflow-clip {!editor ? 'hidden' : ''}" - bind:clientWidth={width} >
From dec7e50b0fcc6fcd6fd0f65c5941b500f7617f98 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 8 Mar 2026 15:44:47 +0000 Subject: [PATCH 31/57] fix: mask secrets in OAuth config debug/log output (#8269) Co-authored-by: Claude Opus 4.6 --- .../windmill-common/src/instance_config.rs | 12 +++++++- backend/windmill-oauth/src/lib.rs | 30 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 4ef88f5b78..d5ab95ee34 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -44,7 +44,7 @@ pub struct EnvRefWrapper { /// /// `Literal` serializes back to a plain JSON string, preserving backwards /// compatibility with existing consumers. -#[derive(Deserialize, Serialize, Clone, Debug)] +#[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] #[serde(untagged)] pub enum StringOrSecretRef { @@ -53,6 +53,16 @@ pub enum StringOrSecretRef { EnvRef(EnvRefWrapper), } +impl fmt::Debug for StringOrSecretRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Literal(_) => f.write_str("Literal(****)"), + Self::SecretRef(w) => f.debug_tuple("SecretRef").field(w).finish(), + Self::EnvRef(w) => f.debug_tuple("EnvRef").field(w).finish(), + } + } +} + impl StringOrSecretRef { /// Returns the literal string value, or `None` if this is an unresolved ref. pub fn as_literal(&self) -> Option<&str> { diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index e0c499c355..26d807dac3 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -94,7 +94,7 @@ pub struct OAuthConfig { } /// OAuth client credentials -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] pub struct OAuthClient { #[serde(default = "empty_string")] pub id: String, @@ -110,6 +110,21 @@ pub struct OAuthClient { pub grant_types: Vec, } +impl std::fmt::Debug for OAuthClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthClient") + .field("id", &self.id) + .field("secret", &"***") + .field("display_name", &self.display_name) + .field("allowed_domains", &self.allowed_domains) + .field("connect_config", &self.connect_config) + .field("login_config", &self.login_config) + .field("tenant", &self.tenant) + .field("grant_types", &self.grant_types) + .finish() + } +} + fn empty_string() -> String { "".to_string() } @@ -608,7 +623,18 @@ pub async fn refresh_token<'c>( .await?; let account = windmill_common::utils::not_found_if_none(account, "Account", &id.to_string())?; - refresh_token_for_account(tx, path, w_id, id, db, account, oauth_clients, http_client, connect_configs_json).await + refresh_token_for_account( + tx, + path, + w_id, + id, + db, + account, + oauth_clients, + http_client, + connect_configs_json, + ) + .await } /// Refresh an OAuth token given pre-fetched account info (no additional SELECT). From a32d0e0da97984df9765c7d92659ed3af8aa3571 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 8 Mar 2026 16:18:22 +0000 Subject: [PATCH 32/57] fix: skip down migrations in potentially_stale checksum comparison (#8271) The potentially_stale block iterated over all migrations including .down.sql reversible migrations. Down migrations share the same version as their up counterpart but have a different checksum, causing the DELETE to remove the up migration row on every startup and triggering re-application of the concurrent index migrations. Co-authored-by: Claude Opus 4.6 --- backend/windmill-api/src/db.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 8b0fb44dfe..c8ed841e19 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -284,6 +284,9 @@ pub async fn migrate( 20260207000004, ]; for m in migrator.migrations.iter() { + if m.migration_type.is_down_migration() { + continue; + } if potentially_stale.contains(&m.version) { if let Err(err) = sqlx::query("DELETE FROM _sqlx_migrations WHERE version = $1 AND checksum != $2") From f7afcdb704fce51adef1af900406df62aa323ac0 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Mon, 9 Mar 2026 08:19:33 +0000 Subject: [PATCH 33/57] fix: guard iteration picker VirtualList against empty items array (#8273) When a flow loops over an empty array, the VirtualList component crashes trying to access index 0 in an empty range. Add a guard to only render VirtualList when items.length > 0, showing a "No iterations" message otherwise. Fixes #8272 Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] Co-authored-by: Claude Opus 4.6 --- frontend/src/lib/components/flows/map/FlowJobsMenu.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte index c945454e52..f6cd1607da 100644 --- a/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte +++ b/frontend/src/lib/components/flows/map/FlowJobsMenu.svelte @@ -142,6 +142,7 @@
{#key items} + {#if items.length > 0} {#snippet header()}{/snippet} {#snippet footer()}{/snippet} @@ -170,6 +171,9 @@
{/snippet} + {:else} +
No iterations
+ {/if} {/key} B[End] - EOF - - 2) Render to SVG (the -p flag is required): - mmdc -i /tmp/diagram.mmd -o /tmp/diagram.svg -p /root/.puppeteerrc.json - - 3) Upload to R2: - aws s3 cp /tmp/diagram.svg - "s3://$(printenv R2_BUCKET)/$(git rev-parse --abbrev-ref HEAD)/diagram.svg" - --endpoint-url "$(printenv R2_ENDPOINT)" - - 4) The public URL will be: - $(printenv R2_PUBLIC_URL)//diagram.svg - - 5) Include in PR descriptions using markdown image syntax. - - IMPORTANT: Read docs/autonomous-mode.md before starting any work. - -linkedRepos: - - repo: windmill-labs/windmill-ee-private - alias: ee +integrations: + github: + linkedRepos: [] + linear: + enabled: true diff --git a/scripts/post-create.sh b/scripts/post-create.sh new file mode 100755 index 0000000000..431e944d74 --- /dev/null +++ b/scripts/post-create.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/worktree-common.sh" + +backend_port="${BACKEND_PORT:-}" +frontend_port="${FRONTEND_PORT:-}" + +if [[ -z "$backend_port" || -z "$frontend_port" ]]; then + echo "Missing BACKEND_PORT or FRONTEND_PORT in hook environment" >&2 + exit 1 +fi + +cat > .env.local <> .env.local +fi + +echo "Created .env.local with ports: backend=$backend_port, frontend=$frontend_port" +wm_shared_post_create "$(pwd)" diff --git a/scripts/pre-remove.sh b/scripts/pre-remove.sh new file mode 100755 index 0000000000..a4bac4431e --- /dev/null +++ b/scripts/pre-remove.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/worktree-common.sh" + +wm_kill_processes_from_env_file "$(pwd)/.env.local" +wm_shared_pre_remove "$(pwd)" diff --git a/scripts/worktree-cleanup b/scripts/worktree-cleanup index c9a2769a58..f876011235 100755 --- a/scripts/worktree-cleanup +++ b/scripts/worktree-cleanup @@ -1,73 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -# Use WM_WORKTREE_PATH (set by workmux) so this works regardless of cwd +source "$(dirname "${BASH_SOURCE[0]}")/worktree-common.sh" + wt_dir="${WM_WORKTREE_PATH:-.}" -echo "[cleanup] cwd=$(pwd) WM_WORKTREE_PATH=${WM_WORKTREE_PATH:-} wt_dir=$wt_dir" - -# Kill backend/frontend processes using this worktree's ports -if [ -f "$wt_dir/.env.local" ]; then - source "$wt_dir/.env.local" - echo "[cleanup] .env.local found: BACKEND_PORT=${BACKEND_PORT:-} FRONTEND_PORT=${FRONTEND_PORT:-}" - for port in "${BACKEND_PORT:-}" "${FRONTEND_PORT:-}"; do - [ -z "$port" ] && continue - pid=$(lsof -ti "TCP:${port}" -sTCP:LISTEN 2>/dev/null || true) - if [ -n "$pid" ]; then - kill "$pid" 2>/dev/null && echo "[cleanup] Killed process $pid on port $port" \ - || echo "[cleanup] Warning: Could not kill process $pid on port $port" - else - echo "[cleanup] No process listening on port $port" - fi - done -else - echo "[cleanup] No .env.local at $wt_dir/.env.local" -fi - -# Drop per-worktree database -if [ -n "${WM_DB_NAME:-}" ]; then - db_conn="postgres://postgres:changeme@127.0.0.1:5432" - if command -v psql &>/dev/null; then - psql "$db_conn/postgres" -c "DROP DATABASE IF EXISTS ${WM_DB_NAME} WITH (FORCE)" 2>/dev/null \ - && echo "[cleanup] Dropped database $WM_DB_NAME" \ - || echo "[cleanup] Warning: Could not drop database $WM_DB_NAME" - else - echo "[cleanup] psql not found, skipping database cleanup for $WM_DB_NAME" - fi -else - echo "[cleanup] No WM_DB_NAME in .env.local, skipping database cleanup" -fi - -# Remove the matching windmill-ee-private worktree if one exists -wt_basename=$(basename "$wt_dir") - -# Find ee repo using same discovery logic as worktree-env -main_repo_root="$(cd "$(git -C "$wt_dir" rev-parse --git-common-dir 2>/dev/null)/.." && pwd)" -parent_dir="$(cd "$wt_dir/.." && pwd)" -echo "[cleanup] wt_basename=$wt_basename main_repo_root=$main_repo_root parent_dir=$parent_dir" - -ee_repo="" -for candidate in \ - "${main_repo_root:+${main_repo_root}/../windmill-ee-private}" \ - "${parent_dir}/windmill-ee-private" \ - "${HOME}/windmill-ee-private" \ - "${HOME}/projects/windmill-ee-private"; do - if [ -n "$candidate" ] && [ -d "$candidate" ]; then - ee_repo="$(cd "$candidate" && pwd)" - break - fi -done - -if [ -z "$ee_repo" ]; then - echo "[cleanup] Could not find windmill-ee-private repo, skipping EE worktree cleanup" -fi - -ee_worktree_dir="${ee_repo:+${ee_repo}__worktrees/${wt_basename}}" -echo "[cleanup] ee_repo=${ee_repo:-} ee_worktree_dir=${ee_worktree_dir:-} exists=$([ -n "$ee_worktree_dir" ] && [ -d "$ee_worktree_dir" ] && echo yes || echo no)" -if [ -n "$ee_worktree_dir" ] && [ -d "$ee_worktree_dir" ]; then - git -C "$ee_repo" worktree remove "$ee_worktree_dir" --force 2>/dev/null \ - && echo "[cleanup] Removed EE worktree at $ee_worktree_dir" \ - || echo "[cleanup] Warning: Could not remove EE worktree at $ee_worktree_dir" -fi - -# Clean up Cursor grouped tmux session -tmux kill-session -t "cursor-${wt_basename}" 2>/dev/null || true +wm_kill_processes_from_env_file "${wt_dir}/.env.local" +wm_shared_pre_remove "$wt_dir" diff --git a/scripts/worktree-common.sh b/scripts/worktree-common.sh new file mode 100755 index 0000000000..0112d271fa --- /dev/null +++ b/scripts/worktree-common.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +set -euo pipefail + +wm_is_true() { + case "${1:-}" in + 1|true|TRUE|yes|YES|on|ON) return 0 ;; + *) return 1 ;; + esac +} + +wm_main_repo_root() { + local repo_root=${1:-.} + cd "$(git -C "$repo_root" rev-parse --git-common-dir 2>/dev/null)/.." && pwd +} + +wm_setup_database() { + local repo_root=$1 + local env_file=$2 + local wt_basename db_name db_conn db_url license_key + + wt_basename="$(basename "$repo_root")" + db_name="windmill_${wt_basename//-/_}" + db_conn="postgres://postgres:changeme@127.0.0.1:5432" + + if ! command -v psql >/dev/null 2>&1; then + echo "WARNING: psql not found, skipping per-worktree database creation" >&2 + return + fi + + if psql "$db_conn/postgres" -tc "SELECT 1 FROM pg_database WHERE datname = '${db_name}'" 2>/dev/null | grep -q 1; then + echo "Database $db_name already exists" + else + if wm_is_true "${WM_CLONE_DB:-}"; then + psql "$db_conn/postgres" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'windmill' AND pid <> pg_backend_pid();" 2>/dev/null || true + psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name} TEMPLATE windmill" 2>/dev/null \ + && echo "Created database $db_name (template: windmill)" \ + || echo "WARNING: Could not create database $db_name from template windmill" >&2 + else + psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name}" 2>/dev/null \ + && echo "Created database $db_name" \ + || echo "WARNING: Could not create database $db_name (is PostgreSQL running?)" >&2 + fi + fi + + db_url="${db_conn}/${db_name}?sslmode=disable" + DATABASE_URL="$db_url" sqlx migrate run --source "${repo_root}/backend/migrations" \ + && echo "Migrations applied to $db_name" \ + || echo "WARNING: Could not run migrations on $db_name" >&2 + + license_key="$(psql "$db_conn/windmill" -t -A -c "SELECT value FROM global_settings WHERE name = 'license_key'" 2>/dev/null || true)" + if [[ -n "$license_key" ]]; then + psql "$db_url" -c "INSERT INTO global_settings (name, value) VALUES ('license_key', '${license_key}'::jsonb) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value" 2>/dev/null \ + && echo "Copied license_key to $db_name" \ + || echo "WARNING: Could not copy license_key to $db_name" >&2 + fi + + cat >> "$env_file" <&2 + fi +} + +wm_allow_direnv() { + local repo_root=$1 + if command -v direnv >/dev/null 2>&1 && [[ -f "${repo_root}/.envrc" ]]; then + (cd "$repo_root" && direnv allow) + echo "direnv allowed" + fi +} + +wm_trust_claude() { + local repo_root=$1 + local claude_json="${HOME}/.claude.json" + + if [[ ! -f "$claude_json" ]] || ! command -v python3 >/dev/null 2>&1; then + return + fi + + REPO_ROOT="$repo_root" CLAUDE_JSON="$claude_json" python3 - <<'PY' \ + && echo "Added $repo_root to Claude Code trusted directories" \ + || echo "Warning: Could not update Claude Code trusted directories" +import json +import os + +path = os.environ["REPO_ROOT"] +claude_json = os.environ["CLAUDE_JSON"] +with open(claude_json, "r") as f: + data = json.load(f) +projects = data.setdefault("projects", {}) +proj = projects.setdefault(path, {}) +proj["hasTrustDialogAccepted"] = True +proj["hasCompletedProjectOnboarding"] = True +with open(claude_json, "w") as f: + json.dump(data, f, indent=2) +PY +} + +wm_find_ee_repo() { + local repo_root=$1 + local main_repo_root=$2 + local candidate + + for candidate in \ + "${main_repo_root}/../windmill-ee-private" \ + "${repo_root}/../windmill-ee-private" \ + "${HOME}/windmill-ee-private" \ + "${HOME}/projects/windmill-ee-private"; do + if [[ -d "$candidate" ]]; then + cd "$candidate" && pwd + return 0 + fi + done + + return 1 +} + +wm_setup_ee_worktree() { + local repo_root=$1 + local main_repo_root=$2 + local ee_repo branch wt_basename ee_worktree_dir ee_rel rust_plugin + + if ! ee_repo="$(wm_find_ee_repo "$repo_root" "$main_repo_root")"; then + return + fi + + branch="$(git -C "$repo_root" branch --show-current 2>/dev/null || true)" + wt_basename="$(basename "$repo_root")" + ee_worktree_dir="${ee_repo}__worktrees/${wt_basename}" + + if [[ -n "$branch" && ! -d "$ee_worktree_dir" ]]; then + mkdir -p "$(dirname "$ee_worktree_dir")" + git -C "$ee_repo" fetch --quiet 2>/dev/null || true + + if git -C "$ee_repo" worktree add "$ee_worktree_dir" "$branch" 2>/dev/null; then + echo "Created EE worktree at $ee_worktree_dir (branch: $branch)" + elif git -C "$ee_repo" worktree add -b "$branch" "$ee_worktree_dir" main 2>/dev/null; then + echo "Created EE worktree at $ee_worktree_dir (new branch: $branch from main)" + else + echo "Warning: Could not create EE worktree for branch $branch" + fi + elif [[ -d "$ee_worktree_dir" ]]; then + echo "EE worktree already exists at $ee_worktree_dir" + fi + + if [[ ! -d "$ee_worktree_dir" ]]; then + return + fi + + ee_rel="$(REPO_ROOT="$repo_root" EE_WORKTREE_DIR="$ee_worktree_dir" python3 - <<'PY' 2>/dev/null || echo "$ee_worktree_dir" +import os +print(os.path.relpath(os.environ["EE_WORKTREE_DIR"], os.environ["REPO_ROOT"])) +PY +)" + mkdir -p "${repo_root}/.claude" + rust_plugin="" + if wm_is_true "${USE_RUST_PLUGIN:-}"; then + rust_plugin=', + "enabledPlugins": { + "rust-analyzer-lsp@claude-plugins-official": true + }' + fi + cat > "${repo_root}/.claude/settings.local.json" </dev/null || true)" + if [[ -n "$pid" ]]; then + kill "$pid" 2>/dev/null && echo "Killed process $pid on port $port" \ + || echo "Warning: Could not kill process $pid on port $port" + fi + done +} + +wm_shared_pre_remove() { + local repo_root=$1 + local env_file="${repo_root}/.env.local" + local db_conn wt_basename main_repo_root ee_repo ee_worktree_dir + + if [[ -f "$env_file" ]]; then + # shellcheck disable=SC1090 + source "$env_file" + fi + + if [[ -n "${WM_DB_NAME:-}" ]]; then + db_conn="postgres://postgres:changeme@127.0.0.1:5432" + if command -v psql >/dev/null 2>&1; then + psql "$db_conn/postgres" -c "DROP DATABASE IF EXISTS ${WM_DB_NAME} WITH (FORCE)" 2>/dev/null \ + && echo "Dropped database $WM_DB_NAME" \ + || echo "Warning: Could not drop database $WM_DB_NAME" + else + echo "psql not found, skipping database cleanup for $WM_DB_NAME" + fi + fi + + main_repo_root="$(wm_main_repo_root "$repo_root")" + wt_basename="$(basename "$repo_root")" + if ee_repo="$(wm_find_ee_repo "$repo_root" "$main_repo_root")"; then + ee_worktree_dir="${ee_repo}__worktrees/${wt_basename}" + if [[ -d "$ee_worktree_dir" ]]; then + git -C "$ee_repo" worktree remove "$ee_worktree_dir" --force 2>/dev/null \ + && echo "Removed EE worktree at $ee_worktree_dir" \ + || echo "Warning: Could not remove EE worktree at $ee_worktree_dir" + fi + fi + + tmux kill-session -t "cursor-${wt_basename}" 2>/dev/null || true +} diff --git a/scripts/worktree-env b/scripts/worktree-env index 5fd8490bc2..e45daba75b 100755 --- a/scripts/worktree-env +++ b/scripts/worktree-env @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/worktree-common.sh" + port_in_use() { lsof -nP -iTCP:"$1" -sTCP:LISTEN &>/dev/null } @@ -51,154 +53,4 @@ if [[ -n "${CARGO_FEATURES:-}" ]]; then fi echo "Created .env.local with ports: backend=$backend_port, frontend=$frontend_port" - -# --- Create per-worktree database --- -wt_basename=$(basename "$(pwd)") -db_name="windmill_${wt_basename//-/_}" -db_conn="postgres://postgres:changeme@127.0.0.1:5432" - -if command -v psql &>/dev/null; then - if psql "$db_conn/postgres" -tc "SELECT 1 FROM pg_database WHERE datname = '${db_name}'" 2>/dev/null | grep -q 1; then - echo "Database $db_name already exists" - else - if [[ "${WM_CLONE_DB:-}" == "1" || "${WM_CLONE_DB:-}" == "true" ]]; then - # Terminate active connections so CREATE DATABASE ... TEMPLATE works - psql "$db_conn/postgres" -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'windmill' AND pid <> pg_backend_pid();" 2>/dev/null || true - psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name} TEMPLATE windmill" 2>/dev/null \ - && echo "Created database $db_name (template: windmill)" \ - || echo "WARNING: Could not create database $db_name from template windmill" >&2 - else - psql "$db_conn/postgres" -c "CREATE DATABASE ${db_name}" 2>/dev/null \ - && echo "Created database $db_name" \ - || echo "WARNING: Could not create database $db_name (is PostgreSQL running?)" >&2 - fi - fi - db_url="${db_conn}/${db_name}?sslmode=disable" - # Run migrations against the new database - DATABASE_URL="$db_url" sqlx migrate run --source backend/migrations \ - && echo "Migrations applied to $db_name" \ - || echo "WARNING: Could not run migrations on $db_name" >&2 - # Copy license_key from the main windmill database to the new database - license_key=$(psql "$db_conn/windmill" -t -A -c "SELECT value FROM global_settings WHERE name = 'license_key'" 2>/dev/null || true) - if [[ -n "$license_key" ]]; then - psql "$db_url" -c "INSERT INTO global_settings (name, value) VALUES ('license_key', '${license_key}'::jsonb) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value" 2>/dev/null \ - && echo "Copied license_key to $db_name" \ - || echo "WARNING: Could not copy license_key to $db_name" >&2 - fi - # Use export so DATABASE_URL overrides the nix devshell value for child processes - cat >> .env.local <&2 -fi - -# --- Copy frontend/node_modules preserving symlinks --- -# cp -a preserves .bin/ symlinks that cp -r would dereference, breaking require() paths -main_repo_root="$(cd "$(git rev-parse --git-common-dir 2>/dev/null)/.." && pwd)" -if [[ -n "$main_repo_root" && -d "$main_repo_root/frontend/node_modules" ]]; then - cp -a "$main_repo_root/frontend/node_modules" frontend/ - echo "Copied frontend/node_modules (with symlinks preserved)" -fi - -# --- Install cli deps and generate client --- -if [[ -n "$main_repo_root" && -d "$main_repo_root/cli/node_modules" ]]; then - cp -a "$main_repo_root/cli/node_modules" cli/ - echo "Copied cli/node_modules (with symlinks preserved)" -fi -(cd cli && npm install && npm run gen-client) \ - && echo "CLI deps installed and client generated" \ - || echo "WARNING: CLI setup failed" >&2 - -# --- Allow direnv so the nix devshell activates in pane commands --- -if command -v direnv &>/dev/null && [ -f .envrc ]; then - direnv allow - echo "direnv allowed" -fi - -# --- Trust worktree directory in Claude Code --- -claude_json="$HOME/.claude.json" -if [ -f "$claude_json" ]; then - wt_path="$(pwd)" - python3 -c " -import json, sys -path = '$wt_path' -with open('$claude_json', 'r') as f: - data = json.load(f) -projects = data.setdefault('projects', {}) -proj = projects.setdefault(path, {}) -proj['hasTrustDialogAccepted'] = True -proj['hasCompletedProjectOnboarding'] = True -with open('$claude_json', 'w') as f: - json.dump(data, f, indent=2) -" && echo "Added $wt_path to Claude Code trusted directories" \ - || echo "Warning: Could not update Claude Code trusted directories" -fi - -# --- Create matching windmill-ee-private worktree --- -# Find ee repo: sibling to the main worktree (git toplevel of the main checkout), -# then try parent of cwd, then fall back to home -ee_repo="" -for candidate in \ - "${main_repo_root:+${main_repo_root}/../windmill-ee-private}" \ - "$(pwd)/../windmill-ee-private" \ - "${HOME}/windmill-ee-private" \ - "${HOME}/projects/windmill-ee-private"; do - if [ -n "$candidate" ] && [ -d "$candidate" ]; then - ee_repo="$(cd "$candidate" && pwd)" - break - fi -done -if [ -n "$ee_repo" ]; then - branch=$(git branch --show-current 2>/dev/null || true) - wt_basename=$(basename "$(pwd)") - ee_worktree_dir="${ee_repo}__worktrees/${wt_basename}" - - if [ -n "$branch" ] && [ ! -d "$ee_worktree_dir" ]; then - mkdir -p "$(dirname "$ee_worktree_dir")" - - # Fetch latest so we can check out remote branches - git -C "$ee_repo" fetch --quiet 2>/dev/null || true - - # Try: existing branch, then new branch from main - if git -C "$ee_repo" worktree add "$ee_worktree_dir" "$branch" 2>/dev/null; then - echo "Created EE worktree at $ee_worktree_dir (branch: $branch)" - elif git -C "$ee_repo" worktree add -b "$branch" "$ee_worktree_dir" main 2>/dev/null; then - echo "Created EE worktree at $ee_worktree_dir (new branch: $branch from main)" - else - echo "Warning: Could not create EE worktree for branch $branch" - fi - elif [ -d "$ee_worktree_dir" ]; then - echo "EE worktree already exists at $ee_worktree_dir" - fi - - # Point Claude Code additionalDirectories at the EE worktree - if [ -d "$ee_worktree_dir" ]; then - ee_rel=$(python3 -c "import os; print(os.path.relpath('$ee_worktree_dir', '$(pwd)'))" 2>/dev/null || echo "$ee_worktree_dir") - mkdir -p .claude - rust_plugin="" - if [[ "${USE_RUST_PLUGIN:-}" == "1" || "${USE_RUST_PLUGIN:-}" == "true" ]]; then - rust_plugin=', - "enabledPlugins": { - "rust-analyzer-lsp@claude-plugins-official": true - }' - fi - cat > .claude/settings.local.json < Date: Mon, 9 Mar 2026 13:11:16 +0000 Subject: [PATCH 38/57] warn about missing in nuget config and make description optional (#8281) Co-authored-by: Claude Opus 4.6 --- backend/windmill-types/src/scripts.rs | 19 ++++++++++--------- .../lib/components/InstanceSettings.svelte | 6 +++++- .../src/lib/components/instanceSettings.ts | 3 ++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index bcbe0ede33..28d3a1e449 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -105,15 +105,15 @@ impl ScriptLang { pub fn is_native(&self) -> bool { matches!( self, - ScriptLang::Bunnative | - ScriptLang::Nativets | - ScriptLang::Postgresql | - ScriptLang::Mysql | - ScriptLang::Graphql | - ScriptLang::Snowflake | - ScriptLang::Mssql | - ScriptLang::Bigquery | - ScriptLang::OracleDB + ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Postgresql + | ScriptLang::Mysql + | ScriptLang::Graphql + | ScriptLang::Snowflake + | ScriptLang::Mssql + | ScriptLang::Bigquery + | ScriptLang::OracleDB ) } @@ -459,6 +459,7 @@ pub struct NewScript { pub path: String, pub parent_hash: Option, pub summary: String, + #[serde(default)] pub description: String, pub content: String, pub schema: Option, diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 152b58dc96..64fdfad16d 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1038,7 +1038,11 @@ {oauths} warning={setting.key === 'base_url' && baseUrlIsFallback ? 'Auto-detected from browser — not yet saved' - : undefined} + : setting.key === 'nuget_config' && + $values['nuget_config'] && + !//.test($values['nuget_config']) + ? 'Missing in . Without it, default sources (like nuget.org) are merged with your custom sources, which is likely not what you want.' + : undefined} /> {/if} {#if quickSetup && category === 'Core' && setting.key === 'base_url'} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 80017582f5..29c730b736 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -464,7 +464,8 @@ export const settings: Record = { }, { label: 'Nuget Config', - description: 'Write a nuget.config file to set custom package sources and credentials', + description: + 'Write a nuget.config file to set custom package sources and credentials. Use inside to remove default sources and only use your custom ones', key: 'nuget_config', fieldType: 'codearea', codeAreaLang: 'xml', From 738f618a711d68bab123cc5c4bcc015dcf69ee73 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:13:23 +0100 Subject: [PATCH 39/57] fix webmux config (#8282) --- .webmux.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.webmux.yaml b/.webmux.yaml index 397f96f88f..b01d98b8d2 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -1,9 +1,9 @@ # Project display name in the dashboard -name: windmill +name: Windmill workspace: mainBranch: main - worktreeRoot: ../__worktrees + worktreeRoot: ../windmill__worktrees defaultAgent: claude startupEnvs: From 05d2d78f50e8da4e2c52f1d3bc4fdd53cd27dc0d Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:15:37 +0100 Subject: [PATCH 40/57] refactor: extract google ai logic to windmill-common and use native gemini api in chat proxy (#8115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: extract google ai logic to windmill-common and use native gemini api in chat proxy Co-Authored-By: Claude Sonnet 4.6 * fix: use x-goog-api-key header for google ai non-chat requests Co-Authored-By: Claude Sonnet 4.6 * fix: transform gemini models response to openai format and use correct auth header Co-Authored-By: Claude Sonnet 4.6 * fix: skip thought parts from gemini thinking models in sse stream Co-Authored-By: Claude Opus 4.5 * Revert "fix: skip thought parts from gemini thinking models in sse stream" This reverts commit dfa01d282c617a2b5f40fe002838083216d7b619. * fix: handle tool calls and sanitize schemas in gemini chat proxy Co-Authored-By: Claude Opus 4.5 * refactor: move Gemini→OpenAI response conversion to windmill-common Extract streaming and non-streaming Gemini response conversion into shared functions in ai_google so the API proxy and worker use the same logic instead of duplicating format translation. Co-Authored-By: Claude Opus 4.6 * fix: review fixes for google ai refactor - Remove duplicate parse_data_url from worker utils, use shared version from windmill_common::ai_google in both google_ai and anthropic providers - Improve error diagnostics in google.rs by including HTTP status code in error messages from Gemini API responses - Change GeminiToolCallEvent::into_extra_content to instance method to_extra_content using &self Co-Authored-By: Claude Opus 4.6 * refactor: deduplicate worker Gemini message conversion using pre-flight pattern Replace the worker's `convert_messages_to_gemini` and `convert_content_to_parts_with_s3` (~130 lines) with the existing pre-flight pattern: `prepare_messages_for_api` converts S3 objects to data URLs, then the shared `openai_messages_to_gemini` handles the rest. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Sonnet 4.6 Co-authored-by: hugocasa --- backend/Cargo.lock | 1 + backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/src/ai.rs | 46 +- backend/windmill-api/src/google.rs | 306 ++++++++ backend/windmill-api/src/lib.rs | 1 + backend/windmill-common/src/ai_google.rs | 726 ++++++++++++++++++ backend/windmill-common/src/lib.rs | 1 + .../src/ai/providers/anthropic.rs | 6 +- .../src/ai/providers/google_ai.rs | 492 +----------- backend/windmill-worker/src/ai/sse.rs | 230 ++---- backend/windmill-worker/src/ai/utils.rs | 13 - 11 files changed, 1175 insertions(+), 648 deletions(-) create mode 100644 backend/windmill-api/src/google.rs create mode 100644 backend/windmill-common/src/ai_google.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1ce936a245..3f0ee056f9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15849,6 +15849,7 @@ dependencies = [ "dashmap 6.1.0", "datafusion", "ed25519-dalek", + "eventsource-stream", "flate2", "futures", "git-version", diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index de8935d097..14069590eb 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -174,6 +174,7 @@ aws-sdk-bedrock = { workspace = true, optional = true } aws-sdk-bedrockruntime = { workspace = true, optional = true } aws-smithy-types = { workspace = true, optional = true } async-trait.workspace = true +eventsource-stream.workspace = true windmill-jseval.workspace = true tar.workspace = true flate2.workspace = true diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 730df2a875..92ff49f4a4 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -29,7 +29,7 @@ const AI_TIMEOUT_MAX_SECS: u64 = 86400; // 24 hours const AI_TIMEOUT_DEFAULT_SECS: u64 = 3600; // 1 hour const HTTP_POOL_MAX_IDLE_PER_HOST: usize = 10; const HTTP_POOL_IDLE_TIMEOUT_SECS: u64 = 90; -const KEEPALIVE_INTERVAL_SECS: u64 = 15; +pub(crate) const KEEPALIVE_INTERVAL_SECS: u64 = 15; lazy_static::lazy_static! { /// AI request timeout in seconds. @@ -87,7 +87,7 @@ lazy_static::lazy_static! { } }; - static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() + pub(crate) static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(*AI_TIMEOUT_SECS)) .pool_max_idle_per_host(HTTP_POOL_MAX_IDLE_PER_HOST) .pool_idle_timeout(Some(std::time::Duration::from_secs(HTTP_POOL_IDLE_TIMEOUT_SECS))) @@ -378,12 +378,7 @@ impl AIRequestConfig { let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let is_google_ai = matches!(provider, AIProvider::GoogleAI); - // GoogleAI uses OpenAI-compatible endpoint in the proxy (for the chat), but not for the ai agent - let base_url = if is_google_ai { - format!("{}/openai", base_url) - } else { - base_url.to_string() - }; + let base_url = base_url.to_string(); let base_url = base_url.as_str(); // Build URL based on provider @@ -428,6 +423,9 @@ impl AIRequestConfig { if let Some(api_key) = self.api_key { if is_azure { request = request.header("api-key", api_key.clone()) + } else if is_google_ai { + // Native Gemini API uses x-goog-api-key, not Authorization: Bearer + request = request.header("x-goog-api-key", api_key.clone()) } else { request = request.header("authorization", format!("Bearer {}", api_key.clone())) } @@ -611,7 +609,7 @@ fn is_sse_response(headers: &HeaderMap) -> bool { .unwrap_or(false) } -fn inject_keepalives( +pub(crate) fn inject_keepalives( upstream: S, interval: Duration, ) -> impl futures::Stream> @@ -830,6 +828,36 @@ async fn proxy( ai_path = chat_path; } + // Handle GoogleAI (Gemini) using the native Gemini API + if matches!(provider, AIProvider::GoogleAI) { + let api_key = request_config.api_key.as_deref().unwrap_or(""); + let base_url = request_config.base_url.trim_end_matches('/'); + + let mut tx = db.begin().await?; + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()), + ) + .await?; + tx.commit().await?; + + return match ai_path.as_str() { + "chat/completions" => { + crate::google::handle_google_ai_chat(&body, api_key, base_url).await + } + "models" => crate::google::handle_google_ai_models(api_key, base_url).await, + _ => Err(Error::BadRequest(format!( + "Unsupported Google AI path: {}", + ai_path + ))), + }; + } + // Handle Bedrock-specific logic when the feature is enabled #[cfg(feature = "bedrock")] { diff --git a/backend/windmill-api/src/google.rs b/backend/windmill-api/src/google.rs new file mode 100644 index 0000000000..bc32c20ba4 --- /dev/null +++ b/backend/windmill-api/src/google.rs @@ -0,0 +1,306 @@ +//! Google AI (Gemini API) handler for the AI chat proxy. +//! +//! Handles POST `chat/completions` requests using the native Gemini API, +//! converting from/to OpenAI format so the existing frontend parsers continue to work. +//! +//! Used by `windmill-api/src/ai.rs` when the provider is `GoogleAI`. +//! Shared conversion logic lives in `windmill_common::ai_google`. + +use axum::body::Body; +use bytes::Bytes; +use eventsource_stream::Eventsource; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; +use windmill_common::{ + ai_google::{ + gemini_event_to_openai_sse_chunks, gemini_response_to_openai, openai_messages_to_gemini, + parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, + GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiTextRequest, GeminiTool, + }, + ai_types::OpenAIMessage, + error::{Error, Result}, +}; + +use crate::ai::{inject_keepalives, HTTP_CLIENT, KEEPALIVE_INTERVAL_SECS}; + +// ============================================================================ +// Request type (OpenAI format received from the frontend) +// ============================================================================ + +#[derive(Deserialize, Debug)] +struct ChatRequest { + model: String, + messages: Vec, + #[serde(default)] + stream: bool, + #[serde(default)] + temperature: Option, + #[serde(default)] + max_tokens: Option, + #[serde(default)] + tools: Option>, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestTool { + function: ChatRequestToolFunction, +} + +#[derive(Deserialize, Debug)] +struct ChatRequestToolFunction { + name: String, + #[serde(default)] + description: Option, + #[serde(default)] + parameters: Option, +} + +// ============================================================================ +// Public handler +// ============================================================================ + +/// Handle a `chat/completions` POST request using the native Gemini API. +/// +/// Converts the incoming OpenAI-format body to a `GeminiTextRequest`, sends it +/// to the appropriate Gemini endpoint, and converts the response back to the +/// OpenAI SSE or JSON format that the frontend expects. +pub async fn handle_google_ai_chat( + body: &Bytes, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let request: ChatRequest = serde_json::from_slice(body) + .map_err(|e| Error::BadRequest(format!("Failed to parse request body: {}", e)))?; + + let (contents, system_instruction) = openai_messages_to_gemini(&request.messages); + + let generation_config = + if request.temperature.is_some() || request.max_tokens.is_some() { + Some(GeminiGenerationConfig { + temperature: request.temperature, + max_output_tokens: request.max_tokens, + response_mime_type: None, + response_schema: None, + }) + } else { + None + }; + + let gemini_tools = request.tools.as_ref().map(|tools| { + let declarations: Vec = tools + .iter() + .map(|t| { + let mut params = t.function.parameters.clone().unwrap_or(json!({})); + sanitize_schema_for_google(&mut params); + GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params, + } + }) + .collect(); + vec![GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }] + }); + + let gemini_request = GeminiTextRequest { + contents, + tools: gemini_tools, + tool_config: None, + system_instruction, + generation_config, + }; + + let request_body = serde_json::to_string(&gemini_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize Gemini request: {}", e)))?; + + let base_url = base_url.trim_end_matches('/'); + + if request.stream { + handle_streaming(&request.model, request_body, api_key, base_url).await + } else { + handle_non_streaming(&request.model, request_body, api_key, base_url).await + } +} + +// ============================================================================ +// Streaming path +// ============================================================================ + +async fn handle_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let model_str = model.to_string(); + + let gemini_sse_stream = response.bytes_stream().eventsource(); + let openai_sse_stream = async_stream::stream! { + tokio::pin!(gemini_sse_stream); + let mut tool_call_index: usize = 0; + while let Some(event) = gemini_sse_stream.next().await { + match event { + Ok(event) => match parse_gemini_sse_event(&event.data) { + Ok(Some(parsed)) => { + for chunk in gemini_event_to_openai_sse_chunks( + &parsed, &id, &model_str, &mut tool_call_index, + ) { + yield Ok::(Bytes::from(chunk)); + } + } + Ok(None) => {} + Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + }, + Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), + } + } + yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "text/event-stream".parse().unwrap()); + headers.insert("cache-control", "no-cache".parse().unwrap()); + headers.insert("connection", "keep-alive".parse().unwrap()); + + Ok(( + http::StatusCode::OK, + headers, + Body::from_stream(inject_keepalives( + Box::pin(openai_sse_stream), + std::time::Duration::from_secs(KEEPALIVE_INTERVAL_SECS), + )), + )) +} + +// ============================================================================ +// Model listing +// ============================================================================ + +/// List available Gemini models and convert to OpenAI format. +/// +/// Gemini returns `{ models: [{ name: "models/gemini-2.5-flash", displayName, ... }] }`. +/// The frontend expects OpenAI format `{ data: [{ id: "models/gemini-2.5-flash", ... }] }`. +pub async fn handle_google_ai_models( + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + #[derive(Deserialize)] + struct GeminiModel { + name: String, + #[serde(rename = "displayName", default)] + display_name: String, + } + + #[derive(Deserialize)] + struct GeminiModelsResponse { + #[serde(default)] + models: Vec, + } + + let endpoint = format!("{}/models", base_url.trim_end_matches('/')); + let response = HTTP_CLIENT + .get(&endpoint) + .header("x-goog-api-key", api_key) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to fetch Gemini models: {}", e)))?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { + Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) + })?; + + let data: Vec = gemini_resp + .models + .into_iter() + .map(|m| { + json!({ + "id": m.name, + "object": "model", + "display_name": m.display_name, + }) + }) + .collect(); + + let body_bytes = serde_json::to_vec(&json!({ "data": data })) + .map_err(|e| Error::internal_err(format!("Failed to serialize models: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} + +// ============================================================================ +// Non-streaming path +// ============================================================================ + +async fn handle_non_streaming( + model: &str, + request_body: String, + api_key: &str, + base_url: &str, +) -> Result<(http::StatusCode, http::HeaderMap, Body)> { + let endpoint = format!("{}/models/{}:generateContent", base_url, model); + + let response = HTTP_CLIENT + .post(&endpoint) + .header("content-type", "application/json") + .header("x-goog-api-key", api_key) + .body(request_body) + .send() + .await + .map_err(|e| { + Error::internal_err(format!("Failed to send request to Gemini API: {}", e)) + })?; + + if let Err(e) = response.error_for_status_ref() { + let status = e.status().map(|s| s.to_string()).unwrap_or_default(); + let body = response.text().await.unwrap_or_default(); + return Err(Error::AIError(format!("{}: {}", status, body))); + } + + let body = response.bytes().await.map_err(|e| { + Error::internal_err(format!("Failed to read Gemini response body: {}", e)) + })?; + + let parsed = parse_gemini_response(&body)?; + let openai_response = gemini_response_to_openai(&parsed, model); + + let body_bytes = serde_json::to_vec(&openai_response) + .map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?; + + let mut headers = http::HeaderMap::new(); + headers.insert("content-type", "application/json".parse().unwrap()); + + Ok((http::StatusCode::OK, headers, Body::from(body_bytes))) +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 073c1a8aa1..a25a431dba 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -64,6 +64,7 @@ use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; mod ai; +mod google; mod apps; pub mod args; mod audit; diff --git a/backend/windmill-common/src/ai_google.rs b/backend/windmill-common/src/ai_google.rs new file mode 100644 index 0000000000..ccf34685e5 --- /dev/null +++ b/backend/windmill-common/src/ai_google.rs @@ -0,0 +1,726 @@ +//! Shared Google AI (Gemini API) types and conversion utilities. +//! +//! This module provides: +//! - Gemini request/response types +//! - OpenAI → Gemini message conversion +//! - Gemini SSE event parsing +//! +//! Used by both windmill-api (chat proxy) and windmill-worker (AI agent). + +use serde::{Deserialize, Serialize}; + +use crate::ai_types::{ContentPart, ExtraContent, GoogleExtraContent, OpenAIContent, OpenAIMessage, ToolDef, UrlCitation}; +use crate::error::Error; + +// ============================================================================ +// Request / Content Types +// ============================================================================ + +/// Inline data for binary content (images). +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +/// A part of content — text, inline data, function call, or function response. +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum GeminiPart { + Text { + text: String, + }, + InlineData { + #[serde(rename = "inlineData")] + inline_data: GeminiInlineData, + }, + FunctionCall { + #[serde(rename = "functionCall")] + function_call: GeminiFunctionCall, + /// Thought signature for Gemini 3+ models — required when replaying function calls. + #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] + thought_signature: Option, + }, + FunctionResponse { + #[serde(rename = "functionResponse")] + function_response: GeminiFunctionResponse, + }, +} + +/// A function call from the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// A function response sent back to the model. +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +/// Content message with an optional role and a list of parts. +#[derive(Serialize, Clone, Debug)] +pub struct GeminiContentMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + pub parts: Vec, +} + +/// Main request body for `generateContent` / `streamGenerateContent`. +#[derive(Serialize)] +pub struct GeminiTextRequest { + pub contents: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] + pub tool_config: Option, + #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] + pub system_instruction: Option, + #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] + pub generation_config: Option, +} + +/// Tool definition — function declarations and/or Google Search grounding. +#[derive(Serialize)] +pub struct GeminiTool { + #[serde(rename = "functionDeclarations", skip_serializing_if = "Option::is_none")] + pub function_declarations: Option>, + #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] + pub google_search: Option, +} + +/// A single function declaration. +/// +/// `parameters` holds a pre-serialized (and, for the worker, pre-sanitized) JSON Schema. +#[derive(Serialize)] +pub struct GeminiFunctionDeclaration { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: serde_json::Value, +} + +/// Tool configuration controlling when and how functions are called. +#[derive(Serialize)] +pub struct GeminiToolConfig { + #[serde(rename = "functionCallingConfig")] + pub function_calling_config: GeminiFunctionCallingConfig, +} + +/// Function calling mode and optional allow-list. +#[derive(Serialize)] +pub struct GeminiFunctionCallingConfig { + pub mode: String, + #[serde(rename = "allowedFunctionNames", skip_serializing_if = "Option::is_none")] + pub allowed_function_names: Option>, +} + +/// Generation parameters (temperature, token limits, structured output). +#[derive(Serialize)] +pub struct GeminiGenerationConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] + pub response_mime_type: Option, + #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] + pub response_schema: Option, +} + +// ============================================================================ +// Image Generation Types +// ============================================================================ + +/// Request body for Imagen / Gemini image generation. +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +/// Content wrapper used in `generateContent` image requests. +#[derive(Serialize)] +pub struct GeminiImageContent { + pub parts: Vec, +} + +/// Prompt wrapper for Imagen `predict` endpoint. +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +/// Top-level response from Gemini/Imagen image generation. +#[derive(Deserialize)] +pub struct GeminiImageResponse { + pub candidates: Option>, + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidate { + pub content: GeminiImageCandidateContent, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidateContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiImageCandidatePart { + #[serde(rename = "inlineData")] + pub inline_data: Option, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, +} + +// ============================================================================ +// SSE Response Types +// ============================================================================ + +/// One part inside a streaming candidate — text, function call, or thought signature. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEPart { + #[serde(default)] + pub text: Option, + #[serde(rename = "functionCall")] + pub function_call: Option, + /// Thought signature for Gemini 3+ models. + #[serde(rename = "thoughtSignature")] + pub thought_signature: Option, +} + +/// Function call contained in a streaming part. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +/// Content block inside a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEContent { + pub parts: Option>, +} + +/// Web source from a Gemini grounding chunk. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunkWeb { + pub uri: String, + #[serde(default)] + pub title: Option, +} + +/// One grounding chunk (search result) from Gemini web search. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingChunk { + pub web: Option, +} + +/// Grounding metadata attached to a streaming candidate. +#[derive(Deserialize, Debug)] +pub struct GeminiGroundingMetadata { + #[serde(rename = "groundingChunks", default)] + pub grounding_chunks: Vec, + #[serde(rename = "webSearchQueries", default)] + pub web_search_queries: Vec, +} + +/// One candidate inside a streaming Gemini response. +#[derive(Deserialize, Debug)] +pub struct GeminiSSECandidate { + pub content: Option, + #[serde(rename = "finishReason")] + pub finish_reason: Option, + #[serde(rename = "groundingMetadata")] + pub grounding_metadata: Option, +} + +/// Token usage from the `usageMetadata` field of a Gemini SSE event. +#[derive(Deserialize, Debug, Clone)] +pub struct GeminiUsageMetadata { + #[serde(rename = "promptTokenCount", default)] + pub prompt_token_count: Option, + #[serde(rename = "candidatesTokenCount", default)] + pub candidates_token_count: Option, + #[serde(rename = "totalTokenCount", default)] + pub total_token_count: Option, +} + +/// Top-level structure of one Gemini SSE event. +#[derive(Deserialize, Debug)] +pub struct GeminiSSEEvent { + pub candidates: Option>, + #[serde(rename = "usageMetadata")] + pub usage_metadata: Option, +} + +// ============================================================================ +// Parsed Event Result +// ============================================================================ + +/// A single function call extracted from a Gemini SSE event. +#[derive(Debug)] +pub struct GeminiToolCallEvent { + pub name: String, + pub args: serde_json::Value, + pub thought_signature: Option, +} + +impl GeminiToolCallEvent { + /// Convert the thought signature (if present) into an [`ExtraContent`]. + pub fn to_extra_content(&self) -> Option { + self.thought_signature.as_ref().map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig.clone()) }), + }) + } +} + +/// Structured result of parsing a Gemini response (streaming SSE event or non-streaming body). +#[derive(Debug, Default)] +pub struct GeminiParsedEvent { + pub text: Option, + pub tool_calls: Vec, + pub annotations: Vec, + pub used_websearch: bool, + pub usage: Option, + pub finish_reason: Option, +} + +// ============================================================================ +// Helper Functions +// ============================================================================ + +/// Parse a data URL into `(mime_type, base64_data)`. +/// +/// Expected format: `data:;base64,`. +pub fn parse_data_url(url: &str) -> Option<(String, String)> { + let rest = url.strip_prefix("data:")?; + let (header, data) = rest.split_once(',')?; + let media_type = header.strip_suffix(";base64")?; + Some((media_type.to_string(), data.to_string())) +} + +/// Find the function name associated with a `tool_call_id` by scanning prior messages. +pub fn find_gemini_function_name(messages: &[OpenAIMessage], tool_call_id: &str) -> String { + messages + .iter() + .filter_map(|msg| msg.tool_calls.as_ref()) + .flatten() + .find(|tc| tc.id == tool_call_id) + .map(|tc| tc.function.name.clone()) + .unwrap_or_else(|| "unknown_function".to_string()) +} + +/// Convert an [`OpenAIContent`] value to a list of [`GeminiPart`]s. +/// +/// Handles text and `image_url` (data URLs). `S3Object` variants are skipped here; +/// the worker handles them by downloading and injecting inline data beforehand. +pub fn convert_content_to_gemini_parts(content: &OpenAIContent) -> Vec { + match content { + OpenAIContent::Text(text) if !text.is_empty() => { + vec![GeminiPart::Text { text: text.clone() }] + } + OpenAIContent::Text(_) => vec![], + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } if !text.is_empty() => { + Some(GeminiPart::Text { text: text.clone() }) + } + ContentPart::ImageUrl { image_url } => { + parse_data_url(&image_url.url).map(|(mime_type, data)| { + GeminiPart::InlineData { + inline_data: GeminiInlineData { mime_type, data }, + } + }) + } + // S3Objects are handled by the worker + _ => None, + }) + .collect(), + } +} + +/// Convert OpenAI-format messages to Gemini `contents` and an optional `systemInstruction`. +/// +/// Returns `(contents, system_instruction)`. +/// +/// `S3Object` images in content parts are skipped (the worker pre-converts them). +/// Tool call history is preserved correctly for multi-turn agent conversations. +pub fn openai_messages_to_gemini( + messages: &[OpenAIMessage], +) -> (Vec, Option) { + let mut contents: Vec = Vec::new(); + let mut system_instruction: Option = None; + + for msg in messages { + match msg.role.as_str() { + "system" => { + if let Some(content) = &msg.content { + let parts = convert_content_to_gemini_parts(content); + if !parts.is_empty() { + system_instruction = + Some(GeminiContentMessage { role: None, parts }); + } + } + } + "tool" => { + if let (Some(tool_call_id), Some(content)) = + (&msg.tool_call_id, &msg.content) + { + let func_name = find_gemini_function_name(messages, tool_call_id); + let response_text = match content { + OpenAIContent::Text(text) => text.clone(), + OpenAIContent::Parts(parts) => parts + .iter() + .filter_map(|p| { + if let ContentPart::Text { text } = p { + Some(text.as_str()) + } else { + None + } + }) + .collect::>() + .join(" "), + }; + contents.push(GeminiContentMessage { + role: Some("user".to_string()), + parts: vec![GeminiPart::FunctionResponse { + function_response: GeminiFunctionResponse { + name: func_name, + response: serde_json::json!({ "result": response_text }), + }, + }], + }); + } + } + role => { + let gemini_role = if role == "assistant" { "model" } else { "user" }; + let mut parts: Vec = Vec::new(); + + if let Some(content) = &msg.content { + parts.extend(convert_content_to_gemini_parts(content)); + } + + if let Some(tool_calls) = &msg.tool_calls { + for tc in tool_calls { + let args: serde_json::Value = + serde_json::from_str(&tc.function.arguments).unwrap_or_default(); + let thought_signature = tc + .extra_content + .as_ref() + .and_then(|ec| ec.google.as_ref()) + .and_then(|g| g.thought_signature.clone()); + parts.push(GeminiPart::FunctionCall { + function_call: GeminiFunctionCall { + name: tc.function.name.clone(), + args, + }, + thought_signature, + }); + } + } + + if !parts.is_empty() { + contents.push(GeminiContentMessage { + role: Some(gemini_role.to_string()), + parts, + }); + } + } + } + } + + (contents, system_instruction) +} + +/// Convert OpenAI tool definitions to Gemini format. +/// +/// `tool_params` must be pre-serialized (and, for the worker, pre-sanitized for Google) +/// JSON schema values, one per entry in `tools` in the same order. +pub fn openai_tools_to_gemini( + tools: &[ToolDef], + tool_params: &[serde_json::Value], + has_websearch: bool, +) -> Option> { + let mut gemini_tools: Vec = Vec::new(); + + let declarations: Vec = tools + .iter() + .zip(tool_params.iter()) + .map(|(t, params)| GeminiFunctionDeclaration { + name: t.function.name.clone(), + description: t.function.description.clone(), + parameters: params.clone(), + }) + .collect(); + + if !declarations.is_empty() { + gemini_tools.push(GeminiTool { + function_declarations: Some(declarations), + google_search: None, + }); + } + + if has_websearch { + gemini_tools.push(GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }); + } + + if gemini_tools.is_empty() { + None + } else { + Some(gemini_tools) + } +} + +/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. +/// +/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). +/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { + if data.is_empty() || data == "[DONE]" { + return Ok(None); + } + + let event: GeminiSSEEvent = match serde_json::from_str(data) { + Ok(e) => e, + Err(e) => { + tracing::error!("Failed to parse Gemini SSE event {}: {}", data, e); + return Ok(None); + } + }; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + let Some(candidates) = event.candidates else { + return Ok(Some(parsed)); + }; + + extract_candidates_into(&candidates, &mut parsed); + + Ok(Some(parsed)) +} + +/// Parse a non-streaming Gemini `generateContent` response body. +pub fn parse_gemini_response(data: &[u8]) -> Result { + let event: GeminiSSEEvent = serde_json::from_slice(data) + .map_err(|e| Error::internal_err(format!("Failed to parse Gemini response: {}", e)))?; + + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; + + if let Some(candidates) = event.candidates { + extract_candidates_into(&candidates, &mut parsed); + } + + Ok(parsed) +} + +// ============================================================================ +// Gemini → OpenAI Format Conversion +// ============================================================================ + +/// Convert a `GeminiParsedEvent` from a non-streaming response to an OpenAI chat completion JSON. +pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> serde_json::Value { + let content = parsed.text.as_deref().unwrap_or_default(); + + let tool_calls: Vec = parsed + .tool_calls + .iter() + .enumerate() + .map(|(i, tc)| { + serde_json::json!({ + "index": i, + "id": format!("call_{}", uuid::Uuid::new_v4().simple()), + "type": "function", + "function": { + "name": tc.name, + "arguments": serde_json::to_string(&tc.args).unwrap_or_default() + } + }) + }) + .collect(); + + let finish_reason = parsed + .finish_reason + .as_deref() + .map(|r| r.to_lowercase()) + .unwrap_or_else(|| "stop".to_string()); + + let usage = parsed.usage.as_ref().map(|u| { + serde_json::json!({ + "prompt_tokens": u.prompt_token_count.unwrap_or(0), + "completion_tokens": u.candidates_token_count.unwrap_or(0), + "total_tokens": u.total_token_count.unwrap_or(0), + }) + }); + + let mut message = serde_json::json!({ + "role": "assistant", + "content": content, + }); + if !tool_calls.is_empty() { + message["tool_calls"] = serde_json::json!(tool_calls); + } + + serde_json::json!({ + "id": format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()), + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": message, + "finish_reason": finish_reason, + }], + "usage": usage, + }) +} + +/// Convert a `GeminiParsedEvent` from a streaming SSE event into OpenAI-format SSE lines. +/// +/// Returns the serialized `"data: {...}\n\n"` lines ready to be written to the response stream. +/// `tool_call_index` is mutated to track the running index across multiple SSE events. +pub fn gemini_event_to_openai_sse_chunks( + parsed: &GeminiParsedEvent, + id: &str, + model: &str, + tool_call_index: &mut usize, +) -> Vec { + let mut chunks = Vec::new(); + + if let Some(text) = &parsed.text { + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { "content": text }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + } + + for tc in &parsed.tool_calls { + let args_str = serde_json::to_string(&tc.args).unwrap_or_default(); + let call_id = format!("call_{}", uuid::Uuid::new_v4().simple()); + let chunk = serde_json::json!({ + "id": id, + "object": "chat.completion.chunk", + "model": model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": *tool_call_index, + "id": call_id, + "type": "function", + "function": { + "name": tc.name, + "arguments": args_str, + } + }] + }, + "finish_reason": null, + }] + }); + chunks.push(format!("data: {}\n\n", chunk)); + *tool_call_index += 1; + } + + chunks +} + +/// Recursively remove JSON Schema fields unsupported by the Gemini API. +pub fn sanitize_schema_for_google(value: &mut serde_json::Value) { + const UNSUPPORTED: &[&str] = &[ + "additionalProperties", + "strict", + "$schema", + "default", + "exclusiveMinimum", + "exclusiveMaximum", + "const", + "multipleOf", + ]; + + if let Some(obj) = value.as_object_mut() { + for field in UNSUPPORTED { + obj.remove(*field); + } + for v in obj.values_mut() { + sanitize_schema_for_google(v); + } + } else if let Some(arr) = value.as_array_mut() { + for v in arr.iter_mut() { + sanitize_schema_for_google(v); + } + } +} + +// ============================================================================ +// Internal Helpers +// ============================================================================ + +fn extract_candidates_into(candidates: &[GeminiSSECandidate], parsed: &mut GeminiParsedEvent) { + for candidate in candidates { + if let Some(content) = &candidate.content { + if let Some(parts) = &content.parts { + for part in parts { + if let Some(text) = &part.text { + if !text.is_empty() { + match parsed.text.as_mut() { + Some(existing) => existing.push_str(text), + None => parsed.text = Some(text.clone()), + } + } + } + + if let Some(function_call) = &part.function_call { + parsed.tool_calls.push(GeminiToolCallEvent { + name: function_call.name.clone(), + args: function_call.args.clone(), + thought_signature: part.thought_signature.clone(), + }); + } + } + } + } + + if candidate.finish_reason.is_some() { + parsed.finish_reason = candidate.finish_reason.clone(); + } + + if let Some(grounding) = &candidate.grounding_metadata { + if !grounding.web_search_queries.is_empty() || !grounding.grounding_chunks.is_empty() { + parsed.used_websearch = true; + } + for chunk in &grounding.grounding_chunks { + if let Some(web) = &chunk.web { + parsed.annotations.push(UrlCitation { + start_index: 0, + end_index: 0, + url: web.uri.clone(), + title: web.title.clone(), + }); + } + } + } + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index c4e0a47cd8..85f5563419 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -29,6 +29,7 @@ use sqlx::{Acquire, Postgres}; pub mod agent_workers; #[cfg(feature = "bedrock")] pub mod ai_bedrock; +pub mod ai_google; pub mod ai_providers; pub mod ai_types; pub mod apps; diff --git a/backend/windmill-worker/src/ai/providers/anthropic.rs b/backend/windmill-worker/src/ai/providers/anthropic.rs index e42e1dc49b..f9f5edf452 100644 --- a/backend/windmill-worker/src/ai/providers/anthropic.rs +++ b/backend/windmill-worker/src/ai/providers/anthropic.rs @@ -1,14 +1,16 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::parse_data_url, ai_providers::AIProvider, client::AuthedClient, error::Error, +}; use crate::ai::{ image_handler::prepare_messages_for_api, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{AnthropicSSEParser, SSEParser}, types::*, - utils::{extract_text_content, parse_data_url, should_use_structured_output_tool}, + utils::{extract_text_content, should_use_structured_output_tool}, }; /// Anthropic API version for standard API diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index 62e6afff75..e2030f3269 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -1,215 +1,21 @@ use async_trait::async_trait; -use serde::{Deserialize, Serialize}; -use windmill_common::{client::AuthedClient, error::Error}; +use windmill_common::{ + ai_google::{ + openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig, + GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, + GeminiPredictContent, GeminiTextRequest, GeminiTool, + }, + client::AuthedClient, + error::Error, +}; use crate::ai::{ - image_handler::download_and_encode_s3_image, + image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{GeminiSSEParser, SSEParser}, types::*, - utils::parse_data_url, }; -// ============================================================================ -// Gemini API Types - Shared between text and image -// ============================================================================ - -/// Inline data for binary content (images) -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiInlineData { - #[serde(rename = "mimeType")] - pub mime_type: String, - pub data: String, -} - -/// A part of content - can be text, inline data, function call, or function response -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -pub enum GeminiPart { - Text { - text: String, - }, - InlineData { - #[serde(rename = "inlineData")] - inline_data: GeminiInlineData, - }, - FunctionCall { - #[serde(rename = "functionCall")] - function_call: GeminiFunctionCall, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature", skip_serializing_if = "Option::is_none")] - thought_signature: Option, - }, - FunctionResponse { - #[serde(rename = "functionResponse")] - function_response: GeminiFunctionResponse, - }, -} - -/// A function call from the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// A function response to send back to the model -#[derive(Serialize, Deserialize, Clone, Debug)] -pub struct GeminiFunctionResponse { - pub name: String, - pub response: serde_json::Value, -} - -// ============================================================================ -// Gemini Text API Request Types -// ============================================================================ - -/// Main request structure for Gemini generateContent -#[derive(Serialize)] -pub struct GeminiTextRequest { - pub contents: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub tools: Option>, - #[serde(rename = "toolConfig", skip_serializing_if = "Option::is_none")] - pub tool_config: Option, - #[serde(rename = "systemInstruction", skip_serializing_if = "Option::is_none")] - pub system_instruction: Option, - #[serde(rename = "generationConfig", skip_serializing_if = "Option::is_none")] - pub generation_config: Option, -} - -/// Content message with role and parts -#[derive(Serialize)] -pub struct GeminiContentMessage { - #[serde(skip_serializing_if = "Option::is_none")] - pub role: Option, - pub parts: Vec, -} - -/// Tool definition - either function declarations or Google Search -#[derive(Serialize)] -pub struct GeminiTool { - #[serde( - rename = "functionDeclarations", - skip_serializing_if = "Option::is_none" - )] - pub function_declarations: Option>, - #[serde(rename = "googleSearch", skip_serializing_if = "Option::is_none")] - pub google_search: Option, -} - -/// Function declaration for tool use -#[derive(Serialize)] -pub struct GeminiFunctionDeclaration { - pub name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - pub parameters: OpenAPISchema, -} - -/// Tool configuration for controlling function calling behavior -#[derive(Serialize)] -pub struct GeminiToolConfig { - #[serde(rename = "functionCallingConfig")] - pub function_calling_config: GeminiFunctionCallingConfig, -} - -/// Function calling configuration -#[derive(Serialize)] -pub struct GeminiFunctionCallingConfig { - pub mode: String, - #[serde( - rename = "allowedFunctionNames", - skip_serializing_if = "Option::is_none" - )] - pub allowed_function_names: Option>, -} - -/// Generation configuration for output format -#[derive(Serialize)] -pub struct GeminiGenerationConfig { - #[serde(skip_serializing_if = "Option::is_none")] - pub temperature: Option, - #[serde(rename = "maxOutputTokens", skip_serializing_if = "Option::is_none")] - pub max_output_tokens: Option, - #[serde(rename = "responseMimeType", skip_serializing_if = "Option::is_none")] - pub response_mime_type: Option, - #[serde(rename = "responseSchema", skip_serializing_if = "Option::is_none")] - pub response_schema: Option, -} - -// ============================================================================ -// Gemini API Response Types -// ============================================================================ - -/// Grounding metadata from Google Search -#[derive(Deserialize)] -#[allow(dead_code)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "webSearchQueries")] - pub web_search_queries: Option>, - #[serde(rename = "groundingChunks")] - pub grounding_chunks: Option>, -} - -// ============================================================================ -// Gemini Image API Types (for Imagen models) -// ============================================================================ - -/// Request for image generation (Imagen models) -#[derive(Serialize)] -pub struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - pub contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub instances: Option>, -} - -/// Content for image generation -#[derive(Serialize)] -pub struct GeminiImageContent { - pub parts: Vec, -} - -/// Content for Imagen predict endpoint -#[derive(Serialize)] -pub struct GeminiPredictContent { - pub prompt: String, -} - -/// Response for image generation -#[derive(Deserialize)] -pub struct GeminiImageResponse { - pub candidates: Option>, - pub predictions: Option>, -} - -/// Image candidate from generateContent -#[derive(Deserialize)] -pub struct GeminiImageCandidate { - pub content: GeminiImageCandidateContent, -} - -/// Content in image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidateContent { - pub parts: Vec, -} - -/// Part of image candidate -#[derive(Deserialize)] -pub struct GeminiImageCandidatePart { - #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] - pub inline_data: Option, -} - -/// Prediction candidate from Imagen -#[derive(Deserialize)] -pub struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - pub bytes_base64_encoded: String, -} - // ============================================================================ // Query Builder Implementation // ============================================================================ @@ -221,34 +27,24 @@ impl GoogleAIQueryBuilder { Self } - /// Build a text request using the native Gemini API format async fn build_text_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, ) -> Result { - // Convert messages to Gemini format - let contents = self - .convert_messages_to_gemini(args.messages, client, workspace_id) - .await?; + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; + let (contents, system_instruction) = openai_messages_to_gemini(&prepared_messages); - // Build tools array let tools = self.convert_tools_to_gemini(args.tools, args.has_websearch); - // Build generation config let generation_config = self.build_generation_config(args); - // Build system instruction from system_prompt - let system_instruction = args.system_prompt.map(|s| GeminiContentMessage { - role: None, - parts: vec![GeminiPart::Text { text: s.to_string() }], - }); - let request = GeminiTextRequest { contents, tools, - tool_config: None, // Use AUTO mode by default + tool_config: None, system_instruction, generation_config, }; @@ -257,7 +53,6 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Build an image generation request async fn build_image_request( &self, args: &BuildRequestArgs<'_>, @@ -267,7 +62,6 @@ impl GoogleAIQueryBuilder { let is_imagen = args.model.contains("imagen"); let request = if is_imagen { - // For Imagen models, use simple prompt format GeminiImageRequest { instances: Some(vec![GeminiPredictContent { prompt: args.user_message.trim().to_string(), @@ -275,7 +69,6 @@ impl GoogleAIQueryBuilder { contents: None, } } else { - // For Gemini models with image generation, build parts let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; if let Some(system_prompt) = args.system_prompt { @@ -285,7 +78,6 @@ impl GoogleAIQueryBuilder { ); } - // Add input images if provided if let Some(images) = args.images { for image in images.iter() { if !image.s3.is_empty() { @@ -308,218 +100,39 @@ impl GoogleAIQueryBuilder { .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) } - /// Convert OpenAI-format messages to Gemini format - async fn convert_messages_to_gemini( - &self, - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut gemini_messages = Vec::new(); - - for msg in messages { - match msg.role.as_str() { - "system" => { - // Skip - handled via args.system_prompt in build_text_request - } - "tool" => { - // Handle tool responses - if let (Some(tool_call_id), Some(content)) = (&msg.tool_call_id, &msg.content) { - let func_name = self.find_function_name_by_id(messages, tool_call_id); - let response_text = match content { - OpenAIContent::Text(text) => text.clone(), - OpenAIContent::Parts(parts) => parts - .iter() - .filter_map(|p| match p { - ContentPart::Text { text } => Some(text.clone()), - _ => None, - }) - .collect::>() - .join(" "), - }; - - gemini_messages.push(GeminiContentMessage { - role: Some("user".to_string()), - parts: vec![GeminiPart::FunctionResponse { - function_response: GeminiFunctionResponse { - name: func_name, - response: serde_json::json!({ "result": response_text }), - }, - }], - }); - } - } - _ => { - // Handle user/assistant messages - let role = match msg.role.as_str() { - "assistant" => "model", - _ => "user", - }; - - let mut parts = Vec::new(); - - // Handle regular content - if let Some(content) = &msg.content { - let content_parts = self - .convert_content_to_parts(&Some(content.clone()), client, workspace_id) - .await?; - parts.extend(content_parts); - } - - // Handle tool calls from assistant - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - let args: serde_json::Value = - serde_json::from_str(&tc.function.arguments).unwrap_or_default(); - // Extract thought_signature from extra_content if present - let thought_signature = tc - .extra_content - .as_ref() - .and_then(|ec| ec.google.as_ref()) - .and_then(|g| g.thought_signature.clone()); - parts.push(GeminiPart::FunctionCall { - function_call: GeminiFunctionCall { - name: tc.function.name.clone(), - args, - }, - thought_signature, - }); - } - } - - if !parts.is_empty() { - gemini_messages - .push(GeminiContentMessage { role: Some(role.to_string()), parts }); - } - } - } - } - - Ok(gemini_messages) - } - - /// Convert OpenAI content to Gemini parts - async fn convert_content_to_parts( - &self, - content: &Option, - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut parts = Vec::new(); - - if let Some(content) = content { - match content { - OpenAIContent::Text(text) => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - OpenAIContent::Parts(content_parts) => { - for part in content_parts { - match part { - ContentPart::Text { text } => { - if !text.is_empty() { - parts.push(GeminiPart::Text { text: text.clone() }); - } - } - ContentPart::ImageUrl { image_url } => { - // Parse data URL format: data:mime_type;base64,data - if let Some((mime_type, data)) = parse_data_url(&image_url.url) { - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - ContentPart::S3Object { s3_object } => { - if !s3_object.s3.is_empty() { - let (mime_type, data) = download_and_encode_s3_image( - s3_object, - client, - workspace_id, - ) - .await?; - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data }, - }); - } - } - } - } - } - } - } - - Ok(parts) - } - - /// Find function name by tool call ID from previous messages - fn find_function_name_by_id(&self, messages: &[OpenAIMessage], tool_call_id: &str) -> String { - for msg in messages { - if let Some(tool_calls) = &msg.tool_calls { - for tc in tool_calls { - if tc.id == tool_call_id { - return tc.function.name.clone(); - } - } - } - } - "unknown_function".to_string() - } - - /// Convert OpenAI tools to Gemini format + /// Convert OpenAI tool definitions to Gemini format. + /// + /// Sanitizes each tool's JSON schema for Google compatibility before delegating + /// to the shared [`openai_tools_to_gemini`] function. fn convert_tools_to_gemini( &self, tools: Option<&[ToolDef]>, has_websearch: bool, ) -> Option> { - let mut gemini_tools = Vec::new(); - - // Add function declarations - if let Some(tool_defs) = tools { - let declarations: Vec = tool_defs - .iter() - .filter_map(|t| { - // Deserialize RawValue into OpenAPISchema, sanitize, then use - let mut schema: OpenAPISchema = - serde_json::from_str(t.function.parameters.get()).ok()?; - schema.sanitize_for_google(); - - Some(GeminiFunctionDeclaration { - name: t.function.name.clone(), - description: t.function.description.clone(), - parameters: schema, - }) - }) - .collect(); - - if !declarations.is_empty() { - gemini_tools.push(GeminiTool { - function_declarations: Some(declarations), - google_search: None, - }); + let Some(tool_defs) = tools else { + if has_websearch { + return Some(vec![GeminiTool { + function_declarations: None, + google_search: Some(serde_json::json!({})), + }]); } - } + return None; + }; - // Add Google Search tool if enabled - if has_websearch { - gemini_tools.push(GeminiTool { - function_declarations: None, - google_search: Some(serde_json::json!({})), - }); - } + let tool_params: Vec = tool_defs + .iter() + .map(|t| { + let mut schema: OpenAPISchema = + serde_json::from_str(t.function.parameters.get()).unwrap_or_default(); + schema.sanitize_for_google(); + serde_json::to_value(&schema).unwrap_or_default() + }) + .collect(); - if gemini_tools.is_empty() { - None - } else { - Some(gemini_tools) - } + openai_tools_to_gemini(tool_defs, &tool_params, has_websearch) } - /// Build generation config for structured output and other settings - fn build_generation_config( - &self, - args: &BuildRequestArgs<'_>, - ) -> Option { + fn build_generation_config(&self, args: &BuildRequestArgs<'_>) -> Option { let has_output_schema = args .output_schema .and_then(|s| s.properties.as_ref()) @@ -529,15 +142,11 @@ impl GoogleAIQueryBuilder { let (response_mime_type, response_schema) = if has_output_schema { let mut schema = args.output_schema.unwrap().clone(); schema.sanitize_for_google(); - ( - Some("application/json".to_string()), - serde_json::to_value(&schema).ok(), - ) + (Some("application/json".to_string()), serde_json::to_value(&schema).ok()) } else { (None, None) }; - // Only create config if there's something to configure if args.temperature.is_some() || args.max_tokens.is_some() || response_mime_type.is_some() { Some(GeminiGenerationConfig { temperature: args.temperature, @@ -554,7 +163,6 @@ impl GoogleAIQueryBuilder { #[async_trait] impl QueryBuilder for GoogleAIQueryBuilder { fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { - // Google AI supports tools only for text output matches!(output_type, OutputType::Text) } @@ -578,7 +186,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { Error::internal_err(format!("Failed to parse Gemini image response: {}", e)) })?; - // First, check Gemini models (candidates -> content -> parts -> inline_data) let image_data_from_gemini = gemini_response.candidates.as_ref().and_then(|candidates| { candidates.iter().find_map(|candidate| { candidate @@ -589,13 +196,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { }) }); - // Then, check Imagen models (predictions -> bytes_base64_encoded) let image_data_from_imagen = gemini_response .predictions .as_ref() .and_then(|predictions| predictions.first().map(|p| &p.bytes_base64_encoded)); - // Image data, preferring Gemini first then Imagen models let image_data = image_data_from_gemini.or(image_data_from_imagen); match image_data { @@ -627,7 +232,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { .. } = gemini_sse_parser; - // Send tool call arguments events for accumulated tool calls for tool_call in accumulated_tool_calls.values() { let event = StreamingEvent::ToolCallArguments { call_id: tool_call.id.clone(), @@ -637,7 +241,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { stream_event_processor.send(event, &mut events_str).await?; } - // Convert Gemini usage metadata to TokenUsage let usage = gemini_usage.map(|u| { TokenUsage::new( u.prompt_token_count, @@ -647,11 +250,7 @@ impl QueryBuilder for GoogleAIQueryBuilder { }); Ok(ParsedResponse::Text { - content: if accumulated_content.is_empty() { - None - } else { - Some(accumulated_content) - }, + content: if accumulated_content.is_empty() { None } else { Some(accumulated_content) }, tool_calls: accumulated_tool_calls.into_values().collect(), events_str: Some(events_str), annotations, @@ -663,17 +262,11 @@ impl QueryBuilder for GoogleAIQueryBuilder { fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { match output_type { OutputType::Text => { - format!( - "{}/models/{}:streamGenerateContent?alt=sse", - base_url, model - ) + format!("{}/models/{}:streamGenerateContent?alt=sse", base_url, model) } OutputType::Image => { - let url_suffix = if model.contains("imagen") { - "predict" - } else { - "generateContent" - }; + let url_suffix = + if model.contains("imagen") { "predict" } else { "generateContent" }; format!("{}/models/{}:{}", base_url, model, url_suffix) } } @@ -685,7 +278,6 @@ impl QueryBuilder for GoogleAIQueryBuilder { _base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - // Native Gemini API always uses x-goog-api-key vec![("x-goog-api-key", api_key.to_string())] } } diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index 77fba3140a..62f13f3494 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -5,15 +5,18 @@ use reqwest::Response; use serde::Deserialize; use serde_json; use tokio_stream::StreamExt; -use windmill_common::{error::Error, utils::rd_string}; +use windmill_common::{ + ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}, + error::Error, + utils::rd_string, +}; use crate::ai::{ query_builder::StreamEventProcessor, types::{StreamingEvent, UrlCitation}, }; -use windmill_common::ai_types::{ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall}; - #[derive(Deserialize)] pub struct OpenAIChoiceDeltaToolCallFunction { pub name: Option, @@ -457,96 +460,19 @@ impl SSEParser for AnthropicSSEParser { // Gemini SSE Parser // ============================================================================ -/// Gemini streaming response part - can be text or function call -#[derive(Deserialize, Debug)] -pub struct GeminiSSEPart { - #[serde(default)] - pub text: Option, - #[serde(rename = "functionCall")] - pub function_call: Option, - /// Thought signature for Gemini 3+ models - required for function calling - #[serde(rename = "thoughtSignature")] - pub thought_signature: Option, -} - -/// Function call in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSEFunctionCall { - pub name: String, - pub args: serde_json::Value, -} - -/// Content in Gemini streaming candidate -#[derive(Deserialize, Debug)] -pub struct GeminiSSEContent { - pub parts: Option>, -} - -/// Web reference in Gemini grounding chunk -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunkWeb { - pub uri: String, - #[serde(default)] - pub title: Option, -} - -/// Grounding chunk from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingChunk { - pub web: Option, -} - -/// Grounding metadata from Gemini web search -#[derive(Deserialize, Debug)] -pub struct GeminiGroundingMetadata { - #[serde(rename = "groundingChunks", default)] - pub grounding_chunks: Vec, - #[serde(rename = "webSearchQueries", default)] - pub web_search_queries: Vec, -} - -/// Candidate in Gemini streaming response -#[derive(Deserialize, Debug)] -pub struct GeminiSSECandidate { - pub content: Option, - #[serde(rename = "finishReason")] - #[allow(dead_code)] - pub finish_reason: Option, - #[serde(rename = "groundingMetadata")] - pub grounding_metadata: Option, -} - -/// Gemini usage metadata from SSE response -#[derive(Deserialize, Debug, Clone)] -pub struct GeminiUsageMetadata { - #[serde(rename = "promptTokenCount", default)] - pub prompt_token_count: Option, - #[serde(rename = "candidatesTokenCount", default)] - pub candidates_token_count: Option, - #[serde(rename = "totalTokenCount", default)] - pub total_token_count: Option, -} - -/// Gemini SSE event structure -#[derive(Deserialize, Debug)] -pub struct GeminiSSEEvent { - pub candidates: Option>, - #[serde(rename = "usageMetadata")] - pub usage_metadata: Option, -} - -/// Gemini SSE Parser for streaming responses +/// Accumulates Gemini streaming events and converts them into the worker's +/// internal [`OpenAIToolCall`] / [`StreamingEvent`] representation. +/// +/// The actual SSE parsing is delegated to [`parse_gemini_sse_event`] from +/// `windmill_common::ai_google` so the logic can be shared with the API proxy. pub struct GeminiSSEParser { pub accumulated_content: String, pub accumulated_tool_calls: HashMap, pub events_str: String, pub stream_event_processor: StreamEventProcessor, tool_call_index: i64, - /// Collected URL citation annotations from web search pub annotations: Vec, - /// Whether web search was used in this response pub used_websearch: bool, - /// Token usage from usageMetadata pub usage: Option, } @@ -567,101 +493,57 @@ impl GeminiSSEParser { impl SSEParser for GeminiSSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: Option = serde_json::from_str(data) - .inspect_err(|e| { - tracing::error!("Failed to parse SSE as a Gemini event {}: {}", data, e); - }) - .ok(); + let Some(parsed) = parse_gemini_sse_event(data)? else { + return Ok(()); + }; - if let Some(event) = event { - if let Some(candidates) = event.candidates { - for candidate in candidates { - if let Some(content) = candidate.content { - if let Some(parts) = content.parts { - for part in parts { - // Handle text content - if let Some(text) = part.text { - if !text.is_empty() { - self.accumulated_content.push_str(&text); - let event = StreamingEvent::TokenDelta { content: text }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } - } + if let Some(text) = parsed.text { + self.accumulated_content.push_str(&text); + self.stream_event_processor + .send(StreamingEvent::TokenDelta { content: text }, &mut self.events_str) + .await?; + } - // Handle function calls - if let Some(function_call) = part.function_call { - let call_id = format!("call_{}", rd_string(24)); - let idx = self.tool_call_index; - self.tool_call_index += 1; + for tool_call in parsed.tool_calls { + let call_id = format!("call_{}", rd_string(24)); + let idx = self.tool_call_index; + self.tool_call_index += 1; - // Send tool call start event - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: function_call.name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; + self.stream_event_processor + .send( + StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: tool_call.name.clone(), + }, + &mut self.events_str, + ) + .await?; - // Build extra_content with thought_signature if present - let extra_content = - part.thought_signature.map(|sig| ExtraContent { - google: Some(GoogleExtraContent { - thought_signature: Some(sig), - }), - }); + let extra_content = tool_call.thought_signature.map(|sig| ExtraContent { + google: Some(GoogleExtraContent { thought_signature: Some(sig) }), + }); - // Store accumulated tool call - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: function_call.name, - arguments: serde_json::to_string( - &function_call.args, - ) - .unwrap_or_else(|_| "{}".to_string()), - }, - r#type: "function".to_string(), - extra_content, - }, - ); - } - } - } - } + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: tool_call.name, + arguments: serde_json::to_string(&tool_call.args) + .unwrap_or_else(|_| "{}".to_string()), + }, + r#type: "function".to_string(), + extra_content, + }, + ); + } - // Handle grounding metadata (web search results) - if let Some(ref grounding_metadata) = candidate.grounding_metadata { - // Set used_websearch if there are search queries or grounding chunks - if !grounding_metadata.web_search_queries.is_empty() - || !grounding_metadata.grounding_chunks.is_empty() - { - self.used_websearch = true; - } - - // Extract citations from grounding chunks - for chunk in &grounding_metadata.grounding_chunks { - if let Some(ref web) = chunk.web { - self.annotations.push(UrlCitation { - start_index: 0, // Gemini doesn't provide character indices - end_index: 0, - url: web.uri.clone(), - title: web.title.clone(), - }); - } - } - } - } - } - - // Extract usage metadata - if let Some(usage_metadata) = event.usage_metadata { - self.usage = Some(usage_metadata); - } + self.annotations.extend(parsed.annotations); + if parsed.used_websearch { + self.used_websearch = true; + } + if let Some(usage) = parsed.usage { + self.usage = Some(usage); } Ok(()) diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index a374db29c5..d095602b40 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -731,16 +731,3 @@ pub fn extract_text_content(content: &OpenAIContent) -> String { .join(""), } } - -/// Parse a data URL to extract media type and base64 data -/// Format: data:mime_type;base64,data -/// Returns (media_type, data) tuple if successful -pub fn parse_data_url(url: &str) -> Option<(String, String)> { - if !url.starts_with("data:") { - return None; - } - let rest = url.strip_prefix("data:")?; - let (header, data) = rest.split_once(",")?; - let media_type = header.strip_suffix(";base64")?; - Some((media_type.to_string(), data.to_string())) -} From 874273d0b716a113350f86a2969de691cc58e18c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 9 Mar 2026 16:12:39 +0000 Subject: [PATCH 41/57] feat: expose OTEL trace context as env vars in job execution (#8277) --- backend/windmill-worker/src/java_executor.rs | 4 ++- backend/windmill-worker/src/php_executor.rs | 4 ++- backend/windmill-worker/src/worker.rs | 33 ++++++++++++++++++-- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index efbe403145..6f558e3328 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -618,6 +618,7 @@ async fn run<'a>( .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .args(vec![ "--config", "run.config.proto", @@ -675,7 +676,8 @@ async fn run<'a>( .env("HOME", &*JAVA_HOME_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .envs(envs) - .envs(reserved_variables); + .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)); if metadata(TRUST_STORE_PATH.clone()).await.is_ok() { cmd.args(&[ &format!("-Djavax.net.ssl.trustStore={}", *TRUST_STORE_PATH), diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index e98755e555..df006fef7f 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -22,7 +22,7 @@ use crate::{ get_reserved_variables, read_result, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, is_sandboxing_enabled, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -316,6 +316,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .args(args) @@ -332,6 +333,7 @@ try {{ .env_clear() .envs(envs) .envs(reserved_variables) + .envs(crate::get_otel_context_envs(&job.id)) .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) .env("BASE_INTERNAL_URL", base_internal_url) .stdin(Stdio::null()) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 1661f39803..8d7f6051ec 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -740,6 +740,24 @@ pub async fn is_otel_tracing_proxy_enabled_for_lang(lang: &ScriptLang) -> bool { } } +/// Get OTEL trace context environment variables for a job (TRACEPARENT, OTEL_TRACE_ID, OTEL_SPAN_ID). +/// Returns an empty vec when OTEL tracing is not enabled or on non-enterprise builds. +pub fn get_otel_context_envs(job_id: &uuid::Uuid) -> Vec<(&'static str, String)> { + #[cfg(all(feature = "private", feature = "enterprise"))] + if windmill_common::OTEL_TRACING_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + let trace_id = format!("{:032x}", job_id.as_u128()); + let span_id = format!("{:016x}", job_id.as_u64_pair().1); + let traceparent = format!("00-{}-{}-01", trace_id, span_id); + return vec![ + ("TRACEPARENT", traceparent), + ("OTEL_TRACE_ID", trace_id), + ("OTEL_SPAN_ID", span_id), + ]; + } + let _ = job_id; + vec![] +} + /// Get proxy environment variables for job execution for a specific language. /// When OTEL tracing proxy is enabled for this language, routes all traffic through the proxy. /// Otherwise, uses the standard HTTP_PROXY/HTTPS_PROXY from environment. @@ -749,12 +767,21 @@ pub async fn get_proxy_envs_for_lang( w_id: &str, conn: &Connection, ) -> anyhow::Result> { + #[allow(unused_mut)] + let mut envs; #[cfg(all(feature = "private", feature = "enterprise"))] if is_otel_tracing_proxy_enabled_for_lang(lang).await { - return get_otel_tracing_proxy_envs(job_id, w_id, conn).await; + envs = get_otel_tracing_proxy_envs(job_id, w_id, conn).await?; + } else { + envs = PROXY_ENVS.clone(); } - let _ = (lang, job_id, w_id, conn); - Ok(PROXY_ENVS.clone()) + #[cfg(not(all(feature = "private", feature = "enterprise")))] + { + let _ = (lang, w_id, conn); + envs = PROXY_ENVS.clone(); + } + envs.extend(get_otel_context_envs(job_id)); + Ok(envs) } #[cfg(all(feature = "private", feature = "enterprise"))] From e6bb26d0a4edde6970504c53ae819fdafd8e40b2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 9 Mar 2026 18:28:10 +0000 Subject: [PATCH 42/57] fix: redact secrets in set_global_setting log line (#8270) --- backend/windmill-api-settings/src/lib.rs | 6 +++++- .../windmill-common/src/instance_config.rs | 21 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 6b408724fd..4198d82e48 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -316,7 +316,11 @@ pub async fn set_global_setting_internal( ) .execute(db) .await?; - tracing::info!("Set global setting {} to {}", key, v); + tracing::info!( + "Set global setting {} to {}", + key, + instance_config::format_setting_value(&key, &v) + ); } }; diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 0225448668..fb70db90ae 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -934,7 +934,7 @@ fn redact_string(s: &str) -> String { } } -fn format_setting_value(key: &str, value: &serde_json::Value) -> String { +pub fn format_setting_value(key: &str, value: &serde_json::Value) -> String { if SENSITIVE_SETTINGS.contains(&key) { return match value { serde_json::Value::String(s) => format!("\"{}\"", redact_string(s)), @@ -2219,6 +2219,25 @@ mod tests { assert_eq!(v, *"world"); } + #[test] + fn string_or_secret_ref_debug_masks_literal() { + let v = StringOrSecretRef::Literal("super-secret-value".to_string()); + let debug = format!("{v:?}"); + assert_eq!(debug, "Literal(****)"); + assert!(!debug.contains("super-secret-value")); + } + + #[test] + fn format_setting_value_redacts_oauth_secrets() { + let val = serde_json::json!({ + "google": {"id": "client-id", "secret": "my-super-secret-12345"} + }); + let formatted = format_setting_value("oauths", &val); + assert!(!formatted.contains("my-super-secret-12345")); + assert!(formatted.contains("client-id")); + assert!(formatted.contains("****")); + } + #[test] #[should_panic(expected = "literal_value() called on unresolved secret ref")] fn string_or_secret_ref_literal_value_panics_on_ref() { From 66fe71bd3c16a69f84041807a007981519f01828 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:04:25 +0100 Subject: [PATCH 43/57] chore: webmux config --- .webmux.yaml | 3 +++ scripts/worktree-common.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.webmux.yaml b/.webmux.yaml index b01d98b8d2..0acd9f08e2 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -15,6 +15,9 @@ lifecycleHooks: postCreate: bash ./scripts/post-create.sh preRemove: bash ./scripts/pre-remove.sh +auto_name: + model: gemini-2.5-flash-lite + # Each service defines a port env var that webmux injects into pane and agent # process environments when creating a worktree. Ports are auto-assigned: # base + (slot x step). diff --git a/scripts/worktree-common.sh b/scripts/worktree-common.sh index 0112d271fa..91ba00a5e1 100755 --- a/scripts/worktree-common.sh +++ b/scripts/worktree-common.sh @@ -202,9 +202,9 @@ wm_shared_post_create() { local main_repo_root main_repo_root="$(wm_main_repo_root "$repo_root")" + wm_allow_direnv "$repo_root" wm_setup_database "$repo_root" "${repo_root}/.env.local" wm_copy_dependencies "$repo_root" "$main_repo_root" - wm_allow_direnv "$repo_root" wm_trust_claude "$repo_root" wm_setup_ee_worktree "$repo_root" "$main_repo_root" } From d24f43679df68e86d5bc28d3457d7caaba20f3b9 Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Mon, 9 Mar 2026 20:28:42 +0100 Subject: [PATCH 44/57] chore: yolo config for webmux (#8286) * chore: yolo config for webmux * systemprompt * nitt --- .webmux.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.webmux.yaml b/.webmux.yaml index 0acd9f08e2..bd88016043 100644 --- a/.webmux.yaml +++ b/.webmux.yaml @@ -34,7 +34,17 @@ services: profiles: default: runtime: host + yolo: true envPassthrough: [] + systemPrompt: > + You are running inside a tmux session with other panes running services. + Pane layout (current window): + - Pane 0: this pane (claude agent) + - Pane 1: backend (cargo watch -x run) + - Pane 2: frontend (npm run dev) + To check logs, use: \`tmux capture-pane -t .1 -p -S -50\` (backend) or \`tmux capture-pane -t .2 -p -S -50\` (frontend). + When restarting backend or frontend, make sure to use ${BACKEND_PORT} and ${FRONTEND_PORT}. + Because we are running backend with cargo watch, to verify your changes, just check the logs in the backend pane. No need for cargo check. panes: - id: agent kind: agent From 7cfbc142871e5f4f5f17a336afda547e47ad01ec Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 9 Mar 2026 19:39:24 +0000 Subject: [PATCH 45/57] feat: workflow-as-code (WAC) v2 (#8172) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: workflow-as-code v2 with @task decorator API Replace ctx.step("name", "script") API with @task decorators where functions are called directly. Users no longer need to pass WorkflowCtx or use string-based step names/script paths. Python: @task decorator with contextvars-based implicit context TypeScript: task() wrapper with module-level context variable Parsers: detect @task function calls instead of ctx.step() calls Worker: updated wrappers to set implicit context Co-Authored-By: Claude Opus 4.6 * feat: WAC v2 checkpoint/replay with _executing_key child dispatch - Rust-side orchestration: parent dispatches child jobs, suspends, resumes on completion - _executing_key in checkpoint tells child which step to execute directly - task() throws StepSuspend(mode="step_complete") after executing target step - result_processor handles child completion and updates parent checkpoint - WacGraph.svelte for runtime execution visualization - Sequential and parallel workflows tested end-to-end Co-Authored-By: Claude Opus 4.6 * fix: WAC v2 bundle cache, globalThis ctx sharing, description optional - Disable bun bundle caching for WAC v2 scripts (wrapper needs windmill-client from node_modules, not available in bundle mode) - Use Reflect.set/get(globalThis, "__wmill_wf_ctx") to share workflow context across dual module instances (wrapper vs user script) - Never-resolving thenable for non-matching steps in child job mode prevents Promise.all race conditions - Make description field optional in NewScript API (defaults to "") Co-Authored-By: Claude Opus 4.6 * feat: add step() primitive for inline checkpointed steps step() executes a function inline (no child job) and persists the result to the checkpoint. On replay, the cached value is returned — ensuring deterministic behavior for non-deterministic operations like Date.now() or Math.random(). - TypeScript: step(name, fn) — executes inline, throws StepSuspend with mode "inline_checkpoint" to persist before continuing - Rust: InlineCheckpoint variant in WacOutput, saves to checkpoint and resets running=false for immediate re-pickup (no zombie wait) - Shared step counter between task() and step() via _allocKey() Co-Authored-By: Claude Opus 4.6 * feat: add Python WAC v2 support with task(), step(), workflow() - Python SDK: WorkflowCtx with _executing_key child mode, _alloc_key shared counter, _run_inline_step for step(), _execute_directly and _never_resolve for child mode, step() async function - Python executor: WAC v2 detection, checkpoint.json writing, WAC wrapper.py generation calling _run_workflow(), post-execution hook into shared handle_wac_v2_output() - Make handle_wac_v2_output pub so both bun and python executors share the same dispatch/suspend/inline-checkpoint logic - 17 Python tests covering dispatch, replay, parallel, conditional, inline checkpoint, and child mode Co-Authored-By: Claude Opus 4.6 * chore: update sqlx prepared queries Co-Authored-By: Claude Opus 4.6 * fix: WacGraph Tooltip→Popover, simplify wacToFlow parsers - Fix type error: Tooltip doesn't accept text snippet, use Popover - Extract shared helpers for task matching and block collection - Replace linear tasks.find() with Map lookups - Remove mutable module-level counter Co-Authored-By: Claude Opus 4.6 * fix: Box::pin WAC v2 output handler to prevent stack overflow handle_python_job's async state machine was too large when combined with handle_wac_v2_output. Box::pin heap-allocates the future. Co-Authored-By: Claude Opus 4.6 * fix: merge WAC v1 and v2 task decorators to preserve backward compat The v2 @task decorator was shadowing the v1 one, breaking WAC v1 scripts that rely on HTTP-based dispatch via /workflow_as_code/ API. The merged decorator handles three modes: - v2: inside @workflow context → checkpoint/replay dispatch - v1: WM_JOB_ID set, no @workflow → HTTP API dispatch + wait_job - standalone: no Windmill env → execute function body directly Co-Authored-By: Claude Opus 4.6 * fix: skip no_main_func detection for WAC v2 scripts in TS and Python parsers Co-Authored-By: Claude Opus 4.6 * fix: prevent empty/noop dispatch causing infinite requeue loop - Validate steps.len() > 0 in WAC dispatch handler (issue 3) - Replace noop StepSuspend throw with never-resolving promise so it can't reach the backend as an empty dispatch (issue 4) Co-Authored-By: Claude Opus 4.6 * fix: Python task wrapper now converts positional args to kwargs in v2 mode Previously only **kwargs were passed to _next_step(), silently dropping positional arguments. Extract shared _merge_args() helper used by both v1 and v2 paths. Co-Authored-By: Claude Opus 4.6 * fix: replace unwrap() with proper error propagation in WAC arg serialization Co-Authored-By: Claude Opus 4.6 * fix: add workspace_id filter to v2_job queries in WAC dispatch Co-Authored-By: Claude Opus 4.6 * fix: prevent race condition in WAC child dispatch Restructure dispatch to save checkpoint + suspend parent + seed child checkpoints in a single transaction BEFORE pushing child jobs. This ensures a fast child can't complete before the parent is suspended. Also wrap InlineCheckpoint save + running reset in a transaction to prevent corrupted state on crash. Use ULID for pre-generated child job IDs (consistent with rest of API). Co-Authored-By: Claude Opus 4.6 * fix: include step key and child job ID in WAC error propagation Move step_key lookup before the success check so failed child errors include which task failed, the child job ID, and the original error. Co-Authored-By: Claude Opus 4.6 * docs: document WAC determinism contract and step dispatch semantics - Document that workflow functions must be deterministic across replays - Document that WacStepDispatch.script/args are metadata, not dispatch targets - Add comments on counter-based key allocation Co-Authored-By: Claude Opus 4.6 * fix: tighten WAC v2 detection to reduce false positives Replace naive substring matching with line-aware checks that skip comments and look for specific patterns: - TS: import from "windmill-client" containing workflow/task - Python: @workflow and @task decorators with wmill import Extracted shared helpers in wac_executor.rs used by both executors. Co-Authored-By: Claude Opus 4.6 * fix: show failed steps in WacGraph when workflow completes with errors When flowDone is true and a pending step isn't in completedSteps, mark it as 'failed' instead of 'running'. The failed state CSS and XCircle icon were already defined but never triggered. Co-Authored-By: Claude Opus 4.6 * fix: unsuspend and fail parent when WAC child push fails Previously if a child push failed mid-batch, the parent remained suspended with suspend = num_steps but fewer children, hanging until the 14-day timeout. Now the push loop catches errors and unsuspends the parent before returning the error. Also adds source hash validation: if the script content changes between replays, the job fails with a clear error instead of silently feeding stale checkpoint data into wrong steps. Co-Authored-By: Claude Opus 4.6 * fix: clear suspend_until when unsuspending WAC parent Set suspend_until = NULL alongside suspend = 0 in both the child failure and all-children-complete paths, so the parent doesn't rely on subtle pull query invariants to be re-picked-up. Co-Authored-By: Claude Opus 4.6 * test: add exhaustive edge case tests for WAC v2 SDK fix: make TS task wrapper non-async to fix unawaited task flush The async wrapper caused microtask-based thenable auto-resolution that fired .then() and threw StepSuspend before _flushPending() could capture unawaited steps — making the flush mechanism completely broken. Now the thenable is returned directly without async wrapping. Backward compatible with v1 (all code paths still return awaitables). Tests added (59 TS + 66 Python) covering: full sequential lifecycle, step after parallel, parallel after parallel, conditional on step result, empty/single-task workflows, 10+ steps, falsy value preservation, inline steps, mixed step/task, unawaited flush, child mode with parallel, key determinism, large parallel groups, and complex mixed patterns. Co-Authored-By: Claude Opus 4.6 * fix: atomic checkpoint updates to prevent parallel child race condition Replace read-modify-write pattern in handle_wac_child_completion with atomic SQL operations: - completed_steps merged via jsonb_set(... || jsonb_build_object(...)) so concurrent children on different workers don't overwrite each other - suspend counter decremented atomically with RETURNING to determine "all done" condition (instead of checking completed_steps in memory) - suspend_until cleared in the same atomic decrement statement Before this fix, two parallel children completing simultaneously could both load the same checkpoint, each add their step, and save — the second write would overwrite the first, silently losing a child result and leaving the parent suspended forever. Co-Authored-By: Claude Opus 4.6 * fix: cancel already-pushed children on partial WAC dispatch failure When pushing child jobs sequentially, if pushing child N fails, children 1..N-1 are already running. Previously the error handler only unsuspended the parent, leaving orphaned children that would complete and corrupt the checkpoint state (decrementing suspend on an already-unsuspended parent, potentially causing duplicate step execution on re-run). Now on partial failure: 1. Cancel all already-pushed children (prevents them from completing and corrupting checkpoint state) 2. Clear pending_steps from checkpoint (so parent doesn't think children are outstanding on re-run) 3. Then unsuspend parent (so the error propagates) Co-Authored-By: Claude Opus 4.6 * fix: skip WAC duration write and child check for non-WAC parents The duration write to workflow_as_code_status was running for every non-flow child with a parent (error handlers, success handlers, run_script children), even though it was only intended for WAC jobs. Add WHERE workflow_as_code_status IS NOT NULL to skip non-WAC parents entirely. Piggyback RETURNING pending_steps.job_ids on the same query so WAC v2 child completion needs zero extra DB round-trips on the success path. Co-Authored-By: Claude Opus 4.6 * fix: seed child checkpoint in same transaction as push The child checkpoint insert was happening before the child job was pushed, violating the FK constraint on v2_job_status. Move it into the push transaction so the job row exists and the child can't be picked up before its checkpoint is ready. Co-Authored-By: Claude Opus 4.6 * fix: set running=false when WAC parent suspends for child dispatch The parent job kept running=true after suspending, so workers wouldn't pick it up when children completed and suspend reached 0. The parent only advanced when the zombie job detector reset it (~90s). Now the dispatch suspend sets running=false so the parent is immediately eligible for pickup. Co-Authored-By: Claude Opus 4.6 * fix: WAC parent suspend/unsuspend lifecycle Keep running=true when suspending the parent so the normal pull query (WHERE running=false) never picks it up. Keep suspend_until non-null when decrementing suspend to 0 so the suspended pull query (WHERE suspend_until IS NOT NULL AND suspend<=0) picks it up. Previously: setting running=false caused infinite restart loops because the normal pull query has no suspend check and would immediately re-pick the parent. Clearing suspend_until on the last child prevented the suspended pull from ever seeing it, requiring the 90s zombie detector. Co-Authored-By: Claude Opus 4.6 * feat: add approval primitive, flow child completion, timeline fixes for WAC v2 Co-Authored-By: Claude Opus 4.6 * feat: add error propagation, task options, sleep, and parallel for WAC v2 Co-Authored-By: Claude Opus 4.6 * test: fix python SDK tests to use name-based keys and add new test coverage Co-Authored-By: Claude Opus 4.6 * fix: address WAC v2 review findings (sleep timing, error marker, atomicity) - Fix sleep using suspend=1 instead of 0 to enforce actual delay - Add approval/sleep resume injection to Python executor - Fix TS SDK concurrency_limit mapping (was reading wrong property) - Namespace error marker as __wmill_error to avoid user data collision - Wrap child completion SQL in transaction for atomicity - Decrement suspend even when step key is missing (prevents hang) - Expand TASK_RE to handle export const, let, var, generics - Validate step key uniqueness before dispatch - Log warning on checkpoint deserialization failure - Remove unimplemented delete_after_use from SDKs - Add TaskError exception class to Python SDK with diagnostic context - Fix extra positional args handling and add functools.wraps - Improve getParamNames to handle typed/destructured params Co-Authored-By: Claude Opus 4.6 * sqlx * sqlx * test: add WAC v1 e2e integration tests for TS and Python Co-Authored-By: Claude Opus 4.6 * fix: revert fake test versions in typescript-client Co-Authored-By: Claude Opus 4.6 * refactor: remove unused WacGraph component and strip wacToFlow to isWorkflowAsCode Co-Authored-By: Claude Opus 4.6 * refactor: extract shared approval/sleep resume logic into wac_executor Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- ...b6efdae570e3416ec6c2493cc04f75c32a699.json | 16 + ...b7ab1f2fc29a2ba79a39576551bdf66b592b6.json | 15 + ...dacbabed4c5ae28101e3ae2694f96fd055a91.json | 2 +- ...50d9d258c288883b2b5b0ab286f5cb50850b5.json | 16 - ...153c43903f929ae5d62fbba12610f89c36d55.json | 2 +- ...e82c45be45409f881292f0d4a3316362ba1f4.json | 14 + ...3e6e08f808f423c8f2d58b9c849aba7d176f5.json | 14 + ...1e91093233cf16af0dae666b4743f3878b22e.json | 14 + ...5be9d9f51071978c0e48df4284d8b90000a4a.json | 22 + ...933aa54523bf44d63e304440e53b9eadd5340.json | 24 + ...e0c9c8c61e1bf609be60ac5dc5d438189353.json} | 10 +- ...706d78a6f24cb0e614d7d81ba1b643805bf06.json | 2 +- ...2808e90cb068d1048717f82992f476377cc20.json | 15 + backend/Cargo.lock | 16 + backend/Cargo.toml | 2 + backend/parsers/windmill-parser-py/src/lib.rs | 5 +- backend/parsers/windmill-parser-ts/src/lib.rs | 14 +- .../parsers/windmill-parser-wac/Cargo.toml | 21 + .../parsers/windmill-parser-wac/src/dag.rs | 44 + .../parsers/windmill-parser-wac/src/lib.rs | 32 + .../parsers/windmill-parser-wac/src/python.rs | 717 ++++++++ .../windmill-parser-wac/src/typescript.rs | 739 ++++++++ .../windmill-parser-wac/src/validation.rs | 64 + .../windmill-parser-wac/tests/python_tests.rs | 266 +++ .../windmill-parser-wac/tests/ts_tests.rs | 245 +++ .../parsers/windmill-parser-wasm/Cargo.toml | 2 + backend/parsers/windmill-parser-wasm/build.nu | 6 + .../parsers/windmill-parser-wasm/src/lib.rs | 7 + backend/test_wac_e2e.sh | 157 ++ backend/windmill-api/openapi.yaml | 1 - backend/windmill-api/src/jobs.rs | 39 +- backend/windmill-common/src/error.rs | 3 + backend/windmill-queue/src/jobs.rs | 35 +- .../nsjail/run.bun.config.proto | 7 + .../windmill-worker/src/ai/image_handler.rs | 2 +- .../windmill-worker/src/ai/query_builder.rs | 4 +- backend/windmill-worker/src/ai/types.rs | 17 +- .../windmill-worker/src/bigquery_executor.rs | 2 +- backend/windmill-worker/src/bun_executor.rs | 963 +++++++++- backend/windmill-worker/src/lib.rs | 1 + .../windmill-worker/src/python_executor.rs | 96 +- .../windmill-worker/src/result_processor.rs | 243 ++- .../windmill-worker/src/snowflake_executor.rs | 2 +- backend/windmill-worker/src/wac_executor.rs | 339 ++++ backend/windmill-worker/src/worker.rs | 7 + backend/windmill-worker/src/worker_flow.rs | 29 +- .../src/lib/components/ScriptBuilder.svelte | 3 +- .../src/lib/components/TimelineBar.svelte | 7 +- .../lib/components/WorkflowTimeline.svelte | 11 +- .../src/lib/components/graph/wacToFlow.ts | 16 + .../lib/components/runs/JobRunsPreview.svelte | 7 +- .../components/scriptEditor/LogPanel.svelte | 8 +- .../(root)/(logged)/run/[...run]/+page.svelte | 21 +- python-client/wmill/pyproject.toml | 6 + python-client/wmill/tests/test_workflow.py | 1114 ++++++++++++ python-client/wmill/wmill/client.py | 553 +++++- typescript-client/build.sh | 24 +- typescript-client/client.ts | 492 ++++- typescript-client/package-lock.json | 4 +- typescript-client/tests/e2e_wac.py | 182 ++ typescript-client/tests/e2e_wac_v1.py | 190 ++ typescript-client/tests/workflow.test.ts | 1596 +++++++++++++++++ 62 files changed, 8318 insertions(+), 209 deletions(-) create mode 100644 backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json create mode 100644 backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json delete mode 100644 backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json create mode 100644 backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json create mode 100644 backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json create mode 100644 backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json create mode 100644 backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json create mode 100644 backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json rename backend/.sqlx/{query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json => query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json} (65%) create mode 100644 backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json create mode 100644 backend/parsers/windmill-parser-wac/Cargo.toml create mode 100644 backend/parsers/windmill-parser-wac/src/dag.rs create mode 100644 backend/parsers/windmill-parser-wac/src/lib.rs create mode 100644 backend/parsers/windmill-parser-wac/src/python.rs create mode 100644 backend/parsers/windmill-parser-wac/src/typescript.rs create mode 100644 backend/parsers/windmill-parser-wac/src/validation.rs create mode 100644 backend/parsers/windmill-parser-wac/tests/python_tests.rs create mode 100644 backend/parsers/windmill-parser-wac/tests/ts_tests.rs create mode 100755 backend/test_wac_e2e.sh create mode 100644 backend/windmill-worker/src/wac_executor.rs create mode 100644 frontend/src/lib/components/graph/wacToFlow.ts create mode 100644 python-client/wmill/tests/test_workflow.py create mode 100644 typescript-client/tests/e2e_wac.py create mode 100644 typescript-client/tests/e2e_wac_v1.py create mode 100644 typescript-client/tests/workflow.test.ts diff --git a/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json new file mode 100644 index 0000000000..b8e52cdbe7 --- /dev/null +++ b/backend/.sqlx/query-0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0cd9cad7109340edc81a5a40620b6efdae570e3416ec6c2493cc04f75c32a699" +} diff --git a/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json new file mode 100644 index 0000000000..3f39982319 --- /dev/null +++ b/backend/.sqlx/query-10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "10af387fce25f6ea7af275e8e93b7ab1f2fc29a2ba79a39576551bdf66b592b6" +} diff --git a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json index da0ce60709..90f38c05a7 100644 --- a/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json +++ b/backend/.sqlx/query-2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - false + true ] }, "hash": "2d6607b3c38fe72b5663c32de58dacbabed4c5ae28101e3ae2694f96fd055a91" diff --git a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json b/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json deleted file mode 100644 index 6b47103c3a..0000000000 --- a/backend/.sqlx/query-56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "56f7325e3b0316866714e76d94b50d9d258c288883b2b5b0ab286f5cb50850b5" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json new file mode 100644 index 0000000000..aedbbf424e --- /dev/null +++ b/backend/.sqlx/query-a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a35164456ade8e79cb8f5418c8fe82c45be45409f881292f0d4a3316362ba1f4" +} diff --git a/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json new file mode 100644 index 0000000000..7a45e6c402 --- /dev/null +++ b/backend/.sqlx/query-a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a684f160d1a366c1928fef27c613e6e08f808f423c8f2d58b9c849aba7d176f5" +} diff --git a/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json new file mode 100644 index 0000000000..4fa871c594 --- /dev/null +++ b/backend/.sqlx/query-a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a72081cb042f09034338dcb49381e91093233cf16af0dae666b4743f3878b22e" +} diff --git a/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json new file mode 100644 index 0000000000..8d09036772 --- /dev/null +++ b/backend/.sqlx/query-b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1 RETURNING suspend", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "suspend", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b663e6baf2f8da00c6d94e5b8e35be9d9f51071978c0e48df4284d8b90000a4a" +} diff --git a/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json new file mode 100644 index 0000000000..efd03ae26e --- /dev/null +++ b/backend/.sqlx/query-c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n workflow_as_code_status,\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'duration_ms'],\n to_jsonb($2::bigint)\n )\n WHERE id = $3 AND workflow_as_code_status IS NOT NULL\n RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS \"job_ids: serde_json::Value\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_ids: serde_json::Value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "c331609952e0b98d36f605bd5d2933aa54523bf44d63e304440e53b9eadd5340" +} diff --git a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json similarity index 65% rename from backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json rename to backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json index 5b82c5288f..5791090fc1 100644 --- a/backend/.sqlx/query-1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82.json +++ b/backend/.sqlx/query-dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE ji.parent_job\n END\n JOIN v2_job j ON j.id = q.id\n JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", + "query": "\n WITH job_info AS (\n SELECT id, kind::text AS kind, parent_job\n FROM v2_job\n WHERE id = $1\n )\n SELECT\n q.id AS \"id!\",\n s.flow_status,\n q.suspend AS \"suspend!\",\n j.runnable_path AS script_path,\n j.permissioned_as_email AS email,\n (ji.kind IN ('flow', 'flowpreview')) AS \"is_flow_level!\",\n (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS \"is_wac!\"\n FROM job_info ji\n JOIN v2_job_queue q ON q.id = CASE\n WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id\n ELSE COALESCE(ji.parent_job, ji.id)\n END\n JOIN v2_job j ON j.id = q.id\n LEFT JOIN v2_job_status s ON s.id = q.id\n FOR UPDATE OF q\n ", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "is_flow_level!", "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "is_wac!", + "type_info": "Bool" } ], "parameters": { @@ -45,8 +50,9 @@ false, true, false, + null, null ] }, - "hash": "1a0ab65bbf2751f702fc696c1e32a7dd9524cdd806be1ad8e9ab88d4c88d3f82" + "hash": "dbc7e74e259b502e700491ee0248e0c9c8c61e1bf609be60ac5dc5d438189353" } diff --git a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json index c96961eac4..0a06188897 100644 --- a/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json +++ b/backend/.sqlx/query-eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - false + true ] }, "hash": "eba16eb819e2644284fb073c891706d78a6f24cb0e614d7d81ba1b643805bf06" diff --git a/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json new file mode 100644 index 0000000000..72acab6120 --- /dev/null +++ b/backend/.sqlx/query-f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "f56c58fea9f27d2e55d33720e032808e90cb068d1048717f82992f476377cc20" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3f0ee056f9..1ab271d577 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16956,6 +16956,22 @@ dependencies = [ "windmill-parser-sql", ] +[[package]] +name = "windmill-parser-wac" +version = "1.651.1" +dependencies = [ + "anyhow", + "rustpython-ast", + "rustpython-parser", + "serde", + "serde_json", + "sha2 0.10.9", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_visit", +] + [[package]] name = "windmill-parser-yaml" version = "1.651.1" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7ad583f31e..0bbdc65bd7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -68,6 +68,7 @@ members = [ "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", + "./parsers/windmill-parser-wac", "./parsers/windmill-sql-datatype-parser-wasm", "./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu", "./windmill-worker-volumes", @@ -332,6 +333,7 @@ windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } windmill-parser-sql = { path = "./parsers/windmill-parser-sql" } windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } +windmill-parser-wac = { path = "./parsers/windmill-parser-wac" } windmill-jseval = { path = "./windmill-jseval" } windmill-runtime-nativets = { path = "./windmill-runtime-nativets" } windmill-api-client = { path = "./windmill-api-client" } diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index c299bff4af..655a1a4ea9 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -296,11 +296,14 @@ pub fn parse_python_signature( // Check if main function was found if params.is_none() { + let is_wac_v2 = (code.contains("@workflow") || code.contains("workflow(")) + && (code.contains("@task") || code.contains("task(")) + && (code.contains("import wmill") || code.contains("from wmill")); return Ok(MainArgSignature { star_args: false, star_kwargs: false, args: vec![], - no_main_func: Some(true), + no_main_func: Some(!is_wac_v2), has_preprocessor: Some(has_preprocessor), }); } diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 04dd345b2f..f0928fe984 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -261,7 +261,9 @@ pub fn parse_deno_signature( for specifier in &named_export.specifiers { if let swc_ecma_ast::ExportSpecifier::Named(spec) = specifier { let export_name = match &spec.exported { - Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => ident.sym.as_ref(), + Some(swc_ecma_ast::ModuleExportName::Ident(ident)) => { + ident.sym.as_ref() + } Some(swc_ecma_ast::ModuleExportName::Str(s)) => s.value.as_ref(), None => match &spec.orig { swc_ecma_ast::ModuleExportName::Ident(ident) => ident.sym.as_ref(), @@ -315,7 +317,11 @@ pub fn parse_deno_signature( let mut c: u16 = 0; - let no_main_func = entrypoint_params.is_none(); + let is_wac_v2 = entrypoint_params.is_none() + && code.contains("workflow(") + && code.contains("task(") + && code.contains("windmill-client"); + let no_main_func = entrypoint_params.is_none() && !is_wac_v2; let mut type_resolver = HashMap::new(); let r = MainArgSignature { star_args: false, @@ -833,7 +839,9 @@ fn tstype_to_typ( false, ), symbol @ _ if symbol.starts_with("DynMultiselect_") => ( - Typ::DynMultiselect(symbol.strip_prefix("DynMultiselect_").unwrap().to_string()), + Typ::DynMultiselect( + symbol.strip_prefix("DynMultiselect_").unwrap().to_string(), + ), false, ), symbol @ _ => { diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml new file mode 100644 index 0000000000..d354b1c653 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "windmill-parser-wac" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_wac" +path = "./src/lib.rs" + +[dependencies] +rustpython-parser.workspace = true +rustpython-ast = { version = "0.4.0", features = ["visitor"] } +swc_common.workspace = true +swc_ecma_parser.workspace = true +swc_ecma_ast.workspace = true +swc_ecma_visit.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +sha2.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/dag.rs b/backend/parsers/windmill-parser-wac/src/dag.rs new file mode 100644 index 0000000000..662f5a06b6 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/dag.rs @@ -0,0 +1,44 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WorkflowDag { + pub nodes: Vec, + pub edges: Vec, + pub params: Vec, + pub source_hash: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct Param { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub typ: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagNode { + pub id: String, + pub node_type: DagNodeType, + pub label: String, + pub line: usize, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(tag = "type")] +pub enum DagNodeType { + Step { name: String, script: String }, + Branch { condition_source: String }, + ParallelStart, + ParallelEnd, + LoopStart { iter_source: String }, + LoopEnd, + Return, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DagEdge { + pub from: String, + pub to: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, +} diff --git a/backend/parsers/windmill-parser-wac/src/lib.rs b/backend/parsers/windmill-parser-wac/src/lib.rs new file mode 100644 index 0000000000..1496b9d6cf --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/lib.rs @@ -0,0 +1,32 @@ +pub mod dag; +pub mod python; +pub mod typescript; +pub mod validation; + +use dag::WorkflowDag; +use validation::CompileError; + +#[derive(Debug, serde::Serialize)] +#[serde(tag = "type")] +pub enum ParseResult { + #[serde(rename = "success")] + Success(WorkflowDag), + #[serde(rename = "error")] + Error { errors: Vec }, +} + +pub fn parse_workflow(code: &str, language: &str) -> ParseResult { + let result = match language { + "python" | "python3" | "py" => python::parse_python_workflow(code), + "typescript" | "ts" | "deno" | "bun" => typescript::parse_ts_workflow(code), + _ => Err(vec![CompileError { + message: format!("Unsupported language: {language}"), + line: 0, + }]), + }; + + match result { + Ok(dag) => ParseResult::Success(dag), + Err(errors) => ParseResult::Error { errors }, + } +} diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs new file mode 100644 index 0000000000..74f0117376 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -0,0 +1,717 @@ +use std::collections::HashMap; + +use rustpython_parser::{ + ast::{ + Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, + StmtTryStar, StmtWhile, + }, + Parse, +}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +struct LineIndex { + newline_offsets: Vec, +} + +impl LineIndex { + fn new(source: &str) -> Self { + let mut offsets = vec![0]; + for (i, c) in source.char_indices() { + if c == '\n' { + offsets.push(i + 1); + } + } + Self { newline_offsets: offsets } + } + + fn line_of(&self, byte_offset: usize) -> usize { + match self.newline_offsets.binary_search(&byte_offset) { + Ok(line) => line + 1, + Err(line) => line, + } + } +} + +/// Maps task function name → optional external path (from `@task(path="...")`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `@task async def foo(...)` declarations. +fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { + let mut tasks = HashMap::new(); + for stmt in stmts { + if let Stmt::AsyncFunctionDef(func) = stmt { + for dec in &func.decorator_list { + match dec { + // @task (bare decorator) + Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { + tasks.insert(func.name.to_string(), None); + } + // @task(path="...") + Expr::Call(call) => { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + if id.as_str() == "task" { + let path = extract_task_path_kwarg(call); + tasks.insert(func.name.to_string(), path); + } + } + } + _ => {} + } + } + } + } + tasks +} + +/// Extract the `path=` keyword argument from a `@task(path="...")` call. +fn extract_task_path_kwarg(call: &ExprCall) -> Option { + for kw in &call.keywords { + if let Some(ref arg) = kw.arg { + if arg.as_str() == "path" { + if let Expr::Constant(c) = &kw.value { + if let rustpython_parser::ast::Constant::Str(s) = &c.value { + return Some(s.to_string()); + } + } + } + } + } + None +} + +struct WacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + line_index: LineIndex, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, + in_comprehension: bool, +} + +impl WacWalker { + fn new(source: &str, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + line_index: LineIndex::new(source), + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + in_comprehension: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn line_of_expr(&self, expr: &Expr) -> usize { + let offset = match expr { + Expr::Call(c) => c.range.start().to_usize(), + Expr::Await(a) => a.range.start().to_usize(), + Expr::Attribute(a) => a.range.start().to_usize(), + Expr::Name(n) => n.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + fn line_of_stmt(&self, stmt: &Stmt) -> usize { + let offset = match stmt { + Stmt::If(s) => s.range.start().to_usize(), + Stmt::For(s) => s.range.start().to_usize(), + Stmt::While(s) => s.range.start().to_usize(), + Stmt::Return(s) => s.range.start().to_usize(), + Stmt::Expr(s) => s.range.start().to_usize(), + Stmt::Try(s) => s.range.start().to_usize(), + Stmt::TryStar(s) => s.range.start().to_usize(), + Stmt::Assign(s) => s.range.start().to_usize(), + Stmt::AnnAssign(s) => s.range.start().to_usize(), + Stmt::FunctionDef(s) => s.range.start().to_usize(), + Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + _ => 0, + }; + self.line_index.line_of(offset) + } + + /// Check if an expression is a call to a known @task function + fn is_task_fn_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + return self.task_functions.contains_key(id.as_str()); + } + } + false + } + + /// Check if an expression is `asyncio.gather(...)` call + fn is_asyncio_gather_call(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = + call.func.as_ref() + { + if attr.as_str() == "gather" { + if let Expr::Name(ExprName { id, .. }) = value.as_ref() { + return id.as_str() == "asyncio"; + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &ExprCall) -> Option<(String, String)> { + if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { + let name = id.to_string(); + let script = self + .task_functions + .get(id.as_str()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + Some((name, script)) + } else { + None + } + } + + fn expr_to_source(expr: &Expr) -> String { + match expr { + Expr::Compare(c) => { + let left = Self::expr_to_source(&c.left); + if let Some(comparator) = c.comparators.first() { + let right = Self::expr_to_source(comparator); + let op = match c.ops.first() { + Some(rustpython_parser::ast::CmpOp::Gt) => ">", + Some(rustpython_parser::ast::CmpOp::Lt) => "<", + Some(rustpython_parser::ast::CmpOp::GtE) => ">=", + Some(rustpython_parser::ast::CmpOp::LtE) => "<=", + Some(rustpython_parser::ast::CmpOp::Eq) => "==", + Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", + Some(rustpython_parser::ast::CmpOp::In) => "in", + Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", + Some(rustpython_parser::ast::CmpOp::Is) => "is", + Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + None => "?", + }; + format!("{left} {op} {right}") + } else { + left + } + } + Expr::Subscript(s) => { + let value = Self::expr_to_source(&s.value); + let slice = Self::expr_to_source(&s.slice); + format!("{value}[{slice}]") + } + Expr::Attribute(a) => { + let value = Self::expr_to_source(&a.value); + format!("{value}.{}", a.attr) + } + Expr::Name(n) => n.id.to_string(), + Expr::Constant(c) => match &c.value { + rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), + rustpython_parser::ast::Constant::Int(i) => i.to_string(), + rustpython_parser::ast::Constant::Float(f) => f.to_string(), + rustpython_parser::ast::Constant::Bool(b) => b.to_string(), + rustpython_parser::ast::Constant::None => "None".to_string(), + _ => "...".to_string(), + }, + _ => "...".to_string(), + } + } + + /// Check if a statement body contains any task function calls (recursively) + fn body_contains_step(&self, body: &[Stmt]) -> bool { + for stmt in body { + if self.stmt_contains_step(stmt) { + return true; + } + } + false + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), + Stmt::Assign(a) => self.expr_contains_step(&a.value), + Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::While(s) => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } + Stmt::Try(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::TryStar(s) => { + self.body_contains_step(&s.body) + || self.body_contains_step(&s.orelse) + || self.body_contains_step(&s.finalbody) + || s.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }) + } + Stmt::Return(_) => false, + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_fn_call(expr) { + return true; + } + match expr { + Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), + Expr::Call(call) => { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return true; + } + if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(a)); + } + false + } + _ => false, + } + } + + /// Walk a list of statements, returning (first_node_id, last_node_id) + fn walk_body(&mut self, body: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in body { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), + Stmt::Assign(a) => self.walk_expr_stmt(&a.value), + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_stmt(stmt), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(ExprAwait { value, .. }) = expr { + // await task_fn(...) + if let Expr::Call(call) = value.as_ref() { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await asyncio.gather(task_fn(...), task_fn(...), ...) + if Self::is_asyncio_gather_call(value) { + if let Expr::Call(gather_call) = value.as_ref() { + return self.emit_parallel(gather_call, expr); + } + } + } + + // Bare task_fn() without await — validation error + if self.is_task_fn_call(expr) { + self.errors + .push(validation::error_missing_await(self.line_of_expr(expr))); + } + + None + } + + fn emit_step(&mut self, call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.line_of_expr(expr), + )); + return None; + } + if self.in_comprehension { + self.errors.push(validation::error_step_in_comprehension( + self.line_of_expr(expr), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(expr), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, gather_call: &ExprCall, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_try(self.line_of_expr(expr))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.line_of_expr(expr))); + return None; + } + + let line = self.line_of_expr(expr); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + for arg in &gather_call.args { + // Each arg should be task_fn(...) + if let Expr::Call(call) = arg { + if self.is_task_fn_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.line_of_expr(arg), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(&if_stmt.body); + let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + + if !has_steps_in_body && !has_steps_in_else { + return None; + } + + let line = self.line_index.line_of(if_stmt.range.start().to_usize()); + let condition_source = Self::expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let merge_id = format!("{branch_id}_merge"); + + let mut last_ids = Vec::new(); + + if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + if !if_stmt.orelse.is_empty() { + if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + Some((branch_node_id, merge_id)) + } + } + + fn walk_for(&mut self, for_stmt: &StmtFor) -> Option<(String, String)> { + if !self.body_contains_step(&for_stmt.body) { + return None; + } + + let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let iter_source = Self::expr_to_source(&for_stmt.iter); + + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_body(&for_stmt.body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &StmtWhile) -> Option<(String, String)> { + if self.body_contains_step(&while_stmt.body) { + let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_while(line)); + } + None + } + + fn walk_try(&mut self, try_stmt: &StmtTry) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.body) + || self.body_contains_step(&try_stmt.orelse) + || self.body_contains_step(&try_stmt.finalbody) + || try_stmt.handlers.iter().any(|h| match h { + rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + self.body_contains_step(&eh.body) + } + }); + + if has_steps { + let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + self.errors.push(validation::error_step_in_try(line)); + } + None + } + + fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { + let line = self.line_index.line_of(ret.range.start().to_usize()); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function parameters (no longer skips ctx) +fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { + let mut params = Vec::new(); + for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { + let name = arg_with_default.def.arg.to_string(); + let typ = arg_with_default + .def + .annotation + .as_ref() + .map(|ann| WacWalker::expr_to_source(ann)); + params.push(Param { name, typ }); + } + params +} + +pub fn parse_python_workflow(code: &str) -> Result> { + let ast = rustpython_parser::ast::Suite::parse(code, "") + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + + // First pass: collect @task functions + let task_functions = collect_task_functions(&ast); + + // Find the @workflow async def + let workflow_fn = ast.iter().find_map(|stmt| { + if let Stmt::AsyncFunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return Some(func); + } + } + // Also check non-async for error reporting + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + return None; // Will be reported as not-async below + } + } + None + }); + + // Check for non-async workflow function + let non_async_workflow = ast.iter().find_map(|stmt| { + if let Stmt::FunctionDef(func) = stmt { + let has_workflow_decorator = func.decorator_list.iter().any(|dec| { + if let Expr::Name(ExprName { id, .. }) = dec { + id.as_str() == "workflow" + } else { + false + } + }); + if has_workflow_decorator { + let line_index = LineIndex::new(code); + return Some(line_index.line_of(func.range.start().to_usize())); + } + } + None + }); + + if let Some(line) = non_async_workflow { + if workflow_fn.is_none() { + return Err(vec![validation::error_not_async(line)]); + } + } + + let workflow_fn = workflow_fn.ok_or_else(|| { + vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }] + })?; + + let params = extract_params(&workflow_fn.args); + let source_hash = compute_source_hash(code); + + let mut walker = WacWalker::new(code, task_functions); + walker.walk_body(&workflow_fn.body); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} + +trait ToUsize { + fn to_usize(self) -> usize; +} + +impl ToUsize for rustpython_parser::text_size::TextSize { + fn to_usize(self) -> usize { + u32::from(self) as usize + } +} diff --git a/backend/parsers/windmill-parser-wac/src/typescript.rs b/backend/parsers/windmill-parser-wac/src/typescript.rs new file mode 100644 index 0000000000..bd777749ea --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/typescript.rs @@ -0,0 +1,739 @@ +use std::collections::HashMap; + +use swc_common::{sync::Lrc, FileName, SourceMap, SourceMapper, Spanned}; +use swc_ecma_ast::*; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; + +use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; +use crate::validation::{self, CompileError}; + +/// Maps task function name → optional external path (from `task("f/path", ...)`) +type TaskFunctions = HashMap>; + +/// First pass: scan top-level `const foo = task(async (...) => {})` or +/// `const foo = task("f/path", async (...) => {})` declarations. +fn collect_task_functions(module: &Module) -> TaskFunctions { + let mut tasks = HashMap::new(); + for item in &module.body { + // const foo = task(async (...) => { ... }) + // const foo = task("f/path", async (...) => { ... }) + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + // export const foo = task(...) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(export)) = item { + if let Decl::Var(var_decl) = &export.decl { + for decl in &var_decl.decls { + if let (Some(name), Some(init)) = (extract_var_name(&decl.name), &decl.init) { + if let Some(path) = extract_task_call_info(init) { + tasks.insert(name, path); + } + } + } + } + } + } + tasks +} + +/// Extract variable name from a pattern (simple ident case) +fn extract_var_name(pat: &Pat) -> Option { + if let Pat::Ident(BindingIdent { id, .. }) = pat { + Some(id.sym.to_string()) + } else { + None + } +} + +/// Check if expr is `task(async fn)` or `task("path", async fn)`. +/// Returns Some(optional_path) if it is a task() call. +fn extract_task_call_info(expr: &Expr) -> Option> { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + if ident.sym.as_ref() == "task" { + // task("f/path", async fn) or task(async fn) + if call.args.len() == 2 { + // task("f/path", async fn) + let path = extract_string_lit(&call.args[0].expr); + return Some(path); + } else if call.args.len() == 1 { + // task(async fn) + return Some(None); + } + } + } + } + } + None +} + +struct TsWacWalker { + nodes: Vec, + edges: Vec, + errors: Vec, + node_counter: usize, + cm: Lrc, + task_functions: TaskFunctions, + in_try: bool, + in_while: bool, + in_nested_func: bool, +} + +impl TsWacWalker { + fn new(cm: Lrc, task_functions: TaskFunctions) -> Self { + Self { + nodes: Vec::new(), + edges: Vec::new(), + errors: Vec::new(), + node_counter: 0, + cm, + task_functions, + in_try: false, + in_while: false, + in_nested_func: false, + } + } + + fn next_id(&mut self) -> String { + let id = format!("step_{}", self.node_counter); + self.node_counter += 1; + id + } + + fn add_node(&mut self, node: DagNode) -> String { + let id = node.id.clone(); + self.nodes.push(node); + id + } + + fn add_edge(&mut self, from: &str, to: &str, label: Option) { + self.edges + .push(DagEdge { from: from.to_string(), to: to.to_string(), label }); + } + + fn span_line(&self, span: swc_common::Span) -> usize { + let loc = self.cm.lookup_char_pos(span.lo); + loc.line + } + + /// Check if expr is a call to a known task function + fn is_task_call(&self, expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + return self.task_functions.contains_key(ident.sym.as_ref()); + } + } + } + false + } + + /// Check if expr is `Promise.all([...])` + fn is_promise_all(expr: &Expr) -> bool { + if let Expr::Call(call) = expr { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Member(MemberExpr { obj, prop: MemberProp::Ident(prop), .. }) = + callee.as_ref() + { + if prop.sym.as_ref() == "all" { + if let Expr::Ident(ident) = obj.as_ref() { + return ident.sym.as_ref() == "Promise"; + } + } + } + } + } + false + } + + /// Extract step name and script from a task function call. + /// Name = function name, script = task_path or function name. + fn extract_step_info_from_task_call(&self, call: &CallExpr) -> Option<(String, String)> { + if let Callee::Expr(callee) = &call.callee { + if let Expr::Ident(ident) = callee.as_ref() { + let name = ident.sym.to_string(); + let script = self + .task_functions + .get(ident.sym.as_ref()) + .and_then(|p| p.clone()) + .unwrap_or_else(|| name.clone()); + return Some((name, script)); + } + } + None + } + + fn expr_to_source(&self, expr: &Expr) -> String { + let span = expr.span(); + self.cm + .span_to_snippet(span) + .unwrap_or_else(|_| "...".to_string()) + } + + fn body_contains_step(&self, stmts: &[Stmt]) -> bool { + stmts.iter().any(|s| self.stmt_contains_step(s)) + } + + fn stmt_contains_step(&self, stmt: &Stmt) -> bool { + match stmt { + Stmt::Expr(expr_stmt) => self.expr_contains_step(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => var_decl.decls.iter().any(|d| { + d.init + .as_ref() + .map_or(false, |init| self.expr_contains_step(init)) + }), + Stmt::If(if_stmt) => { + self.stmt_contains_step(&if_stmt.cons) + || if_stmt + .alt + .as_ref() + .map_or(false, |alt| self.stmt_contains_step(alt)) + } + Stmt::Block(block) => self.body_contains_step(&block.stmts), + Stmt::For(for_stmt) => self.stmt_contains_step(&for_stmt.body), + Stmt::ForIn(for_in) => self.stmt_contains_step(&for_in.body), + Stmt::ForOf(for_of) => self.stmt_contains_step(&for_of.body), + Stmt::While(while_stmt) => self.stmt_contains_step(&while_stmt.body), + Stmt::Try(try_stmt) => { + self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)) + } + Stmt::Return(ret) => ret + .arg + .as_ref() + .map_or(false, |arg| self.expr_contains_step(arg)), + _ => false, + } + } + + fn expr_contains_step(&self, expr: &Expr) -> bool { + if self.is_task_call(expr) { + return true; + } + match expr { + Expr::Await(await_expr) => self.expr_contains_step(&await_expr.arg), + Expr::Call(call) => { + if Self::is_promise_all(&Expr::Call(call.clone())) { + return call.args.iter().any(|a| self.expr_contains_step(&a.expr)); + } + false + } + Expr::Paren(p) => self.expr_contains_step(&p.expr), + _ => false, + } + } + + fn walk_body(&mut self, stmts: &[Stmt]) -> Option<(String, String)> { + let mut first_id: Option = None; + let mut prev_id: Option = None; + + for stmt in stmts { + if let Some((stmt_first, stmt_last)) = self.walk_stmt(stmt) { + if let Some(ref prev) = prev_id { + self.add_edge(prev, &stmt_first, None); + } + if first_id.is_none() { + first_id = Some(stmt_first); + } + prev_id = Some(stmt_last); + } + } + + match (first_id, prev_id) { + (Some(f), Some(l)) => Some((f, l)), + _ => None, + } + } + + fn walk_stmt(&mut self, stmt: &Stmt) -> Option<(String, String)> { + match stmt { + Stmt::Expr(expr_stmt) => self.walk_expr_stmt(&expr_stmt.expr), + Stmt::Decl(Decl::Var(var_decl)) => { + // const result = await task_fn(...) + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = self.walk_expr_stmt(init) { + return Some(result); + } + } + } + None + } + Stmt::If(if_stmt) => self.walk_if(if_stmt), + Stmt::For(for_stmt) => self.walk_for_stmt(for_stmt), + Stmt::ForIn(for_in) => self.walk_for_in(for_in), + Stmt::ForOf(for_of) => self.walk_for_of(for_of), + Stmt::While(while_stmt) => self.walk_while(while_stmt), + Stmt::Try(try_stmt) => self.walk_try(try_stmt), + Stmt::Block(block) => self.walk_body(&block.stmts), + Stmt::Return(ret) => self.walk_return(ret), + Stmt::Decl(Decl::Fn(_)) => { + if self.stmt_contains_step(stmt) { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(stmt.span()), + )); + } + None + } + _ => None, + } + } + + fn walk_expr_stmt(&mut self, expr: &Expr) -> Option<(String, String)> { + // await task_fn(...) + if let Expr::Await(await_expr) = expr { + if let Expr::Call(call) = await_expr.arg.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + return self.emit_step(call, expr); + } + } + // await Promise.all([task_fn(...), ...]) + if Self::is_promise_all(&await_expr.arg) { + if let Expr::Call(promise_call) = await_expr.arg.as_ref() { + return self.emit_parallel(promise_call, expr); + } + } + } + + // Bare task_fn() without await + if self.is_task_call(expr) { + self.errors + .push(validation::error_missing_await(self.span_line(expr.span()))); + } + + None + } + + fn emit_step(&mut self, call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + if self.in_nested_func { + self.errors.push(validation::error_step_in_nested_function( + self.span_line(expr.span()), + )); + return None; + } + + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(expr.span()), + }); + Some((node_id.clone(), node_id)) + } + + fn emit_parallel(&mut self, promise_call: &CallExpr, expr: &Expr) -> Option<(String, String)> { + if self.in_try { + self.errors + .push(validation::error_step_in_catch(self.span_line(expr.span()))); + return None; + } + if self.in_while { + self.errors + .push(validation::error_step_in_while(self.span_line(expr.span()))); + return None; + } + + let line = self.span_line(expr.span()); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::ParallelStart, + label: "parallel".to_string(), + line, + }); + + let mut step_ids = Vec::new(); + + // Promise.all takes an array as first argument + if let Some(first_arg) = promise_call.args.first() { + if let Expr::Array(ArrayLit { elems, .. }) = first_arg.expr.as_ref() { + for elem in elems.iter().flatten() { + if let Expr::Call(call) = elem.expr.as_ref() { + if self.is_task_call(&Expr::Call(call.clone())) { + let (name, script) = self + .extract_step_info_from_task_call(call) + .unwrap_or(("unknown".into(), "unknown".into())); + let step_id = self.next_id(); + let node_id = self.add_node(DagNode { + id: step_id.clone(), + node_type: DagNodeType::Step { name: name.clone(), script }, + label: name, + line: self.span_line(elem.expr.span()), + }); + self.add_edge(&start_node_id, &node_id, None); + step_ids.push(node_id); + } + } + } + } + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::ParallelEnd, + label: "join".to_string(), + line, + }); + + for step_id in &step_ids { + self.add_edge(step_id, &end_node_id, None); + } + + Some((start_node_id, end_node_id)) + } + + fn walk_if(&mut self, if_stmt: &IfStmt) -> Option<(String, String)> { + let has_steps_cons = self.stmt_contains_step(&if_stmt.cons); + let has_steps_alt = if_stmt + .alt + .as_ref() + .map_or(false, |a| self.stmt_contains_step(a)); + + if !has_steps_cons && !has_steps_alt { + return None; + } + + let line = self.span_line(if_stmt.span); + let condition_source = self.expr_to_source(&if_stmt.test); + + let branch_id = self.next_id(); + let branch_node_id = self.add_node(DagNode { + id: branch_id.clone(), + node_type: DagNodeType::Branch { condition_source }, + label: "if".to_string(), + line, + }); + + let mut last_ids = Vec::new(); + + // True branch + if let Some((true_first, true_last)) = self.walk_stmt(&if_stmt.cons) { + self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); + last_ids.push(true_last); + } else { + last_ids.push(branch_node_id.clone()); + } + + // False branch + if let Some(alt) = &if_stmt.alt { + if let Some((else_first, else_last)) = self.walk_stmt(alt) { + self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); + last_ids.push(else_last); + } else { + last_ids.push(branch_node_id.clone()); + } + } + + if last_ids.len() == 1 { + Some((branch_node_id, last_ids.into_iter().next().unwrap())) + } else { + let merge_id = format!("{branch_id}_merge"); + Some((branch_node_id, merge_id)) + } + } + + fn walk_for_stmt(&mut self, for_stmt: &ForStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_stmt.body) { + return None; + } + self.walk_loop_body(&for_stmt.body, for_stmt.span, "for") + } + + fn walk_for_in(&mut self, for_in: &ForInStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_in.body) { + return None; + } + let iter_source = self.expr_to_source(&for_in.right); + self.walk_loop_body_with_iter(&for_in.body, for_in.span, &iter_source) + } + + fn walk_for_of(&mut self, for_of: &ForOfStmt) -> Option<(String, String)> { + if !self.stmt_contains_step(&for_of.body) { + return None; + } + let iter_source = self.expr_to_source(&for_of.right); + self.walk_loop_body_with_iter(&for_of.body, for_of.span, &iter_source) + } + + fn walk_loop_body( + &mut self, + body: &Stmt, + span: swc_common::Span, + _label: &str, + ) -> Option<(String, String)> { + self.walk_loop_body_with_iter(body, span, "...") + } + + fn walk_loop_body_with_iter( + &mut self, + body: &Stmt, + span: swc_common::Span, + iter_source: &str, + ) -> Option<(String, String)> { + let line = self.span_line(span); + let start_id = self.next_id(); + let start_node_id = self.add_node(DagNode { + id: start_id.clone(), + node_type: DagNodeType::LoopStart { iter_source: iter_source.to_string() }, + label: "for".to_string(), + line, + }); + + if let Some((body_first, body_last)) = self.walk_stmt(body) { + self.add_edge(&start_node_id, &body_first, None); + self.add_edge(&body_last, &start_node_id, Some("next".to_string())); + } + + let end_id = self.next_id(); + let end_node_id = self.add_node(DagNode { + id: end_id.clone(), + node_type: DagNodeType::LoopEnd, + label: "end for".to_string(), + line, + }); + self.add_edge(&start_node_id, &end_node_id, Some("done".to_string())); + + Some((start_node_id, end_node_id)) + } + + fn walk_while(&mut self, while_stmt: &WhileStmt) -> Option<(String, String)> { + if self.stmt_contains_step(&while_stmt.body) { + self.errors.push(validation::error_step_in_while( + self.span_line(while_stmt.span), + )); + } + None + } + + fn walk_try(&mut self, try_stmt: &TryStmt) -> Option<(String, String)> { + let has_steps = self.body_contains_step(&try_stmt.block.stmts) + || try_stmt + .handler + .as_ref() + .map_or(false, |h| self.body_contains_step(&h.body.stmts)) + || try_stmt + .finalizer + .as_ref() + .map_or(false, |f| self.body_contains_step(&f.stmts)); + + if has_steps { + self.errors.push(validation::error_step_in_catch( + self.span_line(try_stmt.span), + )); + } + None + } + + fn walk_return(&mut self, ret: &ReturnStmt) -> Option<(String, String)> { + let line = self.span_line(ret.span); + let id = self.next_id(); + let node_id = self.add_node(DagNode { + id: id.clone(), + node_type: DagNodeType::Return, + label: "return".to_string(), + line, + }); + Some((node_id.clone(), node_id)) + } +} + +/// Extract workflow function params (no longer skips ctx) +fn extract_ts_params(params: &[swc_ecma_ast::Param], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for param in params { + let (name, typ) = match ¶m.pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + (name, typ) + } + _ => continue, + }; + result.push(Param { name, typ }); + } + result +} + +pub fn parse_ts_workflow(code: &str) -> Result> { + let cm: Lrc = Default::default(); + let fm = cm.new_source_file(FileName::Custom("workflow.ts".into()).into(), code.into()); + let lexer = Lexer::new( + Syntax::Typescript(TsSyntax::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + let module = parser + .parse_module() + .map_err(|e| vec![CompileError { message: format!("Parse error: {e:?}"), line: 0 }])?; + + // First pass: collect task functions + let task_functions = collect_task_functions(&module); + + // Find: export default workflow(async (...) => { ... }) + // or: export default workflow(async function(...) { ... }) + let mut workflow_body: Option<(&[Stmt], Vec)> = None; + + for item in &module.body { + // export default workflow(async (...) => { ... }) + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultExpr(export)) = item { + if let Some(result) = find_workflow_call(&export.expr, &cm) { + workflow_body = Some(result); + break; + } + } + // const wf = workflow(async (...) => { ... }); export default wf; + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDefaultDecl(export)) = item { + if let DefaultDecl::Fn(_) = &export.decl { + // `export default async function(...) { ... }` — not wrapped in workflow(), skip + } + } + if let ModuleItem::Stmt(Stmt::Decl(Decl::Var(var_decl))) = item { + for decl in &var_decl.decls { + if let Some(init) = &decl.init { + if let Some(result) = find_workflow_call(init, &cm) { + workflow_body = Some(result); + break; + } + } + } + } + } + + let (stmts, params) = workflow_body.ok_or_else(|| { + vec![CompileError { + message: "No workflow() wrapped async function found.".to_string(), + line: 0, + }] + })?; + + let source_hash = compute_source_hash(code); + + let mut walker = TsWacWalker::new(cm, task_functions); + walker.walk_body(stmts); + + if !walker.errors.is_empty() { + return Err(walker.errors); + } + + Ok(WorkflowDag { nodes: walker.nodes, edges: walker.edges, params, source_hash }) +} + +/// Find workflow(async (...) => { ... }) or workflow(async function(...) { ... }) +fn find_workflow_call<'a>(expr: &'a Expr, cm: &Lrc) -> Option<(&'a [Stmt], Vec)> { + if let Expr::Call(call) = expr { + // Check if callee is `workflow` + let is_workflow = match &call.callee { + Callee::Expr(callee_expr) => { + if let Expr::Ident(ident) = callee_expr.as_ref() { + ident.sym.as_ref() == "workflow" + } else { + false + } + } + _ => false, + }; + + if is_workflow { + if let Some(first_arg) = call.args.first() { + return extract_async_fn_body(&first_arg.expr, cm); + } + } + } + None +} + +fn extract_async_fn_body<'a>( + expr: &'a Expr, + cm: &Lrc, +) -> Option<(&'a [Stmt], Vec)> { + match expr { + Expr::Arrow(arrow) if arrow.is_async => { + let params = extract_arrow_params(&arrow.params, cm); + match &*arrow.body { + BlockStmtOrExpr::BlockStmt(block) => Some((&block.stmts, params)), + _ => None, + } + } + Expr::Fn(fn_expr) if fn_expr.function.is_async => { + let params = extract_ts_params(&fn_expr.function.params, cm); + fn_expr + .function + .body + .as_ref() + .map(|body| (body.stmts.as_slice(), params)) + } + Expr::Paren(p) => extract_async_fn_body(&p.expr, cm), + _ => None, + } +} + +/// Extract arrow function params (no longer skips ctx) +fn extract_arrow_params(pats: &[Pat], cm: &Lrc) -> Vec { + let mut result = Vec::new(); + for pat in pats { + match pat { + Pat::Ident(BindingIdent { id, type_ann, .. }) => { + let name = id.sym.to_string(); + let typ = type_ann.as_ref().map(|ann| { + cm.span_to_snippet(ann.type_ann.span()) + .unwrap_or_else(|_| "unknown".to_string()) + }); + result.push(Param { name, typ }); + } + _ => {} + } + } + result +} + +fn extract_string_lit(expr: &Expr) -> Option { + match expr { + Expr::Lit(Lit::Str(s)) => Some(s.value.to_string()), + Expr::Tpl(tpl) if tpl.exprs.is_empty() && tpl.quasis.len() == 1 => { + tpl.quasis.first().map(|q| q.raw.to_string()) + } + _ => None, + } +} + +fn compute_source_hash(code: &str) -> String { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(code.as_bytes()); + format!("{:x}", hasher.finalize()) +} diff --git a/backend/parsers/windmill-parser-wac/src/validation.rs b/backend/parsers/windmill-parser-wac/src/validation.rs new file mode 100644 index 0000000000..e3a57c6811 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/src/validation.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct CompileError { + pub message: String, + pub line: usize, +} + +impl std::fmt::Display for CompileError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "line {}: {}", self.line, self.message) + } +} + +pub fn error_step_in_try(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside try/except are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} + +pub fn error_step_in_while(line: usize) -> CompileError { + CompileError { + message: "Task calls inside while loops are not allowed. Use for loops instead." + .to_string(), + line, + } +} + +pub fn error_step_in_nested_function(line: usize) -> CompileError { + CompileError { + message: "Task calls inside nested functions, closures, or lambdas are not allowed." + .to_string(), + line, + } +} + +pub fn error_step_in_comprehension(line: usize) -> CompileError { + CompileError { message: "Task calls inside comprehensions are not allowed.".to_string(), line } +} + +pub fn error_not_async(line: usize) -> CompileError { + CompileError { message: "Workflow function must be async.".to_string(), line } +} + +pub fn error_missing_await(line: usize) -> CompileError { + CompileError { + message: + "Task calls must be awaited directly or used inside asyncio.gather()/Promise.all()." + .to_string(), + line, + } +} + +pub fn error_step_in_catch(line: usize) -> CompileError { + CompileError { + message: + "Task calls inside catch blocks are not allowed. Steps have built-in error handling." + .to_string(), + line, + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs new file mode 100644 index 0000000000..59f0b59f5c --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -0,0 +1,266 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::python::parse_python_workflow; + +#[test] +fn test_simple_sequential_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def load_data(data: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + await load_data(data=raw) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); // step0->step1, step1->return + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("str")); + + // Check first step + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + // Check second step + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + // Check return + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + + // Check source hash is non-empty + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(url: str): ... +@task +async def clean_data(data: list): ... +@task +async def compute_stats(data: list): ... +@task +async def load_to_warehouse(rows: list): ... + +@workflow +async def my_etl(url: str): + raw = await extract_data(url=url) + cleaned, stats = await asyncio.gather( + clean_data(data=raw), + compute_stats(data=raw), + ) + await load_to_warehouse(rows=cleaned) + return {"status": "done"} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def send_alert(msg: str): ... +@task +async def load_data(): ... + +@workflow +async def my_etl(count: int): + if count > 100: + await send_alert(msg="large") + await load_data() + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // Branch, notify step, load step, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_for_loop_workflow() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def process_item(item: str): ... + +@workflow +async def my_etl(items: list): + for item in items: + await process_item(item=item) + return {"done": True} +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::LoopEnd)); +} + +#[test] +fn test_reject_step_in_try() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + try: + await extract_data() + except Exception: + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("try/except")); +} + +#[test] +fn test_reject_step_in_while() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + while True: + await extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_non_async() { + let code = r#" +from wmill import workflow + +@workflow +def my_etl(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("async")); +} + +#[test] +fn test_reject_missing_await() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task +async def extract_data(): ... + +@workflow +async def my_etl(): + extract_data() +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_function() { + let code = r#" +async def my_func(): + pass +"#; + + let result = parse_python_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No @workflow")); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import asyncio +from wmill import workflow, task + +@task(path="f/external_script") +async def run_external(x: int): ... + +@workflow +async def my_wf(x: int): + result = await run_external(x=x) + return result +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // 1 step + 1 return (bare `return` is not a step node but walk_return creates one) + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wac/tests/ts_tests.rs b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs new file mode 100644 index 0000000000..949f326b90 --- /dev/null +++ b/backend/parsers/windmill-parser-wac/tests/ts_tests.rs @@ -0,0 +1,245 @@ +use windmill_parser_wac::dag::DagNodeType; +use windmill_parser_wac::typescript::parse_ts_workflow; + +#[test] +fn test_simple_sequential_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const load_data = task(async (data: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + await load_data(raw); + return { status: "done" }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 3); // 2 steps + 1 return + assert_eq!(dag.edges.len(), 2); + + // Check params (url — no ctx to skip) + assert_eq!(dag.params.len(), 1); + assert_eq!(dag.params[0].name, "url"); + assert_eq!(dag.params[0].typ.as_deref(), Some("string")); + + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "extract_data"); + assert_eq!(script, "extract_data"); + } + _ => panic!("expected Step node"), + } + + match &dag.nodes[1].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "load_data"); + assert_eq!(script, "load_data"); + } + _ => panic!("expected Step node"), + } + + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Return)); + assert!(!dag.source_hash.is_empty()); +} + +#[test] +fn test_parallel_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async (url: string) => {}); +const clean_data = task(async (data: any) => {}); +const compute_stats = task(async (data: any) => {}); +const load_to_warehouse = task(async (rows: any) => {}); + +export default workflow(async (url: string) => { + const raw = await extract_data(url); + + const [cleaned, stats] = await Promise.all([ + clean_data(raw), + compute_stats(raw), + ]); + + await load_to_warehouse(cleaned); + return { status: "done", rows: stats.rowCount }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // extract, ParallelStart, clean, stats, ParallelEnd, load, return = 7 + assert_eq!(dag.nodes.len(), 7); + + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[1].node_type, DagNodeType::ParallelStart)); + assert!(matches!(dag.nodes[2].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[3].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[4].node_type, DagNodeType::ParallelEnd)); + assert!(matches!(dag.nodes[5].node_type, DagNodeType::Step { .. })); + assert!(matches!(dag.nodes[6].node_type, DagNodeType::Return)); +} + +#[test] +fn test_conditional_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const send_alert = task(async (msg: string) => {}); +const load_data = task(async () => {}); + +export default workflow(async (count: number) => { + if (count > 100) { + await send_alert("large"); + } + await load_data(); + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // Branch, notify, load, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Branch { .. })); +} + +#[test] +fn test_for_of_ts_workflow() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const process_item = task(async (item: string) => {}); + +export default workflow(async (items: string[]) => { + for (const item of items) { + await process_item(item); + } + return { done: true }; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + // LoopStart, step, LoopEnd, return = 4 + assert_eq!(dag.nodes.len(), 4); + assert!(matches!( + dag.nodes[0].node_type, + DagNodeType::LoopStart { .. } + )); +} + +#[test] +fn test_reject_step_in_try_catch() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + try { + await extract_data(); + } catch (e) { + console.log(e); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("catch")); +} + +#[test] +fn test_reject_step_in_while_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + while (true) { + await extract_data(); + } +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("while")); +} + +#[test] +fn test_reject_missing_await_ts() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const extract_data = task(async () => {}); + +export default workflow(async () => { + extract_data(); +}); +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("awaited")); +} + +#[test] +fn test_no_workflow_wrapper() { + let code = r#" +export default async function main(ctx: any) { + return {}; +} +"#; + + let result = parse_ts_workflow(code); + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert!(errors[0].message.contains("No workflow()")); +} + +#[test] +fn test_variable_declaration_with_step() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const compute = task(async () => {}); + +export default workflow(async () => { + const result = await compute(); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + assert!(matches!(dag.nodes[0].node_type, DagNodeType::Step { .. })); +} + +#[test] +fn test_task_with_external_path() { + let code = r#" +import { workflow, task } from "windmill-client"; + +const run_external = task("f/external_script", async (x: number) => {}); + +export default workflow(async (x: number) => { + const result = await run_external(x); + return result; +}); +"#; + + let dag = parse_ts_workflow(code).expect("should parse"); + assert_eq!(dag.nodes.len(), 2); // step + return + match &dag.nodes[0].node_type { + DagNodeType::Step { name, script } => { + assert_eq!(name, "run_external"); + assert_eq!(script, "f/external_script"); + } + _ => panic!("expected Step node"), + } +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 1a0d425704..c4891c091b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -38,6 +38,7 @@ csharp-parser = [ "dep:windmill-parser-csharp"] nu-parser = [ "dep:windmill-parser-nu"] java-parser = [ "dep:windmill-parser-java"] ruby-parser = [ "dep:windmill-parser-ruby"] +wac-parser = [ "dep:windmill-parser-wac"] [dependencies] anyhow.workspace = true @@ -55,6 +56,7 @@ windmill-parser-csharp = { workspace = true, optional = true } windmill-parser-nu = { workspace = true, optional = true } windmill-parser-java = { workspace = true, optional = true } windmill-parser-ruby = { workspace = true, optional = true } +windmill-parser-wac = { workspace = true, optional = true } wasm-bindgen.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-wasm/build.nu b/backend/parsers/windmill-parser-wasm/build.nu index d954366113..cbed629c58 100755 --- a/backend/parsers/windmill-parser-wasm/build.nu +++ b/backend/parsers/windmill-parser-wasm/build.nu @@ -56,6 +56,12 @@ const targets = [ features: "ruby-parser", env: "tree-sitter", }, + { + ident: "wac", + desc: "Workflow-as-Code", + features: "wac-parser", + env: "default", + }, # ^^^ Add new entry here ^^^ ]; # NOTE: This is legacy command for building all, but it is not more used diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 61cc11d81d..634f2348e1 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -223,4 +223,11 @@ pub fn parse_assets_ansible(code: &str) -> String { } } +#[cfg(feature = "wac-parser")] +#[wasm_bindgen] +pub fn parse_workflow_as_code(code: &str, language: &str) -> String { + let result = windmill_parser_wac::parse_workflow(code, language); + serde_json::to_string(&result).unwrap_or_else(|_| "{\"type\": \"error\"}".to_string()) +} + // for related places search: ADD_NEW_LANG diff --git a/backend/test_wac_e2e.sh b/backend/test_wac_e2e.sh new file mode 100755 index 0000000000..4379ffde42 --- /dev/null +++ b/backend/test_wac_e2e.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# E2E test for WAC v2 workflow-as-code suspend/resume lifecycle +set -euo pipefail + +BASE_URL="${BASE_URL:-http://localhost:8070}" +TOKEN="${WM_TOKEN:-}" +WORKSPACE="dev" +TIMEOUT=60 # seconds + +# Get auth token if not set +if [ -z "$TOKEN" ]; then + TOKEN=$(curl -s "${BASE_URL}/api/auth/login" \ + -H "Content-Type: application/json" \ + -d '{"email":"admin@windmill.dev","password":"changeme"}' | tr -d '"') +fi + +echo "=== WAC v2 E2E Test ===" +echo "Base URL: $BASE_URL" +echo "" + +WAC_CODE='import { task, workflow } from "windmill-client@1.999.19"; + +const double = task(async (x: number): Promise => { + console.log("[double] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[double] END at " + new Date().toISOString()); + return x * 2; +}); + +const increment = task(async (x: number): Promise => { + console.log("[increment] START at " + new Date().toISOString()); + await new Promise(r => setTimeout(r, 2000)); + console.log("[increment] END at " + new Date().toISOString()); + return x + 1; +}); + +export const main = workflow(async (x: number = 10) => { + const [doubled, incremented] = await Promise.all([ + double(x), + increment(x), + ]); + const final_result = await double(incremented); + return { doubled, incremented, final_result }; +});' + +echo "Step 1: Submitting preview job..." +JOB_ID=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs/run/preview" \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $TOKEN" \ + -d "$(jq -n --arg code "$WAC_CODE" '{ + content: $code, + language: "bun", + args: {"x": 10} + }')" | tr -d '"') + +echo "Job ID: $JOB_ID" + +if [ -z "$JOB_ID" ] || [ "$JOB_ID" = "null" ]; then + echo "FAIL: Could not create job" + exit 1 +fi + +echo "" +echo "Step 2: Polling for completion (timeout: ${TIMEOUT}s)..." + +START=$SECONDS +LAST_STATUS="" +while true; do + ELAPSED=$((SECONDS - START)) + if [ $ELAPSED -gt $TIMEOUT ]; then + echo "FAIL: Timed out after ${TIMEOUT}s" + # Dump job state for debugging + echo "" + echo "=== Debug info ===" + echo "Parent job queue state:" + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until, canceled_by FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Child jobs:" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at" 2>/dev/null + echo "Completed children:" + psql "$DATABASE_URL" -c "SELECT id FROM completed_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + echo "Checkpoint:" + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + echo "Total child count:" + psql "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + # Check completed job + RESULT=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/completed/get_result/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null) + + if [ "$HTTP_CODE" = "200" ]; then + echo "Job completed in ${ELAPSED}s!" + echo "" + echo "Step 3: Checking result..." + echo "Result: $RESULT" + + # Validate + DOUBLED=$(echo "$RESULT" | jq -r '.doubled // empty') + INCREMENTED=$(echo "$RESULT" | jq -r '.incremented // empty') + FINAL=$(echo "$RESULT" | jq -r '.final_result // empty') + + PASS=true + if [ "$DOUBLED" != "20" ]; then + echo "FAIL: doubled = $DOUBLED, expected 20" + PASS=false + fi + if [ "$INCREMENTED" != "11" ]; then + echo "FAIL: incremented = $INCREMENTED, expected 11" + PASS=false + fi + if [ "$FINAL" != "22" ]; then + echo "FAIL: final_result = $FINAL, expected 22" + PASS=false + fi + + if $PASS; then + echo "PASS: All values correct!" + # Check no excessive child jobs + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + echo "Total child jobs created: $CHILD_COUNT (expected: 3)" + if [ "$CHILD_COUNT" -gt "3" ]; then + echo "WARN: More children than expected ($CHILD_COUNT > 3)" + fi + exit 0 + else + exit 1 + fi + fi + + # Show progress + STATUS=$(curl -s "${BASE_URL}/api/w/${WORKSPACE}/jobs_u/get/${JOB_ID}" \ + -H "Authorization: Bearer $TOKEN" 2>/dev/null | jq -r '.type // empty') + if [ "$STATUS" != "$LAST_STATUS" ]; then + echo " [${ELAPSED}s] Status: $STATUS" + LAST_STATUS="$STATUS" + fi + + # Check for runaway child creation + source /home/rfiszel/windmill__worktrees/workflows-as-code-v2/.env.local 2>/dev/null + CHILD_COUNT=$(psql -tA "$DATABASE_URL" -c "SELECT count(*) FROM v2_job WHERE parent_job = '$JOB_ID'::uuid" 2>/dev/null) + if [ "$CHILD_COUNT" -gt "10" ]; then + echo "FAIL: Runaway child creation detected! $CHILD_COUNT children (expected 3)" + echo "" + echo "=== Debug info ===" + psql "$DATABASE_URL" -c "SELECT id, running, suspend, suspend_until FROM v2_job_queue WHERE id = '$JOB_ID'::uuid" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT id, running, suspend, created_at FROM v2_job_queue WHERE parent_job = '$JOB_ID'::uuid ORDER BY created_at LIMIT 20" 2>/dev/null + psql "$DATABASE_URL" -c "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = '$JOB_ID'::uuid" 2>/dev/null + exit 1 + fi + + sleep 1 +done diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1750276778..8960dfb2a1 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -18803,7 +18803,6 @@ components: required: - path - summary - - description - content - language diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index ac5a9a306b..184602dc87 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -2255,12 +2255,13 @@ async fn resume_suspended_job_internal( let value = value.unwrap_or(serde_json::Value::Null); verify_suspended_secret(&w_id, &db, job_id, resume_id, &approver, secret).await?; - // Get flow info - works for both step-level (job_id is a step) and flow-level (job_id is the flow) - let (flow_info, is_flow_level) = get_flow_info_for_resume(job_id, &db).await?; + // Get flow info - works for step-level, flow-level, and WAC approval + let (flow_info, is_flow_level, is_wac) = get_flow_info_for_resume(job_id, &db).await?; // For step-level resumes, verify user auth and flow status // For flow-level resumes (pre-approvals), the flow might not be at a suspended step yet - if !is_flow_level { + // For WAC approvals, skip flow status checks (there is no flow) + if !is_flow_level && !is_wac { let parent_flow = GetQuery::new() .without_logs() .without_code() @@ -2322,6 +2323,16 @@ async fn resume_suspended_job_internal( ) .execute(&mut *tx) .await?; + } else if is_wac { + // WAC approval: decrement suspend counter directly on the WAC parent job + if flow_info.suspend > 0 { + sqlx::query!( + "UPDATE v2_job_queue SET suspend = GREATEST(suspend - 1, 0) WHERE id = $1", + flow_info.id, + ) + .execute(&mut *tx) + .await?; + } } else if is_flow_level { // For flow-level resumes, decrement the suspend counter if the flow is currently suspended // The approval will be matched when the worker checks for resumes (both step-level and flow-level) @@ -2479,10 +2490,15 @@ struct FlowInfo { email: Option, } -/// Get flow info from either a step job (by looking up its parent) or a flow job directly. -/// Returns (FlowInfo, is_flow_level) where is_flow_level indicates if job_id was a flow job. -async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool)> { - // Single query that determines if job_id is a flow or step, and fetches the appropriate flow info +/// Get flow info from either a step job (by looking up its parent), a flow job directly, +/// or a WAC workflow job (self-suspended for approval). +/// Returns (FlowInfo, is_flow_level, is_wac) where: +/// - is_flow_level: job_id was a flow job (pre-approval) +/// - is_wac: job_id is a WAC workflow suspended for approval (target is itself) +async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowInfo, bool, bool)> { + // Single query that determines if job_id is a flow, step, or WAC job, + // and fetches the appropriate suspended job info. + // For WAC jobs (no parent, not a flow), the job itself is the suspended target. let result = sqlx::query!( r#" WITH job_info AS ( @@ -2496,14 +2512,15 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI q.suspend AS "suspend!", j.runnable_path AS script_path, j.permissioned_as_email AS email, - (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!" + (ji.kind IN ('flow', 'flowpreview')) AS "is_flow_level!", + (ji.kind NOT IN ('flow', 'flowpreview') AND q.id = ji.id) AS "is_wac!" FROM job_info ji JOIN v2_job_queue q ON q.id = CASE WHEN ji.kind IN ('flow', 'flowpreview') THEN ji.id - ELSE ji.parent_job + ELSE COALESCE(ji.parent_job, ji.id) END JOIN v2_job j ON j.id = q.id - JOIN v2_job_status s ON s.id = q.id + LEFT JOIN v2_job_status s ON s.id = q.id FOR UPDATE OF q "#, job_id, @@ -2520,7 +2537,7 @@ async fn get_flow_info_for_resume(job_id: Uuid, db: &DB) -> error::Result<(FlowI email: Some(result.email), }; - Ok((flow_info, result.is_flow_level)) + Ok((flow_info, result.is_flow_level, result.is_wac)) } async fn get_suspended_flow_info<'c>( diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index eaf7d857c1..49dcf25450 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -78,6 +78,8 @@ pub enum Error { AIError(String), #[error("{0}")] AlreadyCompleted(String), + #[error("WAC job suspended: {0}")] + WacSuspended(String), #[error("Find python error: {0}")] FindPythonError(String), #[error("Problem with arguments: {0}")] @@ -108,6 +110,7 @@ impl Error { Self::JsonErr(_) => "JsonErr", Self::AIError(_) => "AIError", Self::AlreadyCompleted(_) => "AlreadyCompleted", + Self::WacSuspended(_) => "WacSuspended", Self::FindPythonError(_) => "FindPythonError", Self::ArgumentErr(_) => "ArgumentErr", Self::Generic(_, _) => "Generic", diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 806e3e99a2..d8cb3f4de4 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -818,7 +818,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, i64, Option), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -830,7 +830,7 @@ pub async fn add_completed_job( } let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers) = (|| { + let (opt_uuid, duration, _skip_downstream_error_handlers, wac_job_ids) = (|| { commit_completed_job( db, completed_job, @@ -866,7 +866,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, duration, None)); } #[cfg(feature = "cloud")] @@ -887,7 +887,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, duration, wac_job_ids)) } async fn commit_completed_job( @@ -902,7 +902,7 @@ async fn commit_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> windmill_common::error::Result<(Option, i64, bool)> { +) -> windmill_common::error::Result<(Option, i64, bool, Option)> { // let start = std::time::Instant::now(); let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1003,25 +1003,31 @@ async fn commit_completed_job( .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?; } + let mut wac_job_ids: Option = None; if !completed_job.is_flow_step() { if let Some(parent_job) = completed_job.parent_job { - let _ = sqlx::query_scalar!( - "UPDATE v2_job_status SET + // Only update WAC parents (v1 or v2). The WHERE condition skips + // non-WAC parents entirely (error handlers, run_script children, etc.). + // Also returns pending_steps.job_ids so WAC v2 child completion + // doesn't need a separate read. + let row = sqlx::query_scalar!( + r#"UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( jsonb_set( - COALESCE(workflow_as_code_status, '{}'::jsonb), + workflow_as_code_status, array[$1], COALESCE(workflow_as_code_status->$1, '{}'::jsonb) ), array[$1, 'duration_ms'], to_jsonb($2::bigint) ) - WHERE id = $3", + WHERE id = $3 AND workflow_as_code_status IS NOT NULL + RETURNING workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' AS "job_ids: serde_json::Value""#, &completed_job.id.to_string(), duration, parent_job ) - .execute(&mut *tx) + .fetch_optional(&mut *tx) .warn_after_seconds(10) .await .inspect_err(|e| { @@ -1029,7 +1035,10 @@ async fn commit_completed_job( "Could not update parent job `duration_ms` in workflow as code status: {}", e, ) - }); + }) + .ok() + .flatten(); + wac_job_ids = row.flatten(); } } // tracing::error!("Added completed job {:#?}", queued_job); @@ -1250,14 +1259,14 @@ async fn commit_completed_job( completed_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, duration, _skip_downstream_error_handlers)) + Ok((None, duration, _skip_downstream_error_handlers, wac_job_ids)) } async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool), Error>> { +) -> Option, i64, bool, Option), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index 3ba8c73257..afd5c42ba9 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -142,6 +142,13 @@ mount { rw: true } +mount { + src: "{JOB_DIR}/checkpoint.json" + dst: "/tmp/{LANG}/checkpoint.json" + is_bind: true + mandatory: false +} + mount { src: "{JOB_DIR}/result.json" dst: "/tmp/{LANG}/result.json" diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs index 7eb6bb6597..63d8aeaec3 100644 --- a/backend/windmill-worker/src/ai/image_handler.rs +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -2,8 +2,8 @@ use base64::Engine; use futures; use ulid; use windmill_common::{client::AuthedClient, error::Error}; -use windmill_types::s3::S3Object; use windmill_queue::MiniPulledJob; +use windmill_types::s3::S3Object; use crate::ai::types::*; diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 04f1b4b548..73010b5ba1 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -1,7 +1,5 @@ use async_trait::async_trait; -use windmill_common::{ - client::AuthedClient, error::Error, worker::Connection, -}; +use windmill_common::{client::AuthedClient, error::Error, worker::Connection}; use windmill_queue::MiniPulledJob; use windmill_types::s3::S3Object; diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index eebe09bc77..631a8ab113 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -20,8 +20,8 @@ use windmill_common::{ flow_status::AgentAction, flows::FlowModule, }; -use windmill_types::s3::S3Object; use windmill_parser::Typ; +use windmill_types::s3::S3Object; // Re-export shared types from windmill_common::ai_types pub use windmill_common::ai_types::{ @@ -1603,10 +1603,7 @@ mod tests { schema.sanitize_for_google(); - assert!( - schema.multiple_of.is_none(), - "multipleOf should be removed" - ); + assert!(schema.multiple_of.is_none(), "multipleOf should be removed"); } #[test] @@ -1639,7 +1636,10 @@ mod tests { assert!(schema.default.is_none()); let value_prop = schema.properties.as_ref().unwrap().get("value").unwrap(); - assert!(value_prop.default.is_none(), "nested default should be removed"); + assert!( + value_prop.default.is_none(), + "nested default should be removed" + ); assert!( value_prop.exclusive_minimum.is_none(), "nested exclusiveMinimum should be removed" @@ -1652,7 +1652,10 @@ mod tests { value_prop.multiple_of.is_none(), "nested multipleOf should be removed" ); - assert!(value_prop.r#const.is_none(), "nested const should be removed"); + assert!( + value_prop.r#const.is_none(), + "nested const should be removed" + ); assert!(schema.properties.is_some()); assert!(matches!(&schema.r#type, Some(SchemaType::Single(t)) if t == "object")); diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 2f748fe22c..b092e135b6 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -6,9 +6,9 @@ use reqwest::Client; use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; +use windmill_object_store::convert_json_line_stream; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, parse_sql_statement_named_params, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 6ada66c4bb..16900dad8b 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1051,6 +1051,37 @@ pub async fn handle_bun_job( let apply_preprocessor = job.flow_step_id.as_deref() != Some("preprocessor") && job.preprocessed == Some(false); + let is_wac_v2 = main_override.is_none() && crate::wac_executor::is_wac_v2_ts(inner_content); + + // For WAC v2, inject variable names into unnamed task() calls so the + // runtime can use them for step naming (timeline, graph). + // `const double = task(async ...` → `const double = task("double", async ...` + // Also handles: export const, let, var, and optional generic type parameters. + // Skips calls that already have a string argument: `task("path", async ...` + let inner_content = if is_wac_v2 { + use regex::Regex; + use std::borrow::Cow; + lazy_static::lazy_static! { + static ref TASK_RE: Regex = + Regex::new(r#"(?m)((?:export\s+)?(?:const|let|var)\s+)(\w+)(\s*=\s*task\s*(?:<[^>]*>)?\s*\(\s*)(async\b)"#).unwrap(); + } + let replaced = TASK_RE.replace_all(inner_content, r#"${1}${2}${3}"${2}", ${4}"#); + match replaced { + Cow::Borrowed(_) => inner_content.to_string(), + Cow::Owned(s) => s, + } + } else { + inner_content.to_string() + }; + let inner_content = inner_content.as_str(); + + // WAC v2 scripts can't use bundle caching because the wrapper imports + // windmill-client from node_modules, which isn't available in bundle mode + if is_wac_v2 && has_bundle_cache { + has_bundle_cache = false; + let _ = write_file(job_dir, "main.ts", inner_content)?; + } + let mut format = BundleFormat::Cjs; if has_bundle_cache { let target; @@ -1184,13 +1215,20 @@ pub async fn handle_bun_job( return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); - let args = windmill_parser_ts::parse_deno_signature( - inner_content, - true, - false, - main_override.map(ToString::to_string), - )? - .args; + let args = if is_wac_v2 { + // For WAC v2, try to parse "main" args; if that fails, try the default export + windmill_parser_ts::parse_deno_signature(inner_content, true, false, None) + .unwrap_or_default() + .args + } else { + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + false, + main_override.map(ToString::to_string), + )? + .args + }; let pre_args = if apply_preprocessor { Some( @@ -1227,6 +1265,14 @@ pub async fn handle_bun_job( // we cannot use Bun.read and Bun.write because it results in an EBADF error on cloud let main_name = main_override.unwrap_or("main"); + // For WAC child jobs where the parser can't find params (task-wrapped consts), + // fall back to passing arg values directly (filtering out internal fields) + let child_spread = if spread.is_empty() && main_override.is_some() { + "Object.values(Object.fromEntries(Object.entries(args).filter(([k]) => !k.startsWith('_'))))".to_string() + } else { + "argsObjToArr(args)".to_string() + }; + let main_import = if codebase.is_some() || has_bundle_cache { "./main.js" } else { @@ -1250,8 +1296,116 @@ pub async fn handle_bun_job( "".to_string() }; - let wrapper_content = format!( - r#" + let wac_spread = if spread.is_empty() { + "Object.values(args)".to_string() + } else { + format!("argsObjToArr(args)") + }; + + let wrapper_content = if is_wac_v2 { + format!( + r#" +import * as Main from "{main_import}"; +import {{ WorkflowCtx, StepSuspend, setWorkflowCtx }} from "windmill-client"; + +import * as fs from "fs/promises"; + +let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse); +const checkpoint = JSON.parse(await fs.readFile('checkpoint.json', {{ encoding: 'utf8' }})); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} + +BigInt.prototype.toJSON = function () {{ + return this.toString(); +}}; + +// Find the workflow entrypoint (export default) +let workflowFn = Main.default; +if (!workflowFn || !workflowFn._is_workflow) {{ + for (const key of Object.keys(Main)) {{ + if (Main[key]?._is_workflow) {{ + workflowFn = Main[key]; + break; + }} + }} +}} +if (!workflowFn) {{ + throw new Error("No workflow() entrypoint found. Wrap your main function with workflow()."); +}} + +async function run() {{ + {dates} + {preprocessor} + const argsArr = {wac_spread}; + + const ctx = new WorkflowCtx(checkpoint); + setWorkflowCtx(ctx); + + try {{ + const result = await workflowFn(...argsArr); + setWorkflowCtx(null); + // Flush any unawaited tasks (e.g. forgotten await on last statement) + const trailing = ctx._flushPending(); + if (trailing.length > 0) {{ + return {{ type: "dispatch", mode: trailing.length > 1 ? "parallel" : "sequential", steps: trailing }}; + }} + return {{ type: "complete", result: result ?? null }}; + }} catch (e) {{ + setWorkflowCtx(null); + if (e?.name === "StepSuspend" || e instanceof StepSuspend) {{ + const dispatch = e.dispatchInfo ?? e.dispatch_info ?? {{}}; + if (dispatch.mode === "step_complete") {{ + return {{ type: "complete", result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "inline_checkpoint") {{ + return {{ type: "inline_checkpoint", key: dispatch.key, result: dispatch.result ?? null }}; + }} + if (dispatch.mode === "approval") {{ + return {{ type: "approval", key: dispatch.key, timeout: dispatch.timeout, form: dispatch.form }}; + }} + if (dispatch.mode === "sleep") {{ + return {{ type: "sleep", key: dispatch.key, seconds: dispatch.seconds }}; + }} + return {{ type: "dispatch", mode: dispatch.mode ?? "sequential", steps: dispatch.steps ?? [] }}; + }} + throw e; + }} +}} + +try {{ + const output = await run(); + const output_json = JSON.stringify(output, (key, value) => + typeof value === 'undefined' ? null : value + ); + await fs.writeFile("result.json", output_json); + process.exit(0); +}} catch(e) {{ + console.error(e); + let err = {{ message: e.message, name: e.name, stack: e.stack }}; + let step_id = process.env.WM_FLOW_STEP_ID; + if (step_id) {{ + err["step_id"] = step_id; + }} + const extra = {{}}; + Object.getOwnPropertyNames(e).forEach((key) => {{ + if (['line', 'name', 'stack', 'column', 'message', 'sourceURL', 'originalLine', 'originalColumn'].includes(key)) {{ + return; + }} + extra[key] = e[key]; + }}); + if (Object.keys(extra).length > 0) {{ + err["extra"] = extra; + }} + await fs.writeFile("result.json", JSON.stringify(err)); + process.exit(1); +}} + "#, + ) + } else { + format!( + r#" import * as Main from "{main_import}"; import * as fs from "fs/promises"; @@ -1273,11 +1427,14 @@ BigInt.prototype.toJSON = function () {{ async function run() {{ {dates} {preprocessor} - const argsArr = argsObjToArr(args); + // If the entrypoint has no parsed params (spread is empty), pass values directly + // This handles WAC child jobs where tasks are const-wrapped functions + const argsArr = {child_spread}; if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{ throw new Error("{main_name} function is missing"); }} - let res = await Main.{main_name}(...argsArr); + let entrypoint = Main.{main_name}; + let res = await entrypoint(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); @@ -1311,7 +1468,8 @@ try {{ process.exit(1); }} "#, - ); + ) + }; write_file(job_dir, "wrapper.mjs", &wrapper_content)?; Ok(()) as error::Result<()> }; @@ -1336,6 +1494,7 @@ try {{ && !annotation.nobundling && !*DISABLE_BUNDLING && !codebase.is_some() + && !is_wac_v2 && (maybe_lock.get_lock().is_some() || annotation.native); let write_loader_f = async { @@ -1381,6 +1540,23 @@ try {{ write_wrapper_f, write_loader_f )?; + + // For WAC v2, write checkpoint.json before bun runs + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + if !codebase.is_some() && !has_bundle_cache { if build_cache { generate_bun_bundle( @@ -1476,7 +1652,7 @@ try {{ let result = crate::js_eval::eval_fetch_timeout( env_code, - inner_content.clone(), + inner_content.to_string(), js_code, job_args, job.script_entrypoint_override.clone(), @@ -1679,7 +1855,766 @@ try {{ })?; *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend + if is_wac_v2 { + return handle_wac_v2_output(result, job, conn).await; + } + + Ok(result) +} + +/// Handle WAC v2 output after bun/python exits. Parse result as WacOutput, +/// dispatch child jobs on suspend, or return the final result. +pub async fn handle_wac_v2_output( + result: Box, + job: &MiniPulledJob, + conn: &Connection, +) -> error::Result> { + use crate::wac_executor::{ + add_completed_step, load_checkpoint, parse_wac_output, update_checkpoint_for_dispatch, + WacOutput, + }; + use serde_json::Value; + use windmill_common::get_latest_flow_version_info_for_path; + use windmill_common::jobs::{script_path_to_payload, JobKind, JobPayload, RawCode}; + use windmill_common::runnable_settings::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + }; + use windmill_queue::{push, PushArgs, PushIsolationLevel}; + + let output = parse_wac_output(&result)?; + + match output { + WacOutput::Complete { result: value } => { + // Workflow completed — return the inner result value + let raw = serde_json::value::to_raw_value(&value).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize WAC result: {e}")) + })?; + Ok(raw) + } + WacOutput::Dispatch { mode, steps } => { + if steps.is_empty() { + return Err(error::Error::internal_err( + "WAC v2 dispatch with no steps — this is a bug in the workflow SDK".to_string(), + )); + } + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 dispatch requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation: detect if code changed between replays + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + let num_steps = steps.len(); + + tracing::info!( + job_id = %job.id, + mode = %mode, + num_steps = num_steps, + steps = ?steps.iter().map(|s| &s.name).collect::>(), + "WAC v2 dispatching child jobs" + ); + + // Create child jobs for each step. + // Each child re-runs the full workflow with a checkpoint containing + // _executing_key = step_key, so only that step runs its inner function. + // + // IMPORTANT: To prevent a race condition where a fast child completes + // before the parent is suspended, we: + // 1. Pre-generate child UUIDs + // 2. Save checkpoint + suspend parent + seed child checkpoints + // 3. THEN push the child jobs (making them visible to workers) + + // Read the parent's original args for the child jobs + let parent_args: HashMap> = { + let stored: serde_json::Map = checkpoint.input_args.clone(); + if stored.is_empty() { + // First dispatch — read from the parent job's args + let row: Option = sqlx::query_scalar( + "SELECT args FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let args_val = row.unwrap_or(Value::Object(Default::default())); + if let Value::Object(map) = args_val { + // Store for future re-runs + checkpoint.input_args = map.clone(); + map.into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } else { + HashMap::new() + } + } else { + stored + .into_iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(&v).map_err(|e| { + error::Error::internal_err(format!( + "Failed to serialize arg '{k}': {e}" + )) + })?; + Ok((k, raw)) + }) + .collect::>>()? + } + }; + + // Pre-generate child UUIDs so we can save them in the checkpoint + // before the children become visible to workers. + // Validate key uniqueness — duplicate keys would cause one child's + // UUID to be overwritten in the job_ids map, making it unmappable + // on completion (the parent would hang). + { + let mut seen_keys = std::collections::HashSet::new(); + for s in &steps { + if !seen_keys.insert(&s.key) { + return Err(error::Error::internal_err(format!( + "WAC v2 duplicate step key '{}' — each task call must produce a unique key", + s.key + ))); + } + } + } + let job_ids: Vec<(String, Uuid)> = steps + .iter() + .map(|s| (s.key.clone(), ulid::Ulid::new().into())) + .collect(); + + // Resolve job_payload once (same for all children since they re-run + // the parent script) + let job_payload_template = match job.kind { + JobKind::Script => { + if let Some(hash) = job.runnable_id { + Ok(JobPayload::ScriptHash { + hash, + path: job.runnable_path.clone().unwrap_or_default(), + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + priority: job.priority, + apply_preprocessor: false, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), + }) + } else { + Err(error::Error::internal_err( + "WAC v2 Script job missing runnable_id".to_string(), + )) + } + } + JobKind::Preview => { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT raw_code, raw_lock FROM v2_job WHERE id = $1 AND workspace_id = $2", + ) + .bind(&job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + let (code, lock) = row.unwrap_or_default(); + Ok(JobPayload::Code(RawCode { + content: code.unwrap_or_default(), + path: job.runnable_path.clone(), + hash: None, + language: job.script_lang.unwrap_or(ScriptLang::Bun), + lock: lock, + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), + })) + } + _ => Err(error::Error::internal_err(format!( + "WAC v2 unsupported job kind: {:?}", + job.kind + ))), + }?; + + // Step 1: Save checkpoint, suspend parent, and seed child checkpoints + // in a single transaction — all BEFORE children become visible. + { + let mut tx = db.begin().await?; + + // Update checkpoint with pending steps + update_checkpoint_for_dispatch(&mut checkpoint, &steps, &mode, &job_ids); + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Store per-child-job info for the WorkflowTimeline UI + for (step, (_, child_id)) in steps.iter().zip(job_ids.iter()) { + let child_id_str = child_id.to_string(); + let timeline_val = serde_json::json!({ + "scheduled_for": chrono::Utc::now().to_rfc3339(), + "name": step.key, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + ARRAY[$2], + $3 + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&child_id_str) + .bind(&timeline_val) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to update WAC timeline status: {e}" + )) + })?; + } + + // Suspend parent before children become visible. + // Keep running = true so the normal pull query ignores it. + // The suspended pull query picks it up when suspend reaches 0 + // (it checks: suspend_until IS NOT NULL AND suspend <= 0). + let suspend_count = num_steps as i32; + sqlx::query!( + "UPDATE v2_job_queue SET suspend = $2, suspend_until = now() + interval '14 day' WHERE id = $1", + job.id, + suspend_count, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to suspend WAC parent job {}: {e}", + job.id + )) + })?; + + tx.commit().await?; + } + + // Step 2: Push child jobs (now visible to workers). + // Parent is already suspended, so child completions are safe. + // Track successfully pushed children so we can cancel them on + // partial failure (e.g. pushing child 3 of 5 fails). + let mut pushed_ids: Vec = Vec::with_capacity(num_steps); + let push_result: error::Result<()> = async { + for (step, (_, child_uuid)) in steps.iter().zip(job_ids.iter()) { + // Resolve job payload based on dispatch_type + let (job_payload, child_args, is_external) = match step.dispatch_type.as_str() { + "script" => { + // Resolve script path to job payload (handles hash, lang, etc.) + let (payload, _, _, _, _) = script_path_to_payload( + &step.script, + None, // no authed db for background workers + db.clone(), + &job.workspace_id, + Some(true), // skip preprocessor + ) + .await?; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + "flow" => { + let flow_info = get_latest_flow_version_info_for_path( + None, + db, + &job.workspace_id, + &step.script, + true, + ) + .await?; + let payload = JobPayload::Flow { + path: step.script.clone(), + dedicated_worker: flow_info.dedicated_worker, + apply_preprocessor: false, + version: flow_info.version, + }; + let step_args: HashMap> = step + .args + .iter() + .map(|(k, v)| { + let raw = serde_json::value::to_raw_value(v).unwrap(); + (k.clone(), raw) + }) + .collect(); + (payload, step_args, true) + } + _ => { + // "inline" — re-run parent with _executing_key + (job_payload_template.clone(), parent_args.clone(), false) + } + }; + + let push_args = PushArgs { args: &child_args, extra: None }; + + // Apply step-level overrides to payload (cache, concurrency) + let mut job_payload = job_payload; + if let Some(cache_ttl) = step.cache_ttl { + match &mut job_payload { + JobPayload::ScriptHash { cache_ttl: ref mut ct, .. } => { + *ct = Some(cache_ttl) + } + JobPayload::Code(ref mut code) => code.cache_ttl = Some(cache_ttl), + _ => {} + } + } + if step.concurrent_limit.is_some() + || step.concurrency_key.is_some() + || step.concurrency_time_window_s.is_some() + { + match &mut job_payload { + JobPayload::ScriptHash { concurrency_settings: ref mut cs, .. } => { + if let Some(limit) = step.concurrent_limit { + cs.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + cs.concurrency_key = Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + cs.concurrency_time_window_s = Some(window); + } + } + JobPayload::Code(ref mut code) => { + if let Some(limit) = step.concurrent_limit { + code.concurrency_settings.concurrent_limit = Some(limit); + } + if let Some(ref key) = step.concurrency_key { + code.concurrency_settings.custom_concurrency_key = + Some(key.clone()); + } + if let Some(window) = step.concurrency_time_window_s { + code.concurrency_settings.concurrency_time_window_s = + Some(window); + } + } + _ => {} + } + } + + let (_, mut tx) = push( + db, + PushIsolationLevel::IsolatedRoot(db.clone()), + &job.workspace_id, + job_payload, + push_args, + &job.created_by, + &job.permissioned_as_email, + job.permissioned_as.clone(), + None, + None, + None, + Some(job.id), // parent_job + job.root_job.or(Some(job.id)), // root_job + job.flow_innermost_root_job, + Some(*child_uuid), // pre-generated job_id + false, // is_flow_step + false, // same_worker + None, // pre_run_error + job.visible_to_owner, + step.tag.clone().or_else(|| Some(job.tag.clone())), + step.timeout.or(job.timeout), + None, // flow_step_id + step.priority, // priority_override + None, // authed + false, // running + None, // end_user_email + None, // trigger + None, // suspended_mode + ) + .await?; + + // Seed child checkpoint only for inline tasks (they need + // _executing_key to know which step to run). External + // scripts/flows don't need a WAC checkpoint. + if !is_external { + let child_checkpoint_json = serde_json::json!({ + "completed_steps": &checkpoint.completed_steps, + "_executing_key": &step.key, + }); + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(child_uuid) + .bind(&child_checkpoint_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to seed child checkpoint: {e}" + )) + })?; + } + + tx.commit().await.map_err(|e| { + error::Error::internal_err(format!("Failed to commit child push: {e}")) + })?; + + pushed_ids.push(*child_uuid); + + tracing::info!( + parent_job = %job.id, + child_job = %child_uuid, + step_name = %step.name, + step_key = %step.key, + "WAC v2 dispatched child job" + ); + } + Ok(()) + } + .await; + + if let Err(e) = push_result { + tracing::error!( + job_id = %job.id, + error = %e, + pushed_count = pushed_ids.len(), + total_count = num_steps, + "WAC v2 failed to push child jobs, cleaning up" + ); + + // Cancel already-pushed children so they don't complete and + // corrupt the checkpoint (they'd decrement suspend on a parent + // that's about to be unsuspended and re-run). + for child_id in &pushed_ids { + let _ = sqlx::query!( + "UPDATE v2_job_queue SET canceled_by = $2, canceled_reason = $3 WHERE id = $1", + child_id, + "system", + "WAC dispatch failed: not all children could be pushed", + ) + .execute(db) + .await; + } + + // Clear pending_steps from checkpoint so the parent doesn't + // think children are outstanding when it re-runs. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&job.id) + .execute(db) + .await; + + // Unsuspend parent so the error propagates instead of a 14-day hang + let _ = sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + job.id, + ) + .execute(db) + .await; + return Err(e); + } + + tracing::info!( + job_id = %job.id, + num_steps = num_steps, + "WAC v2 parent job suspended" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for {} child job(s)", + job.id, num_steps + ))) + } + WacOutput::Approval { key, timeout, form } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 approval requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let timeout_secs = timeout.unwrap_or(1800) as f64; + + // Mark this step as pending approval + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "approval".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Store approval form metadata for the approval page endpoint + let approval_meta = serde_json::json!({ + "key": key, + "form": form, + "timeout": timeout_secs as u32, + }); + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + COALESCE(workflow_as_code_status, '{}'::jsonb), + '{_approval}', + $2::jsonb + ) WHERE id = $1", + ) + .bind(&job.id) + .bind(&approval_meta) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save approval meta: {e}")) + })?; + + // Suspend parent with suspend=1 (waiting for 1 approval event) + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + timeout_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + approval_key = %key, + timeout_secs = timeout_secs, + "WAC v2 parent job suspended waiting for approval" + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} suspended waiting for approval (key: {})", + job.id, key + ))) + } + WacOutput::Sleep { key, seconds } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 sleep requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + let sleep_secs = seconds.max(1) as f64; + + // Mark this step as pending sleep + checkpoint.pending_steps = Some(crate::wac_executor::WacPendingSteps { + mode: "sleep".to_string(), + keys: vec![key.clone()], + job_ids: serde_json::Map::new(), + }); + + let mut tx = db.begin().await?; + + // Save checkpoint + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::internal_err(format!("Failed to save checkpoint: {e}")))?; + + // Suspend parent — it will auto-resume when suspend_until passes. + // Use suspend=1 (not 0) so the suspended pull query only picks it up + // when `suspend_until <= now()`, not via `suspend <= 0`. + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 1, suspend_until = now() + make_interval(secs => $2) WHERE id = $1", + job.id, + sleep_secs, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + tracing::info!( + job_id = %job.id, + sleep_key = %key, + sleep_secs = sleep_secs, + "WAC v2 parent job sleeping for {}s", + sleep_secs + ); + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} sleeping for {}s (key: {})", + job.id, seconds, key + ))) + } + WacOutput::InlineCheckpoint { key, result: value } => { + let db = match conn { + Connection::Sql(db) => db, + _ => { + return Err(error::Error::internal_err( + "WAC v2 inline checkpoint requires SQL connection".to_string(), + )) + } + }; + + let mut checkpoint = load_checkpoint(db, &job.id).await?; + + // Source hash validation (same as Dispatch path) + let current_hash = job.runnable_id.map(|h| h.0.to_string()).unwrap_or_default(); + if !current_hash.is_empty() { + if checkpoint.source_hash.is_empty() { + checkpoint.source_hash = current_hash.clone(); + } else if checkpoint.source_hash != current_hash { + return Err(error::Error::ExecutionErr( + "Workflow source code changed between replays. \ + Cannot safely resume from checkpoint — step keys may have shifted. \ + Please restart this workflow." + .to_string(), + )); + } + } + + tracing::info!( + job_id = %job.id, + step_key = %key, + "WAC v2 inline checkpoint — persisting step result" + ); + + add_completed_step(&mut checkpoint, &key, value); + + // Save checkpoint + reset running in a single transaction + { + let mut tx = db.begin().await?; + let status_json = serde_json::to_value(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(&job.id) + .bind(&status_json) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to save WAC checkpoint: {e}")) + })?; + + // Reset running=false so the job is immediately eligible for pickup. + // Unlike dispatch (which sets suspend>0), inline checkpoints don't suspend — + // the job should be re-run right away to continue past the cached step. + sqlx::query!( + "UPDATE v2_job_queue SET running = false, started_at = null WHERE id = $1", + job.id, + ) + .execute(&mut *tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Failed to reset running state for inline checkpoint: {e}" + )) + })?; + + tx.commit().await?; + } + + Err(error::Error::WacSuspended(format!( + "WAC v2 job {} inline checkpoint for step {}", + job.id, key + ))) + } + } } pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index f6d752558d..2fe56f50b9 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -73,6 +73,7 @@ mod universal_pkg_installer; #[cfg(feature = "private")] mod volume_ee; mod volume_oss; +pub mod wac_executor; mod worker; mod worker_flow; mod worker_lockfiles; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 320bcb3e40..c74feb0a0a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -567,6 +567,9 @@ pub async fn handle_python_job( let annotations = PythonAnnotations::parse(inner_content); + let is_wac_v2 = job.script_entrypoint_override.is_none() + && crate::wac_executor::is_wac_v2_py(inner_content); + if annotations.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(Error::ExecutionErr( "Script has #sandbox annotation but nsjail is not available on this worker. \ @@ -677,8 +680,68 @@ pub async fn handle_python_job( String::new() }; let main_override = main_name.unwrap_or_else(|| "main".to_string()); - let wrapper_content: String = format!( - r#" + let wrapper_content: String = if is_wac_v2 { + format!( + r#" +import os +import json +{import_loader} +{import_base64} +{import_datetime} +import traceback +import sys +from {module_dir_dot} import {last} as inner_script +from wmill.client import _run_workflow + +with open("args.json") as f: + kwargs = json.load(f, strict=False) +args = {{}} +{transforms} + +with open("checkpoint.json") as f: + checkpoint = json.load(f, strict=False) + +result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") + +# Find the @workflow-decorated function +workflow_fn = None +for name in dir(inner_script): + obj = getattr(inner_script, name) + if callable(obj) and getattr(obj, '_is_workflow', False): + workflow_fn = obj + break + +if workflow_fn is None: + raise ValueError("No @workflow function found in script") + +for k, v in list(args.items()): + if v == '': + del args[k] + +try: + output = _run_workflow(workflow_fn, checkpoint, args) + output_json = json.dumps(output, separators=(',', ':'), default=str) + with open(result_json, 'w') as f: + f.write(output_json) +except BaseException as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + tb = traceback.format_tb(exc_traceback) + with open(result_json, 'w') as f: + err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} + extra = e.__dict__ + if extra and len(extra) > 0: + err['extra'] = extra + flow_node_id = os.environ.get('WM_FLOW_STEP_ID') + if flow_node_id: + err['step_id'] = flow_node_id + err_json = json.dumps(err, separators=(',', ':'), default=str).replace('\n', '') + f.write(err_json) + sys.exit(1) +"#, + ) + } else { + format!( + r#" import os import json {import_loader} @@ -751,9 +814,26 @@ except BaseException as e: f.write(err_json) sys.exit(1) "#, - ); + ) + }; write_file(job_dir, "wrapper.py", &wrapper_content)?; + // For WAC v2, write checkpoint.json before python runs. + if is_wac_v2 { + if let Connection::Sql(db) = conn { + let checkpoint = crate::wac_executor::load_checkpoint(db, &job.id).await?; + let checkpoint = + crate::wac_executor::prepare_checkpoint_for_resume(db, &job.id, checkpoint).await?; + + let checkpoint_json = serde_json::to_string(&checkpoint).map_err(|e| { + error::Error::internal_err(format!("Failed to serialize checkpoint: {e}")) + })?; + write_file(job_dir, "checkpoint.json", &checkpoint_json)?; + } else { + write_file(job_dir, "checkpoint.json", r#"{"completed_steps":{}}"#)?; + } + } + tracing::debug!("Finished writing wrapper"); let mut reserved_variables = @@ -936,7 +1016,15 @@ mount {{ *new_args = Some(args.clone()); } - read_result(job_dir, handle_result.result_stream).await + let result = read_result(job_dir, handle_result.result_stream).await?; + + // WAC v2 post-execution: parse output and handle dispatch/suspend. + // Box::pin to avoid bloating handle_python_job's async state machine (stack overflow). + if is_wac_v2 { + return Box::pin(crate::bun_executor::handle_wac_v2_output(result, job, conn)).await; + } + + Ok(result) } async fn prepare_wrapper( diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index c237d8cc02..e26633fc9a 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -665,7 +665,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &job, true, @@ -717,6 +717,29 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = parent_job { + // wac_job_ids is piggybacked from the duration write in + // add_completed_job — no extra query needed. + if let Some(job_ids) = wac_job_ids { + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job_id, + parent_job, + &workspace_id, + result, + true, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } else { let result = add_completed_job_error( @@ -770,11 +793,229 @@ pub async fn process_completed_job( } return Ok(r); } + } else if let Some(parent_job) = job.parent_job { + // WAC child failed — query job_ids from parent (errors are rare, + // so the extra read is acceptable here). + let job_ids_json: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint'->'pending_steps'->'job_ids' \ + FROM v2_job_status WHERE id = $1", + ) + .bind(&parent_job) + .fetch_optional(db) + .await?; + if let Some(Some(job_ids)) = job_ids_json { + let err_result = Arc::new(serde_json::value::to_raw_value(&result).unwrap()); + if let Ok(Some(_)) = handle_wac_child_completion( + db, + &job.id, + parent_job, + &job.workspace_id, + err_result, + false, + job_ids, + ) + .await + { + if let Some(done_tx) = done_tx { + done_tx + .send(()) + .expect("done receiver should still be alive"); + } + return Ok(None); + } + } } } return Ok(None); } +/// Handle a WAC v2 child job completion. +/// Returns Ok(Some(())) if the parent was a WAC job and was handled, +/// Ok(None) if the parent is not a WAC job (caller should fall through). +/// +/// CONCURRENCY: Multiple parallel children may complete simultaneously on +/// different workers. We use atomic SQL operations throughout: +/// - `completed_steps` is merged via `jsonb_set(... || jsonb_build_object(...))` +/// — PostgreSQL serialises concurrent UPDATEs on the same row, so each +/// worker sees the previous worker's writes. +/// - The suspend counter (set to N at dispatch time) is decremented atomically +/// with `RETURNING` to determine the "all done" condition. +pub(crate) async fn handle_wac_child_completion( + db: &DB, + child_job_id: &Uuid, + parent_job_id: Uuid, + workspace_id: &str, + result: Arc>, + success: bool, + job_ids_value: Value, +) -> error::Result> { + let job_ids = match job_ids_value { + Value::Object(m) => m, + _ => return Ok(None), // Not a WAC parent or no pending steps + }; + + let child_id_str = child_job_id.to_string(); + let step_key = job_ids.iter().find_map(|(key, val)| { + if val.as_str() == Some(&child_id_str) { + Some(key.clone()) + } else { + None + } + }); + + let step_key = match step_key { + Some(k) => k, + None => { + if !success { + // No step key and failed — can't store error, fail parent immediately + tracing::error!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child job failed but no step key found, failing parent" + ); + sqlx::query!( + "UPDATE v2_job_queue SET suspend = 0, suspend_until = NULL WHERE id = $1", + parent_job_id, + ) + .execute(db) + .await?; + let parent_mini = get_mini_completed_job(&parent_job_id, workspace_id, db).await?; + if let Some(parent_mini) = parent_mini { + let child_err: Value = + serde_json::from_str(result.get()).unwrap_or(Value::Null); + let err_value = json!({ + "message": format!("WAC child job {} failed (no step key)", child_job_id), + "error": child_err, + }); + let _ = windmill_queue::add_completed_job_error( + db, + &parent_mini, + 0, + None, + err_value, + "wac_child_handler", + false, + None, + ) + .await; + } + return Ok(Some(())); + } + tracing::warn!( + parent_job = %parent_job_id, + child_job = %child_job_id, + "WAC v2 child completed but no matching step key found in checkpoint, decrementing suspend to avoid parent hang" + ); + // Still decrement suspend so the parent doesn't hang indefinitely + let _ = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(db) + .await?; + return Ok(Some(())); + } + }; + + // Build result — wrap errors with _error marker so workflow try/catch can handle them + let result_value: Value = if success { + serde_json::from_str(result.get()).unwrap_or(Value::Null) + } else { + let child_err: Value = serde_json::from_str(result.get()).unwrap_or(Value::Null); + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + "WAC v2 child job failed, storing error for workflow try/catch" + ); + json!({ + "__wmill_error": true, + "message": format!("WAC task '{}' failed (child job {})", step_key, child_job_id), + "child_job_id": child_job_id.to_string(), + "step_key": step_key, + "result": child_err, + }) + }; + + tracing::info!( + parent_job = %parent_job_id, + child_job = %child_job_id, + step_key = %step_key, + success = success, + "WAC v2 child job completed" + ); + + // Use a transaction to ensure completed_steps merge + suspend decrement + // are atomic. Without this, a crash between the two could strand the parent. + let result_json = serde_json::to_value(&result_value) + .map_err(|e| error::Error::InternalErr(format!("Failed to serialize step result: {e}")))?; + + let mut tx = db.begin().await?; + + // Merge the completed step into the checkpoint. + // Uses `|| jsonb_build_object(key, value)` so concurrent children on + // different workers don't overwrite each other — PostgreSQL serialises + // concurrent UPDATEs on the same row and each sees the previous write. + sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = jsonb_set( + workflow_as_code_status, + '{_checkpoint,completed_steps}', + COALESCE(workflow_as_code_status->'_checkpoint'->'completed_steps', '{}'::jsonb) + || jsonb_build_object($2::text, $3::jsonb) + ) WHERE id = $1", + ) + .bind(&parent_job_id) + .bind(&step_key) + .bind(&result_json) + .execute(&mut *tx) + .await + .map_err(|e| error::Error::InternalErr(format!("Failed to add WAC completed step: {e}")))?; + + // Decrement the suspend counter. The counter was set to N (number of + // children) at dispatch time. When it reaches 0 all children are done. + // Keep suspend_until non-null so the suspended pull query + // (`WHERE suspend_until IS NOT NULL AND suspend <= 0`) picks up the parent. + let new_suspend: Option = sqlx::query_scalar!( + "UPDATE v2_job_queue \ + SET suspend = GREATEST(suspend - 1, 0) \ + WHERE id = $1 \ + RETURNING suspend", + parent_job_id, + ) + .fetch_optional(&mut *tx) + .await?; + + let all_done = new_suspend == Some(0); + + if all_done { + // Clear pending_steps from checkpoint since all children are complete. + // This is cosmetic — the next replay will overwrite it anyway — but + // keeps the checkpoint clean for frontend display. + let _ = sqlx::query( + "UPDATE v2_job_status SET workflow_as_code_status = \ + workflow_as_code_status #- '{_checkpoint,pending_steps}' \ + WHERE id = $1", + ) + .bind(&parent_job_id) + .execute(&mut *tx) + .await; + } + + tx.commit().await?; + + if all_done { + tracing::info!( + parent_job = %parent_job_id, + "WAC v2 all child jobs completed, unsuspending parent" + ); + } + + Ok(Some(())) +} + pub async fn handle_non_flow_job_error( db: &DB, job: &MiniCompletedJob, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9f89f2fc44..90d06287bc 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,8 +9,8 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_object_store::convert_json_line_stream; use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; +use windmill_object_store::convert_json_line_stream; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs new file mode 100644 index 0000000000..7c350ac92b --- /dev/null +++ b/backend/windmill-worker/src/wac_executor.rs @@ -0,0 +1,339 @@ +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use serde_json::Value; +use uuid::Uuid; + +use windmill_common::error::{self, Error}; +use windmill_common::DB; + +/// Checkpoint state persisted across workflow invocations. +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +pub struct WacCheckpoint { + #[serde(default)] + pub source_hash: String, + #[serde(default)] + pub completed_steps: serde_json::Map, + #[serde(skip_serializing_if = "Option::is_none")] + pub pending_steps: Option, + #[serde(default)] + pub input_args: serde_json::Map, + /// Accumulated map of step_key → child job UUID across all dispatch rounds. + /// Unlike `pending_steps.job_ids` (cleared after completion), this persists + /// so the frontend can always resolve step keys to child job names. + #[serde(default)] + pub job_ids: serde_json::Map, + /// When set on a child job's checkpoint, indicates which step this child + /// should execute directly (instead of dispatching). + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub _executing_key: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct WacPendingSteps { + pub mode: String, + pub keys: Vec, + pub job_ids: serde_json::Map, +} + +/// Output from a single WAC invocation (parsed from result.json). +#[derive(Debug, Deserialize)] +#[serde(tag = "type")] +pub enum WacOutput { + #[serde(rename = "dispatch")] + Dispatch { mode: String, steps: Vec }, + #[serde(rename = "complete")] + Complete { result: Value }, + /// An inline step executed in the parent process — persist result to + /// checkpoint and re-run immediately (no child job, no suspend). + #[serde(rename = "inline_checkpoint")] + InlineCheckpoint { key: String, result: Value }, + /// Suspend the workflow waiting for an external approval event. + /// No child job is dispatched — the parent suspends directly and resumes + /// when a user hits the resume/cancel endpoint. + #[serde(rename = "approval")] + Approval { key: String, timeout: Option, form: Option }, + /// Server-side sleep — suspend the workflow for a duration without holding a worker. + #[serde(rename = "sleep")] + Sleep { key: String, seconds: u32 }, +} + +/// A step dispatched by the WAC SDK. +/// +/// `dispatch_type` determines how the child job is created: +/// - `"inline"` (default): re-runs the parent workflow with `_executing_key` set +/// - `"script"`: runs a separate Windmill script resolved from `script` path +/// - `"flow"`: runs a separate Windmill flow resolved from `script` path +#[derive(Debug, Deserialize, Clone)] +pub struct WacStepDispatch { + pub name: String, + pub script: String, + pub args: serde_json::Map, + pub key: String, + #[serde(default = "default_dispatch_type")] + pub dispatch_type: String, + // Per-task options forwarded to push() + #[serde(default)] + pub timeout: Option, + #[serde(default)] + pub tag: Option, + #[serde(default)] + pub cache_ttl: Option, + #[serde(default)] + pub priority: Option, + #[serde(default)] + pub concurrent_limit: Option, + #[serde(default)] + pub concurrency_key: Option, + #[serde(default)] + pub concurrency_time_window_s: Option, +} + +fn default_dispatch_type() -> String { + "inline".to_string() +} + +/// Load the WAC checkpoint from `v2_job_status.workflow_as_code_status._checkpoint`. +pub async fn load_checkpoint(db: &DB, job_id: &Uuid) -> error::Result { + let row: Option> = sqlx::query_scalar( + "SELECT workflow_as_code_status->'_checkpoint' FROM v2_job_status WHERE id = $1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + match row { + Some(Some(status)) => { + let checkpoint: WacCheckpoint = match serde_json::from_value(status) { + Ok(c) => c, + Err(e) => { + tracing::warn!( + job_id = %job_id, + error = %e, + "Failed to deserialize WAC checkpoint, resetting to empty" + ); + WacCheckpoint::default() + } + }; + Ok(checkpoint) + } + _ => Ok(WacCheckpoint::default()), + } +} + +/// Save the WAC checkpoint to `v2_job_status.workflow_as_code_status._checkpoint`. +/// The top level of workflow_as_code_status is reserved for per-child-job timeline data. +pub async fn save_checkpoint( + db: &DB, + job_id: &Uuid, + checkpoint: &WacCheckpoint, +) -> error::Result<()> { + let status_json = serde_json::to_value(checkpoint) + .map_err(|e| Error::InternalErr(format!("Failed to serialize checkpoint: {e}")))?; + + sqlx::query( + "INSERT INTO v2_job_status (id, workflow_as_code_status) + VALUES ($1, jsonb_build_object('_checkpoint', $2::jsonb)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = jsonb_set( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::jsonb), + '{_checkpoint}', + $2::jsonb + )", + ) + .bind(job_id) + .bind(&status_json) + .execute(db) + .await + .map_err(|e| Error::InternalErr(format!("Failed to save WAC checkpoint: {e}")))?; + + Ok(()) +} + +/// Parse the WAC result from result.json content. +pub fn parse_wac_output(result: &RawValue) -> error::Result { + serde_json::from_str(result.get()) + .map_err(|e| Error::InternalErr(format!("Failed to parse WAC output: {e}"))) +} + +/// Process a "dispatch" result: update checkpoint with pending steps info. +pub fn update_checkpoint_for_dispatch( + checkpoint: &mut WacCheckpoint, + steps: &[WacStepDispatch], + mode: &str, + job_ids: &[(String, Uuid)], +) { + let ids_map: serde_json::Map = job_ids + .iter() + .map(|(key, id)| (key.clone(), Value::String(id.to_string()))) + .collect(); + // Accumulate into persistent job_ids (survives pending_steps clearing) + for (k, v) in ids_map.iter() { + checkpoint.job_ids.insert(k.clone(), v.clone()); + } + let pending = WacPendingSteps { + mode: mode.to_string(), + keys: steps.iter().map(|s| s.key.clone()).collect(), + job_ids: ids_map, + }; + checkpoint.pending_steps = Some(pending); +} + +/// Process a completed child job result: add to checkpoint's completed_steps. +pub fn add_completed_step(checkpoint: &mut WacCheckpoint, step_key: &str, result: Value) { + checkpoint + .completed_steps + .insert(step_key.to_string(), result); + // If all pending steps are complete, clear pending + if let Some(ref pending) = checkpoint.pending_steps { + let all_done = pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)); + if all_done { + checkpoint.pending_steps = None; + } + } +} + +/// Check if all pending parallel steps are complete. +pub fn all_pending_complete(checkpoint: &WacCheckpoint) -> bool { + match &checkpoint.pending_steps { + None => true, + Some(pending) => pending + .keys + .iter() + .all(|k| checkpoint.completed_steps.contains_key(k)), + } +} + +/// If the checkpoint has a pending approval or sleep, inject the resume result +/// into `completed_steps` and save back to DB. Returns the (possibly modified) checkpoint. +/// +/// Called by both bun and python executors before writing checkpoint.json to disk. +pub async fn prepare_checkpoint_for_resume( + db: &DB, + job_id: &Uuid, + mut checkpoint: WacCheckpoint, +) -> error::Result { + let pending_mode = checkpoint.pending_steps.as_ref().map(|p| p.mode.as_str()); + + match pending_mode { + Some("approval") => { + let approval_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + let resume_row = sqlx::query_as::<_, (sqlx::types::Json>, Option, bool)>( + "SELECT value, approver, approved FROM resume_job WHERE job = $1 ORDER BY created_at ASC LIMIT 1", + ) + .bind(job_id) + .fetch_optional(db) + .await?; + + let approval_result = if let Some((value, approver, approved)) = resume_row { + serde_json::json!({ + "value": serde_json::from_str::(value.get()).unwrap_or(Value::Null), + "approver": approver.unwrap_or_else(|| "anonymous".to_string()), + "approved": approved, + }) + } else { + serde_json::json!({ + "value": null, + "approver": null, + "approved": false, + }) + }; + checkpoint + .completed_steps + .insert(approval_key.clone(), approval_result); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + approval_key = %approval_key, + "WAC v2 injected approval result into checkpoint" + ); + } + Some("sleep") => { + let sleep_key = checkpoint + .pending_steps + .as_ref() + .and_then(|p| p.keys.first().cloned()) + .unwrap_or_default(); + + checkpoint + .completed_steps + .insert(sleep_key.clone(), Value::Bool(true)); + checkpoint.pending_steps = None; + save_checkpoint(db, job_id, &checkpoint).await?; + + tracing::info!( + job_id = %job_id, + sleep_key = %sleep_key, + "WAC v2 resumed from sleep" + ); + } + _ => {} + } + + Ok(checkpoint) +} + +/// Detect WAC v2 patterns in TypeScript/Bun code. +/// Checks for `import ... from "windmill-client"` containing workflow/task, +/// skipping comment lines. +pub fn is_wac_v2_ts(code: &str) -> bool { + let mut has_wac_import = false; + let mut has_workflow = false; + let mut has_task = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("//") { + continue; + } + if trimmed.contains("windmill-client") + && (trimmed.starts_with("import") || trimmed.starts_with("from")) + { + has_wac_import = true; + if trimmed.contains("workflow") { + has_workflow = true; + } + if trimmed.contains("task") { + has_task = true; + } + } + if trimmed.contains("export") && trimmed.contains("workflow(") { + has_workflow = true; + } + } + has_wac_import && has_workflow && has_task +} + +/// Detect WAC v2 patterns in Python code. +/// Checks for `@workflow` decorator and `@task` decorator with wmill import, +/// skipping comment lines. +pub fn is_wac_v2_py(code: &str) -> bool { + let mut has_wmill_import = false; + let mut has_workflow_decorator = false; + let mut has_task_decorator = false; + for line in code.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('#') { + continue; + } + if trimmed.starts_with("import wmill") || trimmed.starts_with("from wmill") { + has_wmill_import = true; + } + if trimmed == "@workflow" || trimmed.starts_with("@workflow(") { + has_workflow_decorator = true; + } + if trimmed == "@task" || trimmed.starts_with("@task(") { + has_task_decorator = true; + } + } + has_wmill_import && has_workflow_decorator && has_task_decorator +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8d7f6051ec..6d86ae2723 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -3528,6 +3528,13 @@ pub async fn handle_queued_job( { return Ok(false); } + if result + .as_ref() + .is_err_and(|err| matches!(err, &Error::WacSuspended(_))) + { + // WAC v2 job suspended while waiting for child jobs — don't complete it + return Ok(true); + } process_result( cjob, result.map(|x| Arc::new(x)), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 8a45f85a90..24dcfb9b9a 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1722,8 +1722,8 @@ pub async fn update_flow_status_after_job_completion_internal( chat_ai_info.conversation_id, ) .await?; - let duration = if success { - let (_, duration) = add_completed_job( + let (duration, wac_job_ids) = if success { + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, true, @@ -1737,9 +1737,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) } else { - let (_, duration) = add_completed_job( + let (_, duration, wac_job_ids) = add_completed_job( db, &cflow_job, false, @@ -1757,11 +1757,30 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + (duration, wac_job_ids) }; flow_job_duration = flow_job .started_at .map(|x| FlowJobDuration { started_at: x, duration_ms: duration }); + + // If this flow is a WAC child (not a flow step, has parent), + // notify the WAC parent of completion. + if !flow_job.is_flow_step() { + if let Some(parent_job) = flow_job.parent_job { + if let Some(job_ids) = wac_job_ids { + let _ = crate::result_processor::handle_wac_child_completion( + db, + &flow_job.id, + parent_job, + &flow_job.workspace_id, + nresult.clone(), + success, + job_ids, + ) + .await; + } + } + } } true } else { diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index e4e166791b..7de0926f0a 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -21,6 +21,7 @@ getPreprocessorFullCode, getMainFunctionPattern } from '$lib/script_helpers' + import { isWorkflowAsCode } from './graph/wacToFlow' import AIFormSettings from './copilot/AIFormSettings.svelte' import { defaultScripts, @@ -591,7 +592,7 @@ if (!disableHistoryChange) { history.replaceState(history.state, '', `/scripts/edit/${script.path}`) } - if (stay || (script.no_main_func && script.kind !== 'preprocessor')) { + if (stay || (script.no_main_func && script.kind !== 'preprocessor' && !isWorkflowAsCode(script.content, script.language))) { script.parent_hash = newHash sendUserToast('Deployed') } else { diff --git a/frontend/src/lib/components/TimelineBar.svelte b/frontend/src/lib/components/TimelineBar.svelte index 7c4e170d93..e4fb511450 100644 --- a/frontend/src/lib/components/TimelineBar.svelte +++ b/frontend/src/lib/components/TimelineBar.svelte @@ -35,7 +35,7 @@ {/if} {/snippet} {#if len > 0} - {#if len}{msToSec(len, 1)}s{/if} {/if} diff --git a/frontend/src/lib/components/WorkflowTimeline.svelte b/frontend/src/lib/components/WorkflowTimeline.svelte index 35cf4445ff..66fbb5b6c5 100644 --- a/frontend/src/lib/components/WorkflowTimeline.svelte +++ b/frontend/src/lib/components/WorkflowTimeline.svelte @@ -3,7 +3,7 @@ import { displayDate, msToSec } from '$lib/utils' import { onDestroy } from 'svelte' import { getDbClockNow } from '$lib/forLater' - import { ExternalLink, Loader2 } from 'lucide-svelte' + import { Loader2 } from 'lucide-svelte' import TimelineBar from './TimelineBar.svelte' import type { WorkflowStatus } from '$lib/gen' @@ -17,13 +17,10 @@ let now = $state(getDbClockNow().getTime()) - let interval = setInterval((x) => { + let interval = setInterval(() => { if (!max) { now = getDbClockNow().getTime() } - if (min && (!max || total == undefined)) { - total = max ? max - min : Math.max(now - min, 2000) - } }, 30) onDestroy(() => { @@ -40,7 +37,7 @@ 0 ) : undefined) - let total = $derived(flowDone && max ? max - min : now - min) + let total = $derived(flowDone && max ? max - min : Math.max(now - min, 2000)) {#if flow_status} @@ -75,7 +72,7 @@
{v.name ?? k} {v.name ?? k}
{#if min && total} diff --git a/frontend/src/lib/components/graph/wacToFlow.ts b/frontend/src/lib/components/graph/wacToFlow.ts new file mode 100644 index 0000000000..91d335ff91 --- /dev/null +++ b/frontend/src/lib/components/graph/wacToFlow.ts @@ -0,0 +1,16 @@ +/** + * Detect whether a script is a workflow-as-code entry point. + */ +export function isWorkflowAsCode(code: string, language: string): boolean { + if (language === 'python3') { + return /^\s*@workflow\s*$/m.test(code) || /from\s+wmill\s+import.*workflow/.test(code) + } + if (language === 'bun' || language === 'deno') { + return ( + /workflow\s*\(/.test(code) && + /task\s*\(/.test(code) && + /import.*(?:workflow|task).*from\s+['"]windmill-client(?:@[^'"]*)?['"]/.test(code) + ) + } + return false +} diff --git a/frontend/src/lib/components/runs/JobRunsPreview.svelte b/frontend/src/lib/components/runs/JobRunsPreview.svelte index 637848c83c..274679d3bc 100644 --- a/frontend/src/lib/components/runs/JobRunsPreview.svelte +++ b/frontend/src/lib/components/runs/JobRunsPreview.svelte @@ -47,7 +47,12 @@ ) function asWorkflowStatus(x: any): Record { - return x as Record + if (!x || typeof x !== 'object') return {} + const result: Record = {} + for (const [k, v] of Object.entries(x)) { + if (!k.startsWith('_')) result[k] = v as WorkflowStatus + } + return result } function handleFilterByConcurrencyKey(key: string) { diff --git a/frontend/src/lib/components/scriptEditor/LogPanel.svelte b/frontend/src/lib/components/scriptEditor/LogPanel.svelte index c47c05c281..63eb9d5a4c 100644 --- a/frontend/src/lib/components/scriptEditor/LogPanel.svelte +++ b/frontend/src/lib/components/scriptEditor/LogPanel.svelte @@ -68,6 +68,7 @@ showCustomResultPanel = false }: Props = $props() + type DContent = { mode: 'json' | Preview['language'] | 'plain' title: string @@ -92,7 +93,12 @@ } function asWorkflowStatus(x: any): Record { - return x as Record + if (!x || typeof x !== 'object') return {} + const result: Record = {} + for (const [k, v] of Object.entries(x)) { + if (!k.startsWith('_')) result[k] = v as WorkflowStatus + } + return result } let forceJson = $state(false) diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 9ecedc87ea..ac73b934fd 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -281,7 +281,12 @@ let redactSensitive = $state(false) function asWorkflowStatus(x: any): Record { - return x as Record + if (!x || typeof x !== 'object') return {} + const result: Record = {} + for (const [k, v] of Object.entries(x)) { + if (!k.startsWith('_')) result[k] = v as WorkflowStatus + } + return result } function forkPreview() { @@ -779,11 +784,15 @@ {/if}
{#if job?.workflow_as_code_status && job.job_kind !== 'aiagent'} -
- +
+

Workflow Timeline

+
+ +
+
{/if} {#if scriptProgress} diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 5fcfad1ddd..cf3c7460a5 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -22,6 +22,12 @@ httpx = ">=0.24" requires = ["poetry>=1.0.2", "poetry-dynamic-versioning"] build-backend = "poetry.masonry.api" +[dependency-groups] +dev = [ + "httpx>=0.28.1", + "pytest>=9.0.2", +] + [tool.poetry-dynamic-versioning] enable = true vcs = "git" diff --git a/python-client/wmill/tests/test_workflow.py b/python-client/wmill/tests/test_workflow.py new file mode 100644 index 0000000000..43a5f22693 --- /dev/null +++ b/python-client/wmill/tests/test_workflow.py @@ -0,0 +1,1114 @@ +"""Tests for the Workflow-as-Code SDK.""" + +import asyncio +import pytest + +from wmill.client import WorkflowCtx, _StepSuspend, TaskError, workflow, task, step, sleep, parallel, _run_workflow + + +@task +async def extract_data(url: str): + pass # body unused in workflow context + + +@task +async def load_data(data=None): + pass + + +@task +async def clean_data(data=None): + pass + + +@task +async def compute_stats(data=None): + pass + + +@task +async def send_alert(msg: str = ""): + pass + + +@task +async def double(x: int): + return x * 2 + + +@task +async def add_one(x: int): + return x + 1 + + +@task +async def noop_task(): + pass + + +# --- Module-level workflow definitions --- + +@workflow +async def simple_workflow(url: str): + raw = await extract_data(url=url) + result = await load_data(data=raw) + return {"status": "done", "result": result} + + +@workflow +async def parallel_workflow(url: str): + raw = await extract_data(url=url) + cleaned, stats = await asyncio.gather( + clean_data(data=raw), + compute_stats(data=raw), + ) + return {"cleaned": cleaned, "stats": stats} + + +@workflow +async def conditional_workflow(count: int): + if count > 100: + await send_alert(msg="large") + await load_data() + return {"done": True} + + +@workflow +async def step_workflow(x: int): + ts = await step("timestamp", lambda: 1234567890) + doubled = await double(x=x) + rid = await step("random_id", lambda: "abc-123") + return {"ts": ts, "doubled": doubled, "id": rid} + + +# Edge case workflows + +@workflow +async def three_step_wf(n: int): + doubled = await double(x=n) + incremented = await add_one(x=doubled) + final = await double(x=incremented) + return {"doubled": doubled, "incremented": incremented, "final": final} + + +@workflow +async def seq_par_seq_wf(url: str): + raw = await extract_data(url=url) + cleaned, stats = await asyncio.gather( + clean_data(data=raw), + compute_stats(data=raw), + ) + loaded = await load_data(data={"cleaned": cleaned, "stats": stats}) + return loaded + + +@workflow +async def double_parallel_wf(): + a, b = await asyncio.gather(double(x=1), double(x=2)) + c, d = await asyncio.gather(add_one(x=a), add_one(x=b)) + return {"a": a, "b": b, "c": c, "d": d} + + +@workflow +async def cond_on_result_wf(): + val = await double(x=5) + if val > 8: + await send_alert(msg="big") + await load_data(data=val) + return {"val": val} + + +@workflow +async def empty_wf(): + return {"status": "empty"} + + +@workflow +async def single_wf(x: int): + result = await double(x=x) + return result + + +@workflow +async def no_arg_wf(): + result = await noop_task() + return result + + +@workflow +async def many_steps_wf(n: int): + val = n + for _ in range(10): + val = await add_one(x=val) + return val + + +@workflow +async def falsy_wf(): + a = await double(x=0) + b = await load_data(data=a) + c = await extract_data(url="") + return {"a": a, "b": b, "c": c} + + +@task(path="f/external_script") +async def run_external(x: int): + return x * 3 + + +@workflow +async def path_wf(x: int): + result = await run_external(x=x) + return result + + +@workflow +async def mixed_step_task_wf(x: int): + ts = await step("get_time", lambda: 999) + doubled = await double(x=x) + config = await step("get_config", lambda: {"retry": 3}) + added = await add_one(x=doubled) + return {"ts": ts, "doubled": doubled, "config": config, "added": added} + + +@workflow +async def par_child_wf(): + a, b = await asyncio.gather(double(x=3), add_one(x=7)) + return {"a": a, "b": b} + + +@workflow +async def det_wf(n: int): + a = await double(x=n) + b = await add_one(x=a) + c = await double(x=b) + return c + + +@workflow +async def par_args_wf(x: int): + base = await double(x=x) + a, b = await asyncio.gather(add_one(x=base), double(x=base)) + return {"a": a, "b": b} + + +@workflow +async def none_return_wf(): + await double(x=1) + + +@workflow +async def large_par_wf(): + results = await asyncio.gather( + double(x=1), double(x=2), double(x=3), double(x=4), double(x=5) + ) + return list(results) + + +@workflow +async def complex_mixed_wf(): + init = await extract_data(url="start") + a, b = await asyncio.gather(double(x=1), double(x=2)) + mid = await load_data(data={"a": a, "b": b}) + c, d = await asyncio.gather(add_one(x=3), add_one(x=4)) + fin = await clean_data(data={"mid": mid, "c": c, "d": d}) + return fin + + +@workflow +async def pre_par_child_wf(x: int): + base = await double(x=x) + a, b = await asyncio.gather(add_one(x=base), double(x=base)) + return {"a": a, "b": b} + + +# --- Tests --- +# NOTE: Python SDK uses name-based keys (e.g. "double", "double_2") +# not index-based keys (e.g. "step_0", "step_1"). + +class TestWorkflowDecorator: + def test_marks_function(self): + assert hasattr(simple_workflow, "_is_workflow") + assert simple_workflow._is_workflow is True + + +class TestTaskDecorator: + def test_marks_function(self): + assert hasattr(extract_data, "_is_task") + assert extract_data._is_task is True + + def test_standalone_execution(self): + """Outside a workflow, @task runs the function body directly.""" + result = asyncio.run(extract_data(url="https://example.com")) + assert result is None # body returns None + + def test_preserves_function_name(self): + assert extract_data.__name__ == "extract_data" + assert double.__name__ == "double" + + +class TestFirstInvocation: + def test_dispatches_first_step(self): + result = _run_workflow(simple_workflow, {}, {"url": "https://example.com"}) + assert result["type"] == "dispatch" + assert result["mode"] == "sequential" + assert len(result["steps"]) == 1 + assert result["steps"][0]["name"] == "extract_data" + assert result["steps"][0]["script"] == "extract_data" + assert result["steps"][0]["key"] == "extract_data" + assert result["steps"][0]["args"] == {"url": "https://example.com"} + + def test_positional_args_converted_to_kwargs(self): + """Positional args should be mapped to parameter names in dispatch.""" + @workflow + async def pos_workflow(): + await extract_data("https://pos.example.com") + + result = _run_workflow(pos_workflow, {}, {}) + assert result["type"] == "dispatch" + assert result["steps"][0]["args"] == {"url": "https://pos.example.com"} + + +class TestReplayWithCheckpoint: + def test_second_invocation_dispatches_second_step(self): + checkpoint = { + "completed_steps": { + "extract_data": {"data": [1, 2, 3]}, + } + } + result = _run_workflow(simple_workflow, checkpoint, {"url": "https://example.com"}) + assert result["type"] == "dispatch" + assert result["mode"] == "sequential" + assert result["steps"][0]["name"] == "load_data" + assert result["steps"][0]["key"] == "load_data" + + def test_all_steps_complete(self): + checkpoint = { + "completed_steps": { + "extract_data": {"data": [1, 2, 3]}, + "load_data": {"loaded": True}, + } + } + result = _run_workflow(simple_workflow, checkpoint, {"url": "https://example.com"}) + assert result["type"] == "complete" + assert result["result"]["status"] == "done" + assert result["result"]["result"] == {"loaded": True} + + +class TestParallelDispatch: + def test_first_invocation(self): + result = _run_workflow(parallel_workflow, {}, {"url": "https://example.com"}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "extract_data" + + def test_parallel_dispatch(self): + checkpoint = { + "completed_steps": { + "extract_data": {"raw": "data"}, + } + } + result = _run_workflow(parallel_workflow, checkpoint, {"url": "https://example.com"}) + assert result["type"] == "dispatch" + assert result["mode"] == "parallel" + assert len(result["steps"]) == 2 + assert result["steps"][0]["name"] == "clean_data" + assert result["steps"][1]["name"] == "compute_stats" + + def test_parallel_complete(self): + checkpoint = { + "completed_steps": { + "extract_data": {"raw": "data"}, + "clean_data": {"cleaned": True}, + "compute_stats": {"count": 42}, + } + } + result = _run_workflow(parallel_workflow, checkpoint, {"url": "https://example.com"}) + assert result["type"] == "complete" + assert result["result"]["cleaned"] == {"cleaned": True} + assert result["result"]["stats"] == {"count": 42} + + +class TestConditionalWorkflow: + def test_condition_true(self): + result = _run_workflow(conditional_workflow, {}, {"count": 200}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "send_alert" + + def test_condition_false(self): + result = _run_workflow(conditional_workflow, {}, {"count": 50}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "load_data" + + +class TestStepInlineCheckpoint: + def test_first_invocation_returns_inline_checkpoint(self): + result = _run_workflow(step_workflow, {}, {"x": 7}) + assert result["type"] == "inline_checkpoint" + assert result["key"] == "timestamp" + assert result["result"] == 1234567890 + + def test_step_cached_then_task_dispatches(self): + checkpoint = {"completed_steps": {"timestamp": 1234567890}} + result = _run_workflow(step_workflow, checkpoint, {"x": 7}) + assert result["type"] == "dispatch" + assert result["mode"] == "sequential" + assert result["steps"][0]["name"] == "double" + assert result["steps"][0]["key"] == "double" + + def test_step_and_task_cached_then_second_step(self): + checkpoint = {"completed_steps": {"timestamp": 1234567890, "double": 14}} + result = _run_workflow(step_workflow, checkpoint, {"x": 7}) + assert result["type"] == "inline_checkpoint" + assert result["key"] == "random_id" + assert result["result"] == "abc-123" + + def test_all_complete(self): + checkpoint = {"completed_steps": {"timestamp": 1234567890, "double": 14, "random_id": "abc-123"}} + result = _run_workflow(step_workflow, checkpoint, {"x": 7}) + assert result["type"] == "complete" + assert result["result"] == {"ts": 1234567890, "doubled": 14, "id": "abc-123"} + + +class TestUnawaitedTask: + def test_unawaited_last_task_is_flushed(self): + @workflow + async def unawaited_workflow(): + await extract_data(url="x") + load_data(data="y") + + checkpoint = {"completed_steps": {"extract_data": "raw"}} + result = _run_workflow(unawaited_workflow, checkpoint, {}) + assert result["type"] == "dispatch" + assert result["mode"] == "sequential" + assert len(result["steps"]) == 1 + assert result["steps"][0]["name"] == "load_data" + + def test_unawaited_multiple_tasks_flushed_as_parallel(self): + @workflow + async def multi_unawaited_workflow(): + await extract_data(url="x") + clean_data(data="y") + compute_stats(data="y") + + checkpoint = {"completed_steps": {"extract_data": "raw"}} + result = _run_workflow(multi_unawaited_workflow, checkpoint, {}) + assert result["type"] == "dispatch" + assert result["mode"] == "parallel" + assert len(result["steps"]) == 2 + assert result["steps"][0]["name"] == "clean_data" + assert result["steps"][1]["name"] == "compute_stats" + + +class TestChildMode: + def test_child_executes_matching_task(self): + checkpoint = {"completed_steps": {"timestamp": 1234567890}, "_executing_key": "double"} + result = _run_workflow(step_workflow, checkpoint, {"x": 7}) + assert result["type"] == "complete" + assert result["result"] == 14 + + def test_child_replays_cached_steps(self): + checkpoint = { + "completed_steps": {"extract_data": {"data": [1, 2, 3]}}, + "_executing_key": "load_data", + } + result = _run_workflow(simple_workflow, checkpoint, {"url": "https://example.com"}) + assert result["type"] == "complete" + assert result["result"] is None + + +# ===================================================================== +# EDGE CASE TESTS +# ===================================================================== + +class TestFullSequentialLifecycle: + def test_replay_0_dispatches_step_0(self): + result = _run_workflow(three_step_wf, {}, {"n": 5}) + assert result["type"] == "dispatch" + assert result["steps"][0]["key"] == "double" + assert result["steps"][0]["name"] == "double" + assert result["steps"][0]["args"] == {"x": 5} + + def test_replay_1_dispatches_step_1_with_step_0_result(self): + result = _run_workflow(three_step_wf, {"completed_steps": {"double": 10}}, {"n": 5}) + assert result["type"] == "dispatch" + assert result["steps"][0]["key"] == "add_one" + assert result["steps"][0]["name"] == "add_one" + assert result["steps"][0]["args"] == {"x": 10} + + def test_replay_2_dispatches_step_2_with_step_1_result(self): + result = _run_workflow( + three_step_wf, {"completed_steps": {"double": 10, "add_one": 11}}, {"n": 5} + ) + assert result["type"] == "dispatch" + assert result["steps"][0]["key"] == "double_2" + assert result["steps"][0]["name"] == "double" + assert result["steps"][0]["args"] == {"x": 11} + + def test_replay_3_all_complete(self): + result = _run_workflow( + three_step_wf, + {"completed_steps": {"double": 10, "add_one": 11, "double_2": 22}}, + {"n": 5}, + ) + assert result["type"] == "complete" + assert result["result"] == {"doubled": 10, "incremented": 11, "final": 22} + + +class TestStepAfterParallelGroup: + def test_dispatches_first_sequential(self): + result = _run_workflow(seq_par_seq_wf, {}, {"url": "http://x"}) + assert result["steps"][0]["name"] == "extract_data" + + def test_dispatches_parallel_group(self): + result = _run_workflow( + seq_par_seq_wf, {"completed_steps": {"extract_data": "raw"}}, {"url": "http://x"} + ) + assert result["mode"] == "parallel" + assert len(result["steps"]) == 2 + + def test_dispatches_final_step_after_parallel(self): + result = _run_workflow( + seq_par_seq_wf, + {"completed_steps": {"extract_data": "raw", "clean_data": "cleaned", "compute_stats": {"count": 5}}}, + {"url": "http://x"}, + ) + assert result["mode"] == "sequential" + assert result["steps"][0]["name"] == "load_data" + assert result["steps"][0]["key"] == "load_data" + + def test_completes_when_final_step_done(self): + result = _run_workflow( + seq_par_seq_wf, + {"completed_steps": {"extract_data": "raw", "clean_data": "cleaned", "compute_stats": {"count": 5}, "load_data": "final"}}, + {"url": "http://x"}, + ) + assert result["type"] == "complete" + assert result["result"] == "final" + + +class TestParallelAfterParallel: + def test_dispatches_first_parallel(self): + result = _run_workflow(double_parallel_wf, {}, {}) + assert result["mode"] == "parallel" + assert len(result["steps"]) == 2 + assert result["steps"][0]["key"] == "double" + assert result["steps"][1]["key"] == "double_2" + + def test_dispatches_second_parallel(self): + result = _run_workflow( + double_parallel_wf, {"completed_steps": {"double": 2, "double_2": 4}}, {} + ) + assert result["mode"] == "parallel" + assert len(result["steps"]) == 2 + assert result["steps"][0]["name"] == "add_one" + assert result["steps"][0]["args"] == {"x": 2} + assert result["steps"][1]["args"] == {"x": 4} + + def test_completes_all_done(self): + result = _run_workflow( + double_parallel_wf, + {"completed_steps": {"double": 2, "double_2": 4, "add_one": 3, "add_one_2": 5}}, + {}, + ) + assert result["type"] == "complete" + assert result["result"] == {"a": 2, "b": 4, "c": 3, "d": 5} + + +class TestConditionalBasedOnStepResult: + def test_condition_true_path(self): + result = _run_workflow(cond_on_result_wf, {"completed_steps": {"double": 10}}, {}) + assert result["steps"][0]["name"] == "send_alert" + assert result["steps"][0]["key"] == "send_alert" + + def test_condition_false_path(self): + result = _run_workflow(cond_on_result_wf, {"completed_steps": {"double": 4}}, {}) + assert result["steps"][0]["name"] == "load_data" + assert result["steps"][0]["key"] == "load_data" + + def test_condition_true_step_after_alert(self): + result = _run_workflow( + cond_on_result_wf, {"completed_steps": {"double": 10, "send_alert": "alerted"}}, {} + ) + assert result["steps"][0]["name"] == "load_data" + assert result["steps"][0]["key"] == "load_data" + + +class TestEmptyWorkflow: + def test_completes_immediately(self): + result = _run_workflow(empty_wf, {}, {}) + assert result["type"] == "complete" + assert result["result"] == {"status": "empty"} + + +class TestSingleTaskWorkflow: + def test_dispatches_single_step(self): + result = _run_workflow(single_wf, {}, {"x": 7}) + assert result["type"] == "dispatch" + assert len(result["steps"]) == 1 + assert result["steps"][0]["name"] == "double" + + def test_completes_with_result(self): + result = _run_workflow(single_wf, {"completed_steps": {"double": 14}}, {"x": 7}) + assert result["type"] == "complete" + assert result["result"] == 14 + + +class TestTaskWithNoArgs: + def test_dispatches_with_empty_args(self): + result = _run_workflow(no_arg_wf, {}, {}) + assert result["type"] == "dispatch" + assert result["steps"][0]["args"] == {} + + +class TestManySteps: + def test_first_dispatches_step_0(self): + result = _run_workflow(many_steps_wf, {}, {"n": 0}) + assert result["steps"][0]["key"] == "add_one" + + def test_with_5_complete_dispatches_step_5(self): + # add_one, add_one_2, add_one_3, add_one_4, add_one_5 + completed = {} + for i in range(5): + key = "add_one" if i == 0 else f"add_one_{i + 1}" + completed[key] = i + 1 + result = _run_workflow(many_steps_wf, {"completed_steps": completed}, {"n": 0}) + assert result["steps"][0]["key"] == "add_one_6" + assert result["steps"][0]["args"] == {"x": 5} + + def test_all_10_complete(self): + completed = {} + for i in range(10): + key = "add_one" if i == 0 else f"add_one_{i + 1}" + completed[key] = i + 1 + result = _run_workflow(many_steps_wf, {"completed_steps": completed}, {"n": 0}) + assert result["type"] == "complete" + assert result["result"] == 10 + + +class TestFalsyValues: + def test_zero_preserved(self): + result = _run_workflow(falsy_wf, {"completed_steps": {"double": 0}}, {}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "load_data" + assert result["steps"][0]["args"] == {"data": 0} + + def test_none_preserved(self): + result = _run_workflow(falsy_wf, {"completed_steps": {"double": 0, "load_data": None}}, {}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "extract_data" + + def test_all_falsy_complete(self): + result = _run_workflow( + falsy_wf, {"completed_steps": {"double": 0, "load_data": None, "extract_data": ""}}, {} + ) + assert result["type"] == "complete" + assert result["result"] == {"a": 0, "b": None, "c": ""} + + def test_false_preserved(self): + @workflow + async def flag_wf(): + val = await load_data(data="check") + if val: + await send_alert(msg="truthy") + return {"val": val} + + result = _run_workflow(flag_wf, {"completed_steps": {"load_data": False}}, {}) + assert result["type"] == "complete" + assert result["result"] == {"val": False} + + +class TestTaskWithExplicitPath: + def test_uses_path_as_script(self): + result = _run_workflow(path_wf, {}, {"x": 42}) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "run_external" + assert result["steps"][0]["script"] == "f/external_script" + assert result["steps"][0]["args"] == {"x": 42} + + +class TestMixedStepAndTask: + def test_step_0_inline(self): + result = _run_workflow(mixed_step_task_wf, {}, {"x": 5}) + assert result["type"] == "inline_checkpoint" + assert result["key"] == "get_time" + assert result["result"] == 999 + + def test_step_1_task_dispatch(self): + result = _run_workflow( + mixed_step_task_wf, {"completed_steps": {"get_time": 999}}, {"x": 5} + ) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "double" + assert result["steps"][0]["key"] == "double" + + def test_step_2_inline(self): + result = _run_workflow( + mixed_step_task_wf, + {"completed_steps": {"get_time": 999, "double": 10}}, + {"x": 5}, + ) + assert result["type"] == "inline_checkpoint" + assert result["key"] == "get_config" + assert result["result"] == {"retry": 3} + + def test_step_3_task_dispatch(self): + result = _run_workflow( + mixed_step_task_wf, + {"completed_steps": {"get_time": 999, "double": 10, "get_config": {"retry": 3}}}, + {"x": 5}, + ) + assert result["type"] == "dispatch" + assert result["steps"][0]["name"] == "add_one" + assert result["steps"][0]["key"] == "add_one" + + def test_all_complete(self): + result = _run_workflow( + mixed_step_task_wf, + {"completed_steps": {"get_time": 999, "double": 10, "get_config": {"retry": 3}, "add_one": 11}}, + {"x": 5}, + ) + assert result["type"] == "complete" + assert result["result"] == {"ts": 999, "doubled": 10, "config": {"retry": 3}, "added": 11} + + +class TestChildModeParallel: + def test_child_executes_first_parallel_step(self): + result = _run_workflow( + par_child_wf, {"completed_steps": {}, "_executing_key": "double"}, {} + ) + assert result["type"] == "complete" + assert result["result"] == 6 + + def test_child_executes_second_parallel_step(self): + result = _run_workflow( + par_child_wf, {"completed_steps": {}, "_executing_key": "add_one"}, {} + ) + assert result["type"] == "complete" + assert result["result"] == 8 + + +class TestKeyDeterminism: + def test_keys_consistent_across_replays(self): + r1 = _run_workflow(det_wf, {}, {"n": 3}) + assert r1["steps"][0]["key"] == "double" + assert r1["steps"][0]["name"] == "double" + + r2 = _run_workflow(det_wf, {"completed_steps": {"double": 6}}, {"n": 3}) + assert r2["steps"][0]["key"] == "add_one" + assert r2["steps"][0]["name"] == "add_one" + + r3 = _run_workflow(det_wf, {"completed_steps": {"double": 6, "add_one": 7}}, {"n": 3}) + assert r3["steps"][0]["key"] == "double_2" + assert r3["steps"][0]["name"] == "double" + + +class TestParallelArgsFromCachedResult: + def test_parallel_steps_receive_cached_args(self): + result = _run_workflow(par_args_wf, {"completed_steps": {"double": 20}}, {"x": 10}) + assert result["mode"] == "parallel" + assert result["steps"][0]["args"] == {"x": 20} + assert result["steps"][1]["args"] == {"x": 20} + + +class TestWorkflowReturningNone: + def test_none_return_captured(self): + result = _run_workflow(none_return_wf, {"completed_steps": {"double": 2}}, {}) + assert result["type"] == "complete" + assert result["result"] is None + + +class TestLargeParallelGroup: + def test_dispatches_5_parallel(self): + result = _run_workflow(large_par_wf, {}, {}) + assert result["mode"] == "parallel" + assert len(result["steps"]) == 5 + keys = [result["steps"][i]["key"] for i in range(5)] + assert keys == ["double", "double_2", "double_3", "double_4", "double_5"] + for i in range(5): + assert result["steps"][i]["args"] == {"x": i + 1} + + +class TestComplexMixedWorkflow: + def test_replay_0_extract(self): + r = _run_workflow(complex_mixed_wf, {}, {}) + assert r["steps"][0]["name"] == "extract_data" + + def test_replay_1_parallel(self): + r = _run_workflow(complex_mixed_wf, {"completed_steps": {"extract_data": "init"}}, {}) + assert r["mode"] == "parallel" + assert len(r["steps"]) == 2 + + def test_replay_2_load(self): + r = _run_workflow( + complex_mixed_wf, + {"completed_steps": {"extract_data": "init", "double": 2, "double_2": 4}}, + {}, + ) + assert r["mode"] == "sequential" + assert r["steps"][0]["name"] == "load_data" + assert r["steps"][0]["key"] == "load_data" + + def test_replay_3_second_parallel(self): + r = _run_workflow( + complex_mixed_wf, + {"completed_steps": {"extract_data": "init", "double": 2, "double_2": 4, "load_data": "mid"}}, + {}, + ) + assert r["mode"] == "parallel" + assert len(r["steps"]) == 2 + assert r["steps"][0]["name"] == "add_one" + + def test_replay_4_clean(self): + r = _run_workflow( + complex_mixed_wf, + {"completed_steps": { + "extract_data": "init", "double": 2, "double_2": 4, + "load_data": "mid", "add_one": 4, "add_one_2": 5, + }}, + {}, + ) + assert r["mode"] == "sequential" + assert r["steps"][0]["name"] == "clean_data" + assert r["steps"][0]["key"] == "clean_data" + + def test_replay_5_all_complete(self): + r = _run_workflow( + complex_mixed_wf, + {"completed_steps": { + "extract_data": "init", "double": 2, "double_2": 4, + "load_data": "mid", "add_one": 4, "add_one_2": 5, "clean_data": "final", + }}, + {}, + ) + assert r["type"] == "complete" + assert r["result"] == "final" + + +class TestChildModeWithCachedStepsBeforeParallel: + def test_child_executes_second_parallel_with_cached_base(self): + result = _run_workflow( + pre_par_child_wf, + {"completed_steps": {"double": 10}, "_executing_key": "double_2"}, + {"x": 5}, + ) + assert result["type"] == "complete" + assert result["result"] == 20 + + def test_child_executes_first_parallel_with_cached_base(self): + result = _run_workflow( + pre_par_child_wf, + {"completed_steps": {"double": 10}, "_executing_key": "add_one"}, + {"x": 5}, + ) + assert result["type"] == "complete" + assert result["result"] == 11 + + +# ===================================================================== +# ERROR PROPAGATION TESTS +# ===================================================================== + + +class TestErrorPropagation: + def test_task_error_is_raised_on_replay(self): + @workflow + async def wf(x: int): + return await double(x=x) + + with pytest.raises(TaskError, match="double"): + _run_workflow( + wf, + { + "completed_steps": { + "double": { + "__wmill_error": True, + "message": "Task 'double' failed", + "result": {"message": "boom"}, + } + } + }, + {"x": 5}, + ) + + def test_error_catchable_with_try_except(self): + @workflow + async def wf(x: int): + try: + result = await double(x=x) + return {"success": True, "result": result} + except Exception as e: + return {"success": False, "error": str(e)} + + r = _run_workflow( + wf, + { + "completed_steps": { + "double": { + "__wmill_error": True, + "message": "Task 'double' failed", + "result": {}, + } + } + }, + {"x": 5}, + ) + assert r["type"] == "complete" + assert r["result"]["success"] is False + assert "double" in r["result"]["error"] + + def test_retry_pattern_with_try_except_loop(self): + @workflow + async def wf(x: int): + for i in range(3): + try: + result = await double(x=x) + return {"result": result, "attempts": i + 1} + except Exception: + if i == 2: + raise + + # First double fails, second succeeds + r = _run_workflow( + wf, + { + "completed_steps": { + "double": {"__wmill_error": True, "message": "temporary", "result": {}}, + "double_2": 10, + } + }, + {"x": 5}, + ) + assert r["type"] == "complete" + assert r["result"]["result"] == 10 + assert r["result"]["attempts"] == 2 + + def test_non_error_object_with_error_false(self): + @workflow + async def wf(): + val = await double(x=5) + return val + + r = _run_workflow( + wf, + {"completed_steps": {"double": {"__wmill_error": False, "data": "ok"}}}, + {}, + ) + assert r["type"] == "complete" + assert r["result"] == {"__wmill_error": False, "data": "ok"} + + def test_inline_step_error(self): + @workflow + async def wf(): + try: + val = await step("risky", lambda: 42) + return {"val": val} + except Exception as e: + return {"caught": str(e)} + + r = _run_workflow( + wf, + {"completed_steps": {"risky": {"__wmill_error": True, "message": "step failed", "result": {}}}}, + {}, + ) + assert r["type"] == "complete" + assert "step failed" in r["result"]["caught"] + + +# ===================================================================== +# TASK OPTIONS TESTS +# ===================================================================== + + +class TestTaskOptions: + def test_options_forwarded_in_dispatch(self): + @task(timeout=600, tag="gpu", cache_ttl=3600, priority=10) + async def heavy(x: int): + return x + + @workflow + async def wf(x: int): + return await heavy(x=x) + + r = _run_workflow(wf, {}, {"x": 42}) + assert r["type"] == "dispatch" + step_info = r["steps"][0] + assert step_info["timeout"] == 600 + assert step_info["tag"] == "gpu" + assert step_info["cache_ttl"] == 3600 + assert step_info["priority"] == 10 + + def test_task_without_options_has_no_extra_fields(self): + @task + async def simple(x: int): + return x + + @workflow + async def wf(x: int): + return await simple(x=x) + + r = _run_workflow(wf, {}, {"x": 1}) + step_info = r["steps"][0] + assert "timeout" not in step_info + assert "tag" not in step_info + + def test_concurrency_options_forwarded(self): + @task(concurrency_limit=5, concurrency_key="my-key", concurrency_time_window_s=60) + async def limited(x: int): + return x + + @workflow + async def wf(x: int): + return await limited(x=x) + + r = _run_workflow(wf, {}, {"x": 1}) + step_info = r["steps"][0] + assert step_info["concurrent_limit"] == 5 + assert step_info["concurrency_key"] == "my-key" + assert step_info["concurrency_time_window_s"] == 60 + + +# ===================================================================== +# SLEEP TESTS +# ===================================================================== + + +class TestSleep: + def test_sleep_returns_sleep_output(self): + @workflow + async def wf(): + await double(x=1) + await sleep(60) + await add_one(x=2) + return "done" + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2}}, + {}, + ) + assert r["type"] == "sleep" + assert r["key"] == "sleep" + assert r["seconds"] == 60 + + def test_sleep_completes_on_replay(self): + @workflow + async def wf(): + await double(x=1) + await sleep(60) + await add_one(x=2) + return "done" + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2, "sleep": True}}, + {}, + ) + assert r["type"] == "dispatch" + assert r["steps"][0]["name"] == "add_one" + assert r["steps"][0]["key"] == "add_one" + + def test_all_steps_with_sleep_complete(self): + @workflow + async def wf(): + await double(x=1) + await sleep(60) + await add_one(x=2) + return "done" + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2, "sleep": True, "add_one": 3}}, + {}, + ) + assert r["type"] == "complete" + assert r["result"] == "done" + + def test_sleep_enforces_minimum(self): + @workflow + async def wf(): + await sleep(0) + return "done" + + r = _run_workflow(wf, {}, {}) + assert r["seconds"] == 1 + + +# ===================================================================== +# PARALLEL UTILITY TESTS +# ===================================================================== + + +class TestParallel: + def test_dispatches_all_items(self): + @workflow + async def wf(): + results = await parallel([1, 2, 3], double) + return results + + r = _run_workflow(wf, {}, {}) + assert r["type"] == "dispatch" + assert r["mode"] == "parallel" + assert len(r["steps"]) == 3 + + def test_completes_with_all_results(self): + @workflow + async def wf(): + results = await parallel([1, 2, 3], double) + return results + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2, "double_2": 4, "double_3": 6}}, + {}, + ) + assert r["type"] == "complete" + assert r["result"] == [2, 4, 6] + + def test_batched_dispatches_first_batch(self): + @workflow + async def wf(): + results = await parallel([1, 2, 3, 4, 5], double, concurrency=2) + return results + + r = _run_workflow(wf, {}, {}) + assert r["type"] == "dispatch" + assert r["mode"] == "parallel" + assert len(r["steps"]) == 2 + + def test_batched_dispatches_second_batch(self): + @workflow + async def wf(): + results = await parallel([1, 2, 3, 4, 5], double, concurrency=2) + return results + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2, "double_2": 4}}, + {}, + ) + assert r["type"] == "dispatch" + assert len(r["steps"]) == 2 + + def test_batched_completes_with_all_results(self): + @workflow + async def wf(): + results = await parallel([1, 2, 3, 4, 5], double, concurrency=2) + return results + + r = _run_workflow( + wf, + {"completed_steps": {"double": 2, "double_2": 4, "double_3": 6, "double_4": 8, "double_5": 10}}, + {}, + ) + assert r["type"] == "complete" + assert r["result"] == [2, 4, 6, 8, 10] + + def test_empty_items_returns_empty(self): + @workflow + async def wf(): + results = await parallel([], double) + return results + + r = _run_workflow(wf, {}, {}) + assert r["type"] == "complete" + assert r["result"] == [] diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 3b9a9f3f45..f5acd7f58c 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2151,69 +2151,6 @@ def ducklake(name: str = "main") -> DucklakeClient: """ return _client.ducklake(name) -def task(*args, **kwargs): - """Decorator to mark a function as a workflow task. - - When executed inside a Windmill job, the decorated function runs as a - separate workflow step. Outside Windmill, it executes normally. - - Args: - tag: Optional worker tag for execution - - Returns: - Decorated function - """ - from inspect import signature - - def f(func, tag: str | None = None): - if ( - os.environ.get("WM_JOB_ID") is None - or os.environ.get("MAIN_OVERRIDE") == func.__name__ - ): - - def inner(*args, **kwargs): - return func(*args, **kwargs) - - return inner - else: - - def inner(*args, **kwargs): - global _client - if _client is None: - _client = Windmill() - w_id = os.environ.get("WM_WORKSPACE") - job_id = os.environ.get("WM_JOB_ID") - f_name = func.__name__ - json = kwargs - params = list(signature(func).parameters) - for i, arg in enumerate(args): - if i < len(params): - p = params[i] - key = p - if key not in kwargs: - json[key] = arg - - params = {} - if tag is not None: - params["tag"] = tag - w_as_code_response = _client.post( - f"/w/{w_id}/jobs/run/workflow_as_code/{job_id}/{f_name}", - json={"args": json}, - params=params, - ) - job_id = w_as_code_response.text - print(f"Executing task {func.__name__} on job {job_id}") - job_result = _client.wait_job(job_id) - print(f"Task {func.__name__} ({job_id}) completed") - return job_result - - return inner - - if len(args) == 1 and len(kwargs) == 0 and callable(args[0]): - return f(args[0], None) - else: - return lambda x: f(x, kwargs.get("tag")) - def parse_resource_syntax(s: str) -> Optional[str]: """Parse resource syntax from string.""" if s is None: @@ -2413,7 +2350,495 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]: name = name schema = None if ":" in name: - name, schema = name.split(":", 1) + name, schema = name.split(":", 1) if not name: name = "main" return name, schema + + +# ── Workflow-as-Code SDK ────────────────────────────────────────────── + +import asyncio as _asyncio +import contextvars as _contextvars + + +class _StepSuspend(BaseException): + """Raised to suspend workflow execution. Inherits from BaseException + so it is not caught by bare `except Exception:` blocks.""" + + def __init__(self, dispatch_info: dict): + self.dispatch_info = dispatch_info + + +class TaskError(Exception): + """Raised when a WAC task step failed. + + Attributes: + step_key: The checkpoint key of the failed step. + child_job_id: The UUID of the failed child job. + result: The error result from the child job. + """ + + def __init__(self, message: str, *, step_key: str = "", child_job_id: str = "", result=None): + super().__init__(message) + self.step_key = step_key + self.child_job_id = child_job_id + self.result = result + + +_workflow_ctx: _contextvars.ContextVar["WorkflowCtx"] = _contextvars.ContextVar( + "_workflow_ctx" +) + + +class WorkflowCtx: + """Internal context for workflow replay/suspension. + + Not user-facing — set implicitly by ``@workflow`` via contextvars. + """ + + def __init__(self, checkpoint: dict | None = None): + checkpoint = checkpoint or {} + self._completed: dict = checkpoint.get("completed_steps", {}) + self._counters: dict[str, int] = {} + self._pending: list = [] + self._executing_key: str | None = checkpoint.get("_executing_key") + + def _alloc_key(self, name: str = "step") -> str: + """Name-based key: ``double`` for first call, ``double_2``, ``double_3`` for subsequent.""" + n = self._counters.get(name, 0) + 1 + self._counters[name] = n + return name if n == 1 else f"{name}_{n}" + + def _next_step(self, name: str, script: str, func=None, dispatch_type: str = "inline", _task_options: Optional[dict] = None, **kwargs): + """Return an awaitable that either resolves from cache or suspends.""" + key = self._alloc_key(name or script or "step") + + if key in self._completed: + val = self._completed[key] + if isinstance(val, dict) and val.get("__wmill_error"): + raise TaskError( + val.get("message", f"Task '{name}' failed"), + step_key=val.get("step_key", ""), + child_job_id=val.get("child_job_id", ""), + result=val.get("result"), + ) + return self._resolved(val) + + if self._executing_key is not None: + if key == self._executing_key: + return self._execute_directly(func, **kwargs) + else: + return self._never_resolve() + + info = {"name": name or key, "script": script or key, "args": kwargs, "key": key, "dispatch_type": dispatch_type} + if _task_options: + for opt_key in ("timeout", "tag", "cache_ttl", "priority", "concurrent_limit", "concurrency_key", "concurrency_time_window_s"): + if opt_key in _task_options and _task_options[opt_key] is not None: + info[opt_key] = _task_options[opt_key] + self._pending.append(info) + return self._suspend() + + async def _resolved(self, value): + return value + + async def _execute_directly(self, func, **kwargs): + result = func(**kwargs) + if _asyncio.iscoroutine(result): + result = await result + raise _StepSuspend({"mode": "step_complete", "steps": [], "result": result}) + + async def _never_resolve(self): + await _asyncio.Future() + + async def _suspend(self): + steps = list(self._pending) + self._pending.clear() + raise _StepSuspend( + { + "mode": "parallel" if len(steps) > 1 else "sequential", + "steps": steps, + } + ) + + async def _wait_for_approval( + self, timeout: int = 1800, form: dict | None = None + ): + key = self._alloc_key("approval") + + if key in self._completed: + return self._completed[key] + + if self._executing_key is not None: + await _asyncio.Future() + + raise _StepSuspend({ + "mode": "approval", + "key": key, + "timeout": timeout, + "form": form, + "steps": [], + }) + + async def _sleep(self, seconds: int): + key = self._alloc_key("sleep") + + if key in self._completed: + return + + if self._executing_key is not None: + await _asyncio.Future() + + raise _StepSuspend({ + "mode": "sleep", + "key": key, + "seconds": max(1, int(seconds)), + "steps": [], + }) + + async def _run_inline_step(self, name: str, fn): + key = self._alloc_key(name or "step") + + if key in self._completed: + val = self._completed[key] + if isinstance(val, dict) and val.get("__wmill_error"): + raise TaskError( + val.get("message", f"Step '{name}' failed"), + step_key=val.get("step_key", ""), + child_job_id=val.get("child_job_id", ""), + result=val.get("result"), + ) + return val + + if self._executing_key is not None: + await _asyncio.Future() + + result = fn() + if _asyncio.iscoroutine(result): + result = await result + + raise _StepSuspend({ + "mode": "inline_checkpoint", + "steps": [], + "key": key, + "result": result, + }) + + +def task( + _func=None, + *, + path: Optional[str] = None, + tag: Optional[str] = None, + timeout: Optional[int] = None, + cache_ttl: Optional[int] = None, + priority: Optional[int] = None, + concurrency_limit: Optional[int] = None, + concurrency_key: Optional[str] = None, + concurrency_time_window_s: Optional[int] = None, +): + """Decorator that marks a function as a workflow task. + + Works in both WAC v1 (sync, HTTP-based dispatch) and WAC v2 + (async, checkpoint/replay) modes: + + - **v2 (inside @workflow)**: dispatches as a checkpoint step. + - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. + - **Standalone**: executes the function body directly. + + Usage:: + + @task + async def extract_data(url: str): ... + + @task(path="f/external_script", timeout=600, tag="gpu") + async def run_external(x: int): ... + """ + from inspect import signature as _sig + + _task_opts = { + "timeout": timeout, + "tag": tag, + "cache_ttl": cache_ttl, + "priority": priority, + "concurrent_limit": concurrency_limit, + "concurrency_key": concurrency_key, + "concurrency_time_window_s": concurrency_time_window_s, + } + # Remove None values + _task_opts = {k: v for k, v in _task_opts.items() if v is not None} or None + + def decorator(func): + task_path = path + task_name = func.__name__ + + _params_list = list(_sig(func).parameters) + + def _merge_args(args, kwargs): + merged = dict(kwargs) + for i, arg in enumerate(args): + if i < len(_params_list): + key = _params_list[i] + if key not in merged: + merged[key] = arg + else: + merged[f"arg{i}"] = arg + return merged + + @functools.wraps(func) + def wrapper(*args, **kwargs): + # WAC v2: inside a @workflow context + ctx = _workflow_ctx.get(None) + if ctx is not None: + script = task_path if task_path else task_name + merged = _merge_args(args, kwargs) + return ctx._next_step(task_name, script, func, _task_options=_task_opts, **merged) + + # WAC v1: running inside a Windmill job but not in a @workflow + if ( + os.environ.get("WM_JOB_ID") is not None + and os.environ.get("MAIN_OVERRIDE") != func.__name__ + ): + global _client + if _client is None: + _client = Windmill() + w_id = os.environ.get("WM_WORKSPACE") + job_id = os.environ.get("WM_JOB_ID") + json_args = _merge_args(args, kwargs) + api_params = {} + if tag is not None: + api_params["tag"] = tag + resp = _client.post( + f"/w/{w_id}/jobs/run/workflow_as_code/{job_id}/{func.__name__}", + json={"args": json_args}, + params=api_params, + ) + child_job_id = resp.text + print(f"Executing task {func.__name__} on job {child_job_id}") + job_result = _client.wait_job(child_job_id) + print(f"Task {func.__name__} ({child_job_id}) completed") + return job_result + + # Standalone — execute directly + return func(*args, **kwargs) + + wrapper._is_task = True + wrapper._task_path = task_path + return wrapper + + if _func is not None: + # @task without parentheses + return decorator(_func) + # @task() or @task(path="...", tag="...") + return decorator + + +def task_script( + path: str, + *, + timeout: Optional[int] = None, + tag: Optional[str] = None, + cache_ttl: Optional[int] = None, + priority: Optional[int] = None, + concurrency_limit: Optional[int] = None, + concurrency_key: Optional[str] = None, + concurrency_time_window_s: Optional[int] = None, +): + """Create a task that dispatches to a separate Windmill script. + + Usage:: + + extract = task_script("f/data/extract", timeout=600) + + @workflow + async def main(): + data = await extract(url="https://...") + """ + name = path.rsplit("/", 1)[-1] + _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None + + def wrapper(**kwargs): + ctx = _workflow_ctx.get(None) + if ctx is not None: + return ctx._next_step(name, path, dispatch_type="script", _task_options=_opts, **kwargs) + raise RuntimeError(f'task_script("{path}") can only be called inside a @workflow') + + wrapper.__name__ = name + wrapper._is_task = True + wrapper._task_path = path + return wrapper + + +def task_flow( + path: str, + *, + timeout: Optional[int] = None, + tag: Optional[str] = None, + cache_ttl: Optional[int] = None, + priority: Optional[int] = None, + concurrency_limit: Optional[int] = None, + concurrency_key: Optional[str] = None, + concurrency_time_window_s: Optional[int] = None, +): + """Create a task that dispatches to a separate Windmill flow. + + Usage:: + + pipeline = task_flow("f/etl/pipeline", priority=10) + + @workflow + async def main(): + result = await pipeline(input=data) + """ + name = path.rsplit("/", 1)[-1] + _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s}.items() if v is not None} or None + + def wrapper(**kwargs): + ctx = _workflow_ctx.get(None) + if ctx is not None: + return ctx._next_step(name, path, dispatch_type="flow", _task_options=_opts, **kwargs) + raise RuntimeError(f'task_flow("{path}") can only be called inside a @workflow') + + wrapper.__name__ = name + wrapper._is_task = True + wrapper._task_path = path + return wrapper + + +def workflow(func): + """Decorator marking an async function as a workflow-as-code entry point. + + The function must be **deterministic**: given the same inputs it must call + tasks in the same order on every replay. Branching on task results is fine + (results are replayed from checkpoint), but branching on external state + (current time, random values, external API calls) must use ``step()`` to + checkpoint the value so replays see the same result. + """ + func._is_workflow = True + return func + + +async def step(name: str, fn): + """Execute ``fn`` inline and checkpoint the result. + + On replay the cached value is returned without re-executing ``fn``. + Use for lightweight deterministic operations (timestamps, random IDs, + config reads) that should not incur the overhead of a child job. + """ + ctx: WorkflowCtx | None = _workflow_ctx.get(None) + if ctx is not None: + return await ctx._run_inline_step(name, fn) + result = fn() + if _asyncio.iscoroutine(result): + result = await result + return result + + +async def sleep(seconds: int): + """Server-side sleep — suspend the workflow for the given duration without holding a worker. + + Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. + Outside a workflow, falls back to ``asyncio.sleep``. + """ + ctx: WorkflowCtx | None = _workflow_ctx.get(None) + if ctx is not None: + return await ctx._sleep(seconds) + await _asyncio.sleep(seconds) + + +async def wait_for_approval( + timeout: int = 1800, + form: dict | None = None, +) -> dict: + """Suspend the workflow and wait for an external approval. + + Use ``get_resume_urls()`` (wrapped in ``step()``) to obtain + resume/cancel/approval URLs before calling this function. + + Returns a dict with ``value`` (form data), ``approver``, and ``approved``. + + Example:: + + urls = await step("urls", lambda: get_resume_urls()) + await step("notify", lambda: send_email(urls["approvalPage"])) + result = await wait_for_approval(timeout=3600) + """ + ctx: WorkflowCtx | None = _workflow_ctx.get(None) + if ctx is not None: + return await ctx._wait_for_approval(timeout=timeout, form=form) + raise RuntimeError("wait_for_approval can only be called inside a @workflow") + + +async def parallel(items, fn, *, concurrency: Optional[int] = None): + """Process items in parallel with optional concurrency control. + + Each item is processed by calling ``fn(item)``, which should be a @task. + Items are dispatched in batches of ``concurrency`` (default: all at once). + + Example:: + + @task + async def process(item: str): + ... + + results = await parallel(items, process, concurrency=5) + """ + if not items: + return [] + batch_size = concurrency if concurrency and concurrency > 0 else len(items) + results = [] + for i in range(0, len(items), batch_size): + batch = items[i : i + batch_size] + batch_results = await _asyncio.gather(*(fn(item) for item in batch)) + results.extend(batch_results) + return results + + +async def _run_workflow_async(func, checkpoint: dict, input_args: dict): + ctx = WorkflowCtx(checkpoint) + token = _workflow_ctx.set(ctx) + try: + result = await func(**input_args) + # Flush any unawaited tasks (e.g. forgotten await on last statement) + if ctx._pending: + steps = list(ctx._pending) + ctx._pending.clear() + return { + "type": "dispatch", + "mode": "parallel" if len(steps) > 1 else "sequential", + "steps": steps, + } + return {"type": "complete", "result": result} + except _StepSuspend as e: + info = e.dispatch_info + mode = info.get("mode") + if mode == "step_complete": + return {"type": "complete", "result": info.get("result")} + if mode == "inline_checkpoint": + return { + "type": "inline_checkpoint", + "key": info["key"], + "result": info.get("result"), + } + if mode == "approval": + return { + "type": "approval", + "key": info["key"], + "timeout": info.get("timeout"), + "form": info.get("form"), + } + if mode == "sleep": + return { + "type": "sleep", + "key": info["key"], + "seconds": info.get("seconds"), + } + return {"type": "dispatch", **info} + finally: + _workflow_ctx.reset(token) + + +def _run_workflow(func, checkpoint: dict, input_args: dict): + """Synchronous wrapper that runs the workflow coroutine to completion + or until it suspends.""" + return _asyncio.run(_run_workflow_async(func, checkpoint, input_args)) diff --git a/typescript-client/build.sh b/typescript-client/build.sh index bea84d0ebc..f765340bad 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -40,7 +40,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, type TaskOptions, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" # Build default export by combining client utilities + services # This preserves backward compatibility for `import wmill from "windmill-client"` @@ -68,6 +68,17 @@ import { getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, + taskScript, + taskFlow, + workflow, + step, + sleep, + parallel, + waitForApproval, + WorkflowCtx, + _workflowCtx, + setWorkflowCtx, + StepSuspend, runScript, runScriptAsync, runScriptByPath, @@ -141,6 +152,17 @@ const wmill = { getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, + taskScript, + taskFlow, + workflow, + step, + sleep, + parallel, + waitForApproval, + WorkflowCtx, + _workflowCtx, + setWorkflowCtx, + StepSuspend, runScript, runScriptAsync, runScriptByPath, diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 1b32a4119e..41054f5334 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -311,45 +311,45 @@ export async function getResultMaybe(jobId: string): Promise { } const STRIP_COMMENTS = /(\/\/.*$)|(\/\*[\s\S]*?\*\/)|(\s*=[^,\)]*(('(?:\\'|[^'\r\n])*')|("(?:\\"|[^"\r\n])*"))|(\s*=[^,\)]*))/gm; -const ARGUMENT_NAMES = /([^\s,]+)/g; function getParamNames(func: Function): string[] { const fnStr = func.toString().replace(STRIP_COMMENTS, ""); - let result: string[] | null = fnStr - .slice(fnStr.indexOf("(") + 1, fnStr.indexOf(")")) - .match(ARGUMENT_NAMES); - if (result === null) result = []; - return result; -} - -/** - * Wrap a function to execute as a Windmill task within a flow context - * @param f - Function to wrap as a task - * @returns Async wrapper function that executes as a Windmill job - */ -export function task(f: (_: P) => T): (_: P) => Promise { - return async (...y) => { - const args: Record = {}; - const paramNames = getParamNames(f); - y.forEach((x, i) => (args[paramNames[i]] = x)); - let req = await fetch( - `${OpenAPI.BASE}/w/${getWorkspace()}/jobs/run/workflow_as_code/${getEnv( - "WM_JOB_ID" - )}/${f.name}`, - { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${getEnv("WM_TOKEN")}`, - }, - body: JSON.stringify({ args }), - } - ); - let jobId = await req.text(); - console.log(`Started task ${f.name} as job ${jobId}`); - let r = await waitJob(jobId); - console.log(`Task ${f.name} (${jobId}) completed`); - return r; - }; + // Find the matching closing paren for the parameter list, handling nesting + const openIdx = fnStr.indexOf("("); + if (openIdx === -1) return []; + let depth = 1; + let closeIdx = openIdx + 1; + for (; closeIdx < fnStr.length && depth > 0; closeIdx++) { + if (fnStr[closeIdx] === "(") depth++; + else if (fnStr[closeIdx] === ")") depth--; + } + const paramStr = fnStr.slice(openIdx + 1, closeIdx - 1).trim(); + if (!paramStr) return []; + // Split on commas at depth 0 (skip nested parens, angle brackets, braces) + const params: string[] = []; + let current = ""; + let d = 0; + for (const ch of paramStr) { + if ("(<{".includes(ch)) d++; + else if (")>}".includes(ch)) d--; + if (ch === "," && d === 0) { + params.push(current.trim()); + current = ""; + } else { + current += ch; + } + } + if (current.trim()) params.push(current.trim()); + // Extract the parameter name from each param (strip type annotations, destructuring, rest) + return params.map((p) => { + // Remove rest operator + p = p.replace(/^\.\.\./, ""); + // For destructured params like { url, depth }: Config, use a positional fallback + if (p.startsWith("{") || p.startsWith("[")) return ""; + // Strip type annotation (e.g. "x: number" -> "x", "x?: string" -> "x") + const colonIdx = p.indexOf(":"); + if (colonIdx !== -1) p = p.slice(0, colonIdx); + return p.replace(/\?$/, "").trim(); + }).filter(Boolean); } /** @@ -1448,3 +1448,421 @@ export function parseS3Object(s3Object: S3Object): S3ObjectRecord { function parseVariableSyntax(s: string) { if (s.startsWith("var://")) return s.substring(6); } + +// ── Workflow-as-Code SDK ────────────────────────────────────────────── + +export class StepSuspend extends Error { + constructor(public dispatchInfo: Record) { + super("__step_suspend__"); + this.name = "StepSuspend"; + } +} + +export interface TaskOptions { + timeout?: number; + tag?: string; + cache_ttl?: number; + priority?: number; + concurrency_limit?: number; + concurrency_key?: string; + concurrency_time_window_s?: number; +} + +export let _workflowCtx: WorkflowCtx | null = null; +export function setWorkflowCtx(ctx: WorkflowCtx | null) { + _workflowCtx = ctx; + Reflect.set(globalThis, "__wmill_wf_ctx", ctx); +} + + +export class WorkflowCtx { + private completed: Record; + private counters: Record = {}; + private pending: Array<{ + name: string; + script: string; + args: Record; + key: string; + dispatch_type: string; + [k: string]: any; + }> = []; + private _suspended = false; + /** When set, the task matching this key executes its inner function directly */ + _executingKey: string | null; + + constructor(checkpoint: Record = {}) { + this.completed = checkpoint?.completed_steps ?? {}; + this._executingKey = checkpoint?._executing_key ?? null; + } + + /** Name-based key: `double` for first call, `double_2`, `double_3` for subsequent. */ + _allocKey(name: string): string { + const n = (this.counters[name] ?? 0) + 1; + this.counters[name] = n; + return n === 1 ? name : `${name}_${n}`; + } + + _nextStep( + name: string, + script: string, + args: Record = {}, + dispatch_type: string = "inline", + options?: TaskOptions, + ): PromiseLike { + const key = this._allocKey(name || script || "step"); + + if (key in this.completed) { + const value = this.completed[key]; + if (value && typeof value === "object" && (value as any).__wmill_error) { + const err = new Error((value as any).message || `Task '${name}' failed`); + (err as any).result = (value as any).result; + (err as any).step_key = (value as any).step_key; + (err as any).child_job_id = (value as any).child_job_id; + return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } } as PromiseLike; + } + return { then: (resolve: any) => resolve(value) }; + } + + // If this is a child job executing a specific step, return null to signal + // that the task wrapper should run the inner function directly + if (this._executingKey === key) { + return { then: (resolve: any) => resolve(null), _execute_directly: true } as any; + } + + // In child job mode (_executingKey is set), non-matching uncompleted steps + // should never resolve or throw — the matching step will throw step_complete + // which terminates the workflow. Returning a never-resolving thenable prevents + // race conditions where a non-matching step's StepSuspend fires before step_complete. + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } + + const stepInfo: any = { name: name || key, script: script || key, args, key, dispatch_type }; + if (options) { + if (options.timeout !== undefined) stepInfo.timeout = options.timeout; + if (options.tag !== undefined) stepInfo.tag = options.tag; + if (options.cache_ttl !== undefined) stepInfo.cache_ttl = options.cache_ttl; + if (options.priority !== undefined) stepInfo.priority = options.priority; + if (options.concurrency_limit !== undefined) stepInfo.concurrent_limit = options.concurrency_limit; + if (options.concurrency_key !== undefined) stepInfo.concurrency_key = options.concurrency_key; + if (options.concurrency_time_window_s !== undefined) stepInfo.concurrency_time_window_s = options.concurrency_time_window_s; + } + this.pending.push(stepInfo); + return { + then: (): never => { + // Only the first .then() call throws with all accumulated steps. + // Subsequent calls (e.g. from Promise.all resolving other thenables) + // also throw (they'll be caught by the same handler). + if (this._suspended) return new Promise(() => {}) as never; + this._suspended = true; + const steps = [...this.pending]; + this.pending = []; + throw new StepSuspend({ + mode: steps.length > 1 ? "parallel" : "sequential", + steps, + }); + }, + }; + } + /** Return and clear any pending (unawaited) steps. */ + _flushPending(): Array<{ name: string; script: string; args: Record; key: string; dispatch_type: string }> { + const steps = [...this.pending]; + this.pending = []; + return steps; + } + + _waitForApproval(options?: { + timeout?: number; + form?: object; + }): PromiseLike<{ value: any; approver: string; approved: boolean }> { + const key = this._allocKey("approval"); + + if (key in this.completed) { + const value = this.completed[key]; + return { then: (resolve: any) => resolve(value) }; + } + + // In child job mode, return never-resolving thenable (same as _nextStep) + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } + + // Throw immediately — approval is always a blocking step + throw new StepSuspend({ + mode: "approval", + key, + timeout: options?.timeout ?? 1800, + form: options?.form, + steps: [], + }); + } + + _sleep(seconds: number): PromiseLike { + const key = this._allocKey("sleep"); + + if (key in this.completed) { + return { then: (resolve: any) => resolve(undefined) }; + } + + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } + + throw new StepSuspend({ + mode: "sleep", + key, + seconds: Math.max(1, Math.round(seconds)), + steps: [], + }); + } + + async _runInlineStep(name: string, fn: () => T | Promise): Promise { + const key = this._allocKey(name || "step"); + + if (key in this.completed) { + const value = this.completed[key]; + if (value && typeof value === "object" && (value as any).__wmill_error) { + const err = new Error((value as any).message || `Step '${name}' failed`); + (err as any).result = (value as any).result; + (err as any).step_key = (value as any).step_key; + (err as any).child_job_id = (value as any).child_job_id; + throw err; + } + return value as T; + } + + if (this._executingKey !== null) { + return new Promise(() => {}); + } + + const result = await fn(); + throw new StepSuspend({ mode: "inline_checkpoint", steps: [], key, result }); + } +} + +export async function sleep(seconds: number): Promise { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (ctx) { + return ctx._sleep(seconds) as Promise; + } + // Outside workflow context, just wait locally + await new Promise((r) => setTimeout(r, seconds * 1000)); +} + +export async function step(name: string, fn: () => T | Promise): Promise { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (ctx) { + return ctx._runInlineStep(name, fn); + } + return fn(); +} + +/** + * Wrap an async function as a workflow task. + * + * @example + * const extract_data = task(async (url: string) => { ... }); + * const run_external = task("f/external_script", async (x: number) => { ... }); + * + * Inside a `workflow()`, calling a task dispatches it as a step. + * Outside a workflow, the function body executes directly. + */ +export function task Promise>( + fnOrPath: T | string, + maybeFnOrOptions?: T | TaskOptions, + maybeOptions?: TaskOptions, +): T { + let fn: T; + let taskPath: string | undefined; + let taskOptions: TaskOptions | undefined; + + if (typeof fnOrPath === "string") { + taskPath = fnOrPath; + fn = maybeFnOrOptions as T; + taskOptions = maybeOptions; + } else { + fn = fnOrPath; + taskOptions = maybeFnOrOptions as TaskOptions | undefined; + } + + const taskName = fn.name || taskPath || ""; + + // NOT async — in workflow context we return the thenable directly so that + // unawaited task calls leave the step in ctx.pending (for _flushPending). + // An async wrapper would auto-resolve the thenable in a microtask, calling + // .then() which throws StepSuspend and empties pending before the caller + // can flush. + const wrapper = function (...args: any[]) { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (ctx) { + // Inside a workflow with checkpoint/replay context — dispatch as step + const script = taskPath ?? taskName; + const paramNames = getParamNames(fn); + const kwargs: Record = {}; + for (let i = 0; i < args.length; i++) { + if (paramNames[i]) { + kwargs[paramNames[i]] = args[i]; + } else { + kwargs[`arg${i}`] = args[i]; + } + } + const stepResult = ctx._nextStep(taskName, script, kwargs, "inline", taskOptions); + // If this step should execute directly (child job mode), run the inner function + // and throw StepSuspend with mode "step_complete" to signal that we're done + if ((stepResult as any)?._execute_directly) { + return (async () => { + const result = await fn(...args); + throw new StepSuspend({ mode: "step_complete", steps: [], result }); + })(); + } + return stepResult; + } else if (getEnv("WM_JOB_ID") && !getEnv("WM_FLOW_JOB_ID")) { + // Inside a Windmill root job without checkpoint context — v1 HTTP dispatch + // WM_FLOW_JOB_ID is set on child jobs, so we skip dispatch for those + return (async () => { + const paramNames = getParamNames(fn); + const kwargs: Record = {}; + args.forEach((x, i) => (kwargs[paramNames[i]] = x)); + let req = await fetch( + `${OpenAPI.BASE}/w/${getWorkspace()}/jobs/run/workflow_as_code/${getEnv( + "WM_JOB_ID" + )}/${taskName}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${getEnv("WM_TOKEN")}`, + }, + body: JSON.stringify({ args: kwargs }), + } + ); + let jobId = await req.text(); + console.log(`Started task ${taskName} as job ${jobId}`); + let r = await waitJob(jobId); + console.log(`Task ${taskName} (${jobId}) completed`); + return r; + })(); + } else { + // Standalone — execute directly + return fn(...args); + } + } as unknown as T; + + Object.defineProperty(wrapper, "name", { value: taskName }); + (wrapper as any)._is_task = true; + (wrapper as any)._task_path = taskPath; + return wrapper; +} + +/** + * Create a task that dispatches to a separate Windmill script. + * + * @example + * const extract = taskScript("f/data/extract"); + * // inside workflow: await extract({ url: "https://..." }) + */ +export function taskScript(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike { + const name = path.split("/").pop() || path; + const wrapper = function (...args: any[]) { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (ctx) { + const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null + ? args[0] + : args.reduce((acc, v, i) => { acc[`arg${i}`] = v; return acc; }, {} as Record); + return ctx._nextStep(name, path, kwargs, "script", options); + } + throw new Error(`taskScript("${path}") can only be called inside a workflow()`); + }; + Object.defineProperty(wrapper, "name", { value: name }); + (wrapper as any)._is_task = true; + (wrapper as any)._task_path = path; + return wrapper; +} + +/** + * Create a task that dispatches to a separate Windmill flow. + * + * @example + * const pipeline = taskFlow("f/etl/pipeline"); + * // inside workflow: await pipeline({ input: data }) + */ +export function taskFlow(path: string, options?: TaskOptions): (...args: any[]) => PromiseLike { + const name = path.split("/").pop() || path; + const wrapper = function (...args: any[]) { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (ctx) { + const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null + ? args[0] + : args.reduce((acc, v, i) => { acc[`arg${i}`] = v; return acc; }, {} as Record); + return ctx._nextStep(name, path, kwargs, "flow", options); + } + throw new Error(`taskFlow("${path}") can only be called inside a workflow()`); + }; + Object.defineProperty(wrapper, "name", { value: name }); + (wrapper as any)._is_task = true; + (wrapper as any)._task_path = path; + return wrapper; +} + +/** + * Mark an async function as a workflow-as-code entry point. + * + * The function must be **deterministic**: given the same inputs it must call + * tasks in the same order on every replay. Branching on task results is fine + * (results are replayed from checkpoint), but branching on external state + * (current time, random values, external API calls) must use `step()` to + * checkpoint the value so replays see the same result. + */ +export function workflow(fn: (...args: any[]) => Promise) { + (fn as any)._is_workflow = true; + return fn; +} + +/** + * Suspend the workflow and wait for an external approval. + * + * Use `getResumeUrls()` (wrapped in `step()`) to obtain resume/cancel/approvalPage + * URLs before calling this function. + * + * @example + * const urls = await step("urls", () => getResumeUrls()); + * await step("notify", () => sendEmail(urls.approvalPage)); + * const { value, approver } = await waitForApproval({ timeout: 3600 }); + */ +export function waitForApproval(options?: { + timeout?: number; + form?: object; +}): PromiseLike<{ value: any; approver: string; approved: boolean }> { + const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + if (!ctx) { + throw new Error("waitForApproval can only be called inside a workflow()"); + } + return ctx._waitForApproval(options); +} + +/** + * Process items in parallel with optional concurrency control. + * + * Each item is processed by calling `fn(item)`, which should be a task(). + * Items are dispatched in batches of `concurrency` (default: all at once). + * + * @example + * const process = task(async (item: string) => { ... }); + * const results = await parallel(items, process, { concurrency: 5 }); + */ +export async function parallel( + items: T[], + fn: (item: T) => PromiseLike | R, + options?: { concurrency?: number }, +): Promise { + const concurrency = options?.concurrency ?? items.length; + if (concurrency <= 0 || items.length === 0) return []; + const results: R[] = []; + for (let i = 0; i < items.length; i += concurrency) { + const batch = items.slice(i, i + concurrency); + const batchResults = await Promise.all(batch.map((item) => fn(item))); + results.push(...batchResults); + } + return results; +} + diff --git a/typescript-client/package-lock.json b/typescript-client/package-lock.json index e299444082..a25b51cbcb 100644 --- a/typescript-client/package-lock.json +++ b/typescript-client/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-client", - "version": "1.618.3", + "version": "1.651.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-client", - "version": "1.618.3", + "version": "1.651.1", "license": "Apache 2.0", "devDependencies": { "@types/node": "^20.17.16", diff --git a/typescript-client/tests/e2e_wac.py b/typescript-client/tests/e2e_wac.py new file mode 100644 index 0000000000..28bb629bbe --- /dev/null +++ b/typescript-client/tests/e2e_wac.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +""" +E2E test for WAC v2 (Workflow-as-Code) with the _executing_key approach. + +This test: +1. Creates a preview job with a WAC v2 bun script +2. Waits for the parent to suspend (dispatch child jobs) +3. Waits for child jobs to complete +4. Waits for parent to unsuspend and complete +5. Checks the final result + +Usage: + python3 typescript-client/tests/e2e_wac.py +""" +import json +import sys +import time +import urllib.request + +BASE = "http://localhost:8000" +TOKEN = "" # Will be fetched +WORKSPACE = "admins" + +def api(method, path, data=None): + url = f"{BASE}/api{path}" + headers = {"Content-Type": "application/json"} + if TOKEN: + headers["Authorization"] = f"Bearer {TOKEN}" + body = json.dumps(data).encode() if data else None + req = urllib.request.Request(url, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(req) as resp: + raw = resp.read().decode() + try: + return json.loads(raw, strict=False) + except: + return raw + except urllib.error.HTTPError as e: + body = e.read().decode() + print(f"HTTP {e.code} {method} {path}: {body[:500]}") + raise + +def login(): + global TOKEN + TOKEN = "PdxixPjjfx05H8xJ8kWAll4RtiLGcfXW" + # Verify token works + user = api("GET", "/users/whoami") + print(f"Logged in as: {user.get('email', 'unknown')}") + +def run_preview(code, language="bun"): + """Run a preview job and return the job ID.""" + result = api("POST", f"/w/{WORKSPACE}/jobs/run/preview", { + "content": code, + "language": language, + "args": {"n": 10}, + }) + print(f"Preview job created: {result}") + return result + +def get_job(job_id): + return api("GET", f"/w/{WORKSPACE}/jobs_u/get/{job_id}") + +def get_result(job_id): + return api("GET", f"/w/{WORKSPACE}/jobs_u/completed/get_result/{job_id}") + +def wait_for_job(job_id, timeout=60, check_interval=2): + """Wait for a job to complete. Returns the job object.""" + start = time.time() + while time.time() - start < timeout: + job = get_job(job_id) + job_type = job.get("type", "") + if job_type == "CompletedJob": + return job + # Print status + suspend = job.get("suspend", 0) + status = "suspended" if suspend and suspend > 0 else "running" + print(f" Job {job_id[:8]}... status={status} suspend={suspend} ({time.time()-start:.0f}s)") + time.sleep(check_interval) + raise TimeoutError(f"Job {job_id} did not complete within {timeout}s") + + +WAC_SCRIPT = ''' +import { task, workflow } from "windmill-client"; + +const double = task(async function double(x: number): Promise { + return x * 2; +}); + +const add_one = task(async function add_one(x: number): Promise { + return x + 1; +}); + +export default workflow(async function main(n: number) { + const doubled = await double(n); + const result = await add_one(doubled); + return { doubled, result }; +}); +''' + +def main(): + print("=== WAC v2 E2E Test ===\n") + + # 1. Login + login() + + # 2. Run the WAC preview + print(f"\nRunning WAC v2 preview...") + job_id = run_preview(WAC_SCRIPT) + + # 3. Wait for completion + print(f"\nWaiting for job {job_id} to complete...") + job = wait_for_job(job_id, timeout=120) + + success = job.get("success", False) + result = job.get("result") + + print(f"\nJob completed! success={success}") + print(f"Result: {json.dumps(result, indent=2)}") + + if not success: + print("\nFAILED: Job did not succeed") + # Print logs if available + logs = job.get("logs", "") + if logs: + print(f"\nLogs:\n{logs}") + sys.exit(1) + + # 4. Verify result + expected = {"doubled": 20, "result": 21} + if result == expected: + print(f"\nSUCCESS: Sequential workflow result matches expected {expected}") + else: + print(f"\nFAILED: Expected {expected}, got {result}") + sys.exit(1) + + # 5. Test parallel workflow + print("\n\n=== Parallel Workflow Test ===\n") + parallel_job_id = run_preview(PARALLEL_WAC_SCRIPT) + print(f"\nWaiting for parallel job {parallel_job_id} to complete...") + parallel_job = wait_for_job(parallel_job_id, timeout=120) + + p_success = parallel_job.get("success", False) + p_result = parallel_job.get("result") + + print(f"\nParallel job completed! success={p_success}") + print(f"Result: {json.dumps(p_result, indent=2)}") + + if not p_success: + print("\nFAILED: Parallel job did not succeed") + sys.exit(1) + + p_expected = {"doubled": 20, "incremented": 11, "combined": 31} + if p_result == p_expected: + print(f"\nSUCCESS: Parallel workflow result matches expected {p_expected}") + else: + print(f"\nFAILED: Expected {p_expected}, got {p_result}") + sys.exit(1) + + print("\n\n=== ALL TESTS PASSED ===") + +PARALLEL_WAC_SCRIPT = ''' +import { task, workflow } from "windmill-client"; + +const double = task(async function double(x: number): Promise { + return x * 2; +}); + +const increment = task(async function increment(x: number): Promise { + return x + 1; +}); + +export default workflow(async function main(n: number) { + const [doubled, incremented] = await Promise.all([ + double(n), + increment(n), + ]); + return { doubled, incremented, combined: doubled + incremented }; +}); +''' + +if __name__ == "__main__": + main() diff --git a/typescript-client/tests/e2e_wac_v1.py b/typescript-client/tests/e2e_wac_v1.py new file mode 100644 index 0000000000..64c8f6a0cc --- /dev/null +++ b/typescript-client/tests/e2e_wac_v1.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +""" +E2E test for WAC v1 (Workflow-as-Code) — HTTP-dispatch mode. + +WAC v1 scripts use @task / task() but NOT @workflow / workflow(). +Tasks dispatch via HTTP POST to /jobs/run/workflow_as_code/{job_id}/{task_name}. + +This verifies that v1 still works after the v2 client changes. + +Usage: + python3 typescript-client/tests/e2e_wac_v1.py +""" +import json +import sys +import time +import urllib.request + +BASE = "http://localhost:8000" +TOKEN = "" +WORKSPACE = "dev" + + +def api(method, path, data=None): + url = f"{BASE}/api{path}" + headers = {"Content-Type": "application/json"} + if TOKEN: + headers["Authorization"] = f"Bearer {TOKEN}" + body = json.dumps(data).encode() if data else None + req = urllib.request.Request(url, data=body, headers=headers, method=method) + try: + with urllib.request.urlopen(req) as resp: + raw = resp.read().decode() + try: + return json.loads(raw, strict=False) + except Exception: + return raw + except urllib.error.HTTPError as e: + body = e.read().decode() + print(f"HTTP {e.code} {method} {path}: {body[:500]}") + raise + + +def login(): + global TOKEN + resp = api("POST", "/auth/login", {"email": "admin@windmill.dev", "password": "changeme"}) + TOKEN = resp + user = api("GET", "/users/whoami") + print(f"Logged in as: {user.get('email', 'unknown')}") + + +def run_preview(code, language="bun", args=None): + result = api("POST", f"/w/{WORKSPACE}/jobs/run/preview", { + "content": code, + "language": language, + "args": args or {}, + }) + print(f" Preview job created: {result}") + return result + + +def get_job(job_id): + return api("GET", f"/w/{WORKSPACE}/jobs_u/get/{job_id}") + + +def wait_for_job(job_id, timeout=120, check_interval=2): + start = time.time() + while time.time() - start < timeout: + job = get_job(job_id) + if job.get("type") == "CompletedJob": + return job + elapsed = time.time() - start + print(f" {job_id[:8]}... waiting ({elapsed:.0f}s)") + time.sleep(check_interval) + raise TimeoutError(f"Job {job_id} did not complete within {timeout}s") + + +def check_result(job, expected, label): + success = job.get("success", False) + result = job.get("result") + print(f" success={success} result={json.dumps(result)}") + if not success: + logs = job.get("logs", "") + print(f" FAILED: job did not succeed\n Logs:\n{logs}") + sys.exit(1) + if result != expected: + print(f" FAILED [{label}]: expected {expected}, got {result}") + sys.exit(1) + print(f" PASSED [{label}]") + + +# --------------------------------------------------------------------------- +# WAC v1 TypeScript — no workflow() wrapper, tasks dispatch via HTTP +# --------------------------------------------------------------------------- +TS_V1_SEQUENTIAL = ''' +import { task } from "windmill-client"; + +export const double = task(async function double(x: number): Promise { + return x * 2; +}); + +export const add_one = task(async function add_one(x: number): Promise { + return x + 1; +}); + +export async function main(n: number) { + const doubled = await double(n); + const result = await add_one(doubled); + return { doubled, result }; +} +''' + +TS_V1_MULTI_PARAM = ''' +import { task } from "windmill-client"; + +export const add = task(async function add(a: number, b: number): Promise { + return a + b; +}); + +export async function main(x: number) { + const result = await add(x, 100); + return { result }; +} +''' + +# --------------------------------------------------------------------------- +# WAC v1 Python — no @workflow, tasks dispatch via HTTP +# --------------------------------------------------------------------------- +PY_V1_SEQUENTIAL = ''' +import wmill + +@wmill.task +def double(x: int) -> int: + return x * 2 + +@wmill.task +def add_one(x: int) -> int: + return x + 1 + +def main(n: int): + doubled = double(x=n) + result = add_one(x=doubled) + return {"doubled": doubled, "result": result} +''' + +PY_V1_MULTI_PARAM = ''' +import wmill + +@wmill.task +def add(a: int, b: int) -> int: + return a + b + +def main(x: int): + result = add(a=x, b=100) + return {"result": result} +''' + + +def main(): + print("=== WAC v1 E2E Tests ===\n") + login() + + # --- TypeScript v1: sequential --- + print("\n[1] TypeScript v1 — sequential tasks") + job_id = run_preview(TS_V1_SEQUENTIAL, "bun", {"n": 10}) + job = wait_for_job(job_id) + check_result(job, {"doubled": 20, "result": 21}, "ts_v1_sequential") + + # --- TypeScript v1: multi-param --- + print("\n[2] TypeScript v1 — multi-param task") + job_id = run_preview(TS_V1_MULTI_PARAM, "bun", {"x": 42}) + job = wait_for_job(job_id) + check_result(job, {"result": 142}, "ts_v1_multi_param") + + # --- Python v1: sequential --- + print("\n[3] Python v1 — sequential tasks") + job_id = run_preview(PY_V1_SEQUENTIAL, "python3", {"n": 10}) + job = wait_for_job(job_id) + check_result(job, {"doubled": 20, "result": 21}, "py_v1_sequential") + + # --- Python v1: multi-param --- + print("\n[4] Python v1 — multi-param task") + job_id = run_preview(PY_V1_MULTI_PARAM, "python3", {"x": 42}) + job = wait_for_job(job_id) + check_result(job, {"result": 142}, "py_v1_multi_param") + + print("\n\n=== ALL WAC v1 TESTS PASSED ===") + + +if __name__ == "__main__": + main() diff --git a/typescript-client/tests/workflow.test.ts b/typescript-client/tests/workflow.test.ts new file mode 100644 index 0000000000..a42895ccbb --- /dev/null +++ b/typescript-client/tests/workflow.test.ts @@ -0,0 +1,1596 @@ +/** + * Standalone tests for the Workflow-as-Code TypeScript SDK. + * + * Run with: bun test typescript-client/tests/workflow.test.ts + */ +import { expect, test, describe } from "bun:test"; + +// --- Inline SDK (mirrors client.ts implementation) --- + +class StepSuspend extends Error { + constructor(public dispatchInfo: Record) { + super("__step_suspend__"); + this.name = "StepSuspend"; + } +} + +let _workflowCtx: WorkflowCtx | null = null; + +class WorkflowCtx { + private completed: Record; + private stepIndex = 0; + private pending: Array<{ + name: string; + script: string; + args: Record; + key: string; + }> = []; + private _suspended = false; + _executingKey: string | null; + + constructor(checkpoint: Record = {}) { + this.completed = checkpoint?.completed_steps ?? {}; + this._executingKey = checkpoint?._executing_key ?? null; + } + + _allocKey(): string { + return `step_${this.stepIndex++}`; + } + + _nextStep( + name: string, + script: string, + args: Record = {}, + options?: Record, + ): PromiseLike { + const key = this._allocKey(); + + if (key in this.completed) { + const value = this.completed[key]; + if (value && typeof value === "object" && (value as any).__wmill_error) { + const err = new Error((value as any).message || `Task '${name}' failed`); + (err as any).result = (value as any).result; + (err as any).step_key = (value as any).step_key; + (err as any).child_job_id = (value as any).child_job_id; + return { then: (_resolve: any, reject?: any) => { if (reject) reject(err); else throw err; } }; + } + return { then: (resolve: any) => resolve(value) }; + } + + // Child job mode: execute matching step directly + if (this._executingKey === key) { + return { + then: (resolve: any) => resolve(null), + _execute_directly: true, + } as any; + } + + // Child job mode: non-matching steps never resolve + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } + + const stepInfo: any = { name, script, args, key }; + if (options) Object.assign(stepInfo, options); + this.pending.push(stepInfo); + return { + then: (): never => { + if (this._suspended) return new Promise(() => {}) as never; + this._suspended = true; + const steps = [...this.pending]; + this.pending = []; + throw new StepSuspend({ + mode: steps.length > 1 ? "parallel" : "sequential", + steps, + }); + }, + }; + } + + _flushPending(): Array<{ + name: string; + script: string; + args: Record; + key: string; + }> { + const steps = [...this.pending]; + this.pending = []; + return steps; + } + + _sleep(seconds: number): PromiseLike { + const key = this._allocKey(); + if (key in this.completed) { + return { then: (resolve: any) => resolve(undefined) }; + } + if (this._executingKey !== null) { + return { then: () => new Promise(() => {}) }; + } + throw new StepSuspend({ + mode: "sleep", + key, + seconds: Math.max(1, Math.round(seconds)), + steps: [], + }); + } + + async _runInlineStep( + name: string, + fn: () => T | Promise + ): Promise { + const key = this._allocKey(); + + if (key in this.completed) { + const value = this.completed[key]; + if (value && typeof value === "object" && (value as any).__wmill_error) { + const err = new Error((value as any).message || `Step '${name}' failed`); + (err as any).result = (value as any).result; + throw err; + } + return value as T; + } + + if (this._executingKey !== null) { + return new Promise(() => {}); + } + + const result = await fn(); + throw new StepSuspend({ + mode: "inline_checkpoint", + steps: [], + key, + result, + }); + } +} + +function getParamNames(fn: Function): string[] { + const src = fn.toString(); + const match = src.match(/^(?:async\s+)?(?:function\s*\w*)?\s*\(([^)]*)\)/); + if (!match) return []; + return match[1] + .split(",") + .map((p) => p.trim().replace(/[:=].*/s, "").trim()) + .filter(Boolean); +} + +function task Promise>( + fnOrPath: T | string, + maybeFnOrOptions?: T | Record, + maybeOptions?: Record, +): T { + let fn: T; + let taskPath: string | undefined; + let taskOptions: Record | undefined; + + if (typeof fnOrPath === "string") { + taskPath = fnOrPath; + fn = maybeFnOrOptions as T; + taskOptions = maybeOptions; + } else { + fn = fnOrPath; + taskOptions = maybeFnOrOptions as Record | undefined; + } + + const taskName = fn.name || "anonymous"; + + // Non-async wrapper — returns thenable directly in workflow context so + // unawaited calls leave steps in pending for _flushPending. + const wrapper = function (...args: any[]) { + const ctx = _workflowCtx; + if (ctx) { + const script = taskPath ?? taskName; + const paramNames = getParamNames(fn); + const kwargs: Record = {}; + for (let i = 0; i < args.length; i++) { + if (paramNames[i]) { + kwargs[paramNames[i]] = args[i]; + } else { + kwargs[`arg${i}`] = args[i]; + } + } + const stepResult = ctx._nextStep(taskName, script, kwargs, taskOptions); + if ((stepResult as any)?._execute_directly) { + return (async () => { + const result = await fn(...args); + throw new StepSuspend({ + mode: "step_complete", + steps: [], + result, + }); + })(); + } + return stepResult; + } else { + return fn(...args); + } + } as unknown as T; + + Object.defineProperty(wrapper, "name", { value: taskName }); + (wrapper as any)._is_task = true; + (wrapper as any)._task_path = taskPath; + return wrapper; +} + +async function step( + name: string, + fn: () => T | Promise +): Promise { + const ctx = _workflowCtx; + if (ctx) { + return ctx._runInlineStep(name, fn); + } + return fn(); +} + +async function sleep(seconds: number): Promise { + const ctx = _workflowCtx; + if (ctx) { + return ctx._sleep(seconds) as Promise; + } + await new Promise((r) => setTimeout(r, seconds * 1000)); +} + +async function parallel( + items: T[], + fn: (item: T) => PromiseLike | R, + options?: { concurrency?: number }, +): Promise { + const concurrency = options?.concurrency ?? items.length; + if (concurrency <= 0 || items.length === 0) return []; + const results: R[] = []; + for (let i = 0; i < items.length; i += concurrency) { + const batch = items.slice(i, i + concurrency); + const batchResults = await Promise.all(batch.map((item) => fn(item))); + results.push(...batchResults); + } + return results; +} + +function workflow(fn: (...args: any[]) => Promise) { + (fn as any)._is_workflow = true; + return fn; +} + +// --- Helper to run a workflow with a checkpoint --- + +async function runWorkflow( + fn: Function, + checkpoint: Record, + args: any[] +): Promise { + const ctx = new WorkflowCtx(checkpoint); + _workflowCtx = ctx; + try { + const result = await fn(...args); + // Flush unawaited tasks + const pending = ctx._flushPending(); + if (pending.length > 0) { + return { + type: "dispatch", + mode: pending.length > 1 ? "parallel" : "sequential", + steps: pending, + }; + } + return { type: "complete", result }; + } catch (e: any) { + if (e instanceof StepSuspend) { + const info = e.dispatchInfo; + if (info.mode === "step_complete") { + return { type: "complete", result: info.result }; + } + if (info.mode === "inline_checkpoint") { + return { + type: "inline_checkpoint", + key: info.key, + result: info.result, + }; + } + if (info.mode === "approval") { + return { type: "approval", key: info.key, timeout: info.timeout, form: info.form }; + } + if (info.mode === "sleep") { + return { type: "sleep", key: info.key, seconds: info.seconds }; + } + return { type: "dispatch", ...info }; + } + throw e; + } finally { + _workflowCtx = null; + } +} + +// --- Define tasks --- + +const extract_data = task(async function extract_data(url: string) {}); +const load_data = task(async function load_data(data?: any) {}); +const clean_data = task(async function clean_data(data?: any) {}); +const compute_stats = task(async function compute_stats(data?: any) {}); +const send_alert = task(async function send_alert(msg: string) {}); +const double = task(async function double(x: number) { + return x * 2; +}); +const add_one = task(async function add_one(x: number) { + return x + 1; +}); +const noop_task = task(async function noop_task() {}); + +// --- Define workflows --- + +const simple_workflow = workflow(async (url: string) => { + const raw = await extract_data(url); + const result = await load_data(raw); + return { status: "done", result }; +}); + +const parallel_workflow = workflow(async (url: string) => { + const raw = await extract_data(url); + const [cleaned, stats] = await Promise.all([ + clean_data(raw), + compute_stats(raw), + ]); + return { cleaned, stats }; +}); + +const conditional_workflow = workflow(async (count: number) => { + if (count > 100) { + await send_alert("large"); + } + await load_data(); + return { done: true }; +}); + +// ===================================================================== +// TESTS +// ===================================================================== + +describe("task decorator", () => { + test("marks function as task", () => { + expect((extract_data as any)._is_task).toBe(true); + }); + + test("standalone execution runs body directly", async () => { + const result = await extract_data("https://example.com"); + expect(result).toBeUndefined(); + }); + + test("preserves function name", () => { + expect(extract_data.name).toBe("extract_data"); + expect(double.name).toBe("double"); + }); +}); + +describe("workflow decorator", () => { + test("marks function as workflow", () => { + expect((simple_workflow as any)._is_workflow).toBe(true); + }); +}); + +describe("first invocation", () => { + test("dispatches first step", async () => { + const result = await runWorkflow(simple_workflow, {}, [ + "https://example.com", + ]); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("sequential"); + expect(result.steps).toHaveLength(1); + expect(result.steps[0].name).toBe("extract_data"); + expect(result.steps[0].script).toBe("extract_data"); + expect(result.steps[0].key).toBe("step_0"); + expect(result.steps[0].args).toEqual({ url: "https://example.com" }); + }); +}); + +describe("replay with checkpoint", () => { + test("second invocation dispatches second step", async () => { + const checkpoint = { + completed_steps: { step_0: [1, 2, 3] }, + }; + const result = await runWorkflow(simple_workflow, checkpoint, [ + "https://example.com", + ]); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("sequential"); + expect(result.steps[0].name).toBe("load_data"); + expect(result.steps[0].key).toBe("step_1"); + }); + + test("all steps complete returns result", async () => { + const checkpoint = { + completed_steps: { + step_0: [1, 2, 3], + step_1: { loaded: true }, + }, + }; + const result = await runWorkflow(simple_workflow, checkpoint, [ + "https://example.com", + ]); + expect(result.type).toBe("complete"); + expect(result.result.status).toBe("done"); + expect(result.result.result).toEqual({ loaded: true }); + }); +}); + +describe("parallel dispatch", () => { + test("first invocation dispatches extract", async () => { + const result = await runWorkflow(parallel_workflow, {}, [ + "https://example.com", + ]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("extract_data"); + }); + + test("dispatches parallel steps after extract completes", async () => { + const checkpoint = { + completed_steps: { step_0: { raw: "data" } }, + }; + const result = await runWorkflow(parallel_workflow, checkpoint, [ + "https://example.com", + ]); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].name).toBe("clean_data"); + expect(result.steps[1].name).toBe("compute_stats"); + }); + + test("completes when all parallel steps done", async () => { + const checkpoint = { + completed_steps: { + step_0: { raw: "data" }, + step_1: { cleaned: true }, + step_2: { count: 42 }, + }, + }; + const result = await runWorkflow(parallel_workflow, checkpoint, [ + "https://example.com", + ]); + expect(result.type).toBe("complete"); + expect(result.result.cleaned).toEqual({ cleaned: true }); + expect(result.result.stats).toEqual({ count: 42 }); + }); +}); + +describe("conditional workflow", () => { + test("condition true dispatches send_alert", async () => { + const result = await runWorkflow(conditional_workflow, {}, [200]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("send_alert"); + }); + + test("condition false skips to load_data", async () => { + const result = await runWorkflow(conditional_workflow, {}, [50]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("load_data"); + }); +}); + +describe("task with external path", () => { + const run_external = task( + "f/external_script", + async function run_external(x: number) {} + ); + + test("uses external path as script", async () => { + const wf = workflow(async (x: number) => { + const result = await run_external(x); + return result; + }); + const result = await runWorkflow(wf, {}, [42]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("run_external"); + expect(result.steps[0].script).toBe("f/external_script"); + expect(result.steps[0].args).toEqual({ x: 42 }); + }); +}); + +// ===================================================================== +// EDGE CASE TESTS +// ===================================================================== + +describe("full sequential lifecycle (3 steps)", () => { + const three_step_wf = workflow(async (n: number) => { + const doubled = await double(n); + const incremented = await add_one(doubled); + const final_val = await double(incremented); + return { doubled, incremented, final: final_val }; + }); + + test("replay 0: dispatches step_0", async () => { + const result = await runWorkflow(three_step_wf, {}, [5]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].key).toBe("step_0"); + expect(result.steps[0].name).toBe("double"); + expect(result.steps[0].args).toEqual({ x: 5 }); + }); + + test("replay 1: dispatches step_1 with step_0 result as arg", async () => { + const result = await runWorkflow( + three_step_wf, + { completed_steps: { step_0: 10 } }, + [5] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].key).toBe("step_1"); + expect(result.steps[0].name).toBe("add_one"); + expect(result.steps[0].args).toEqual({ x: 10 }); + }); + + test("replay 2: dispatches step_2 with step_1 result as arg", async () => { + const result = await runWorkflow( + three_step_wf, + { completed_steps: { step_0: 10, step_1: 11 } }, + [5] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].key).toBe("step_2"); + expect(result.steps[0].name).toBe("double"); + expect(result.steps[0].args).toEqual({ x: 11 }); + }); + + test("replay 3: all complete, returns final result", async () => { + const result = await runWorkflow( + three_step_wf, + { completed_steps: { step_0: 10, step_1: 11, step_2: 22 } }, + [5] + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ doubled: 10, incremented: 11, final: 22 }); + }); +}); + +describe("step after parallel group", () => { + const seq_par_seq_wf = workflow(async (url: string) => { + const raw = await extract_data(url); + const [cleaned, stats] = await Promise.all([ + clean_data(raw), + compute_stats(raw), + ]); + const loaded = await load_data({ cleaned, stats }); + return loaded; + }); + + test("dispatches first sequential step", async () => { + const result = await runWorkflow(seq_par_seq_wf, {}, ["http://x"]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("extract_data"); + }); + + test("dispatches parallel group", async () => { + const result = await runWorkflow( + seq_par_seq_wf, + { completed_steps: { step_0: "raw" } }, + ["http://x"] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + }); + + test("dispatches final step after parallel completes", async () => { + const result = await runWorkflow( + seq_par_seq_wf, + { + completed_steps: { + step_0: "raw", + step_1: "cleaned", + step_2: { count: 5 }, + }, + }, + ["http://x"] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("sequential"); + expect(result.steps[0].name).toBe("load_data"); + expect(result.steps[0].key).toBe("step_3"); + }); + + test("completes when final step done", async () => { + const result = await runWorkflow( + seq_par_seq_wf, + { + completed_steps: { + step_0: "raw", + step_1: "cleaned", + step_2: { count: 5 }, + step_3: "final", + }, + }, + ["http://x"] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe("final"); + }); +}); + +describe("parallel after parallel (back to back)", () => { + const double_parallel_wf = workflow(async () => { + const [a, b] = await Promise.all([double(1), double(2)]); + const [c, d] = await Promise.all([add_one(a), add_one(b)]); + return { a, b, c, d }; + }); + + test("dispatches first parallel group", async () => { + const result = await runWorkflow(double_parallel_wf, {}, []); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].name).toBe("double"); + expect(result.steps[1].name).toBe("double"); + expect(result.steps[0].key).toBe("step_0"); + expect(result.steps[1].key).toBe("step_1"); + }); + + test("dispatches second parallel group after first completes", async () => { + const result = await runWorkflow( + double_parallel_wf, + { completed_steps: { step_0: 2, step_1: 4 } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].name).toBe("add_one"); + expect(result.steps[1].name).toBe("add_one"); + expect(result.steps[0].args).toEqual({ x: 2 }); + expect(result.steps[1].args).toEqual({ x: 4 }); + }); + + test("completes when all done", async () => { + const result = await runWorkflow( + double_parallel_wf, + { completed_steps: { step_0: 2, step_1: 4, step_2: 3, step_3: 5 } }, + [] + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ a: 2, b: 4, c: 3, d: 5 }); + }); +}); + +describe("conditional based on step result", () => { + const cond_on_result = workflow(async () => { + const val = await double(5); + if (val > 8) { + await send_alert("big"); + } + await load_data(val); + return { val }; + }); + + test("condition true path (val=10 > 8)", async () => { + const result = await runWorkflow( + cond_on_result, + { completed_steps: { step_0: 10 } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("send_alert"); + expect(result.steps[0].key).toBe("step_1"); + }); + + test("condition false path (val=4 <= 8)", async () => { + const result = await runWorkflow( + cond_on_result, + { completed_steps: { step_0: 4 } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("load_data"); + // When condition is false, send_alert is skipped so step index for + // load_data is step_1 (not step_2) + expect(result.steps[0].key).toBe("step_1"); + }); + + test("condition true: step after alert has key step_2", async () => { + const result = await runWorkflow( + cond_on_result, + { completed_steps: { step_0: 10, step_1: "alerted" } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("load_data"); + expect(result.steps[0].key).toBe("step_2"); + }); +}); + +describe("empty workflow (no tasks)", () => { + const empty_wf = workflow(async () => { + return { status: "empty" }; + }); + + test("completes immediately with no dispatch", async () => { + const result = await runWorkflow(empty_wf, {}, []); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ status: "empty" }); + }); +}); + +describe("single task workflow", () => { + const single_wf = workflow(async (x: number) => { + const result = await double(x); + return result; + }); + + test("dispatches single step", async () => { + const result = await runWorkflow(single_wf, {}, [7]); + expect(result.type).toBe("dispatch"); + expect(result.steps).toHaveLength(1); + expect(result.steps[0].name).toBe("double"); + }); + + test("completes with single result", async () => { + const result = await runWorkflow( + single_wf, + { completed_steps: { step_0: 14 } }, + [7] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(14); + }); +}); + +describe("task with no arguments", () => { + const no_arg_wf = workflow(async () => { + const result = await noop_task(); + return result; + }); + + test("dispatches with empty args", async () => { + const result = await runWorkflow(no_arg_wf, {}, []); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].args).toEqual({}); + }); +}); + +describe("many steps (10+)", () => { + const many_steps_wf = workflow(async (n: number) => { + let val = n; + for (let i = 0; i < 10; i++) { + val = await add_one(val); + } + return val; + }); + + test("first invocation dispatches step_0", async () => { + const result = await runWorkflow(many_steps_wf, {}, [0]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].key).toBe("step_0"); + }); + + test("with 5 steps complete, dispatches step_5", async () => { + const completed: Record = {}; + for (let i = 0; i < 5; i++) completed[`step_${i}`] = i + 1; + const result = await runWorkflow( + many_steps_wf, + { completed_steps: completed }, + [0] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].key).toBe("step_5"); + expect(result.steps[0].args).toEqual({ x: 5 }); + }); + + test("all 10 steps complete returns final value", async () => { + const completed: Record = {}; + for (let i = 0; i < 10; i++) completed[`step_${i}`] = i + 1; + const result = await runWorkflow( + many_steps_wf, + { completed_steps: completed }, + [0] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(10); + }); +}); + +describe("falsy values preserved in checkpoint", () => { + const falsy_wf = workflow(async () => { + const a = await double(0); // result will be 0 + const b = await load_data(a); // result will be null + const c = await extract_data(""); // result will be "" + return { a, b, c }; + }); + + test("zero is preserved", async () => { + const result = await runWorkflow( + falsy_wf, + { completed_steps: { step_0: 0 } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("load_data"); + expect(result.steps[0].args).toEqual({ data: 0 }); + }); + + test("null is preserved", async () => { + const result = await runWorkflow( + falsy_wf, + { completed_steps: { step_0: 0, step_1: null } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("extract_data"); + }); + + test("all falsy values complete correctly", async () => { + const result = await runWorkflow( + falsy_wf, + { completed_steps: { step_0: 0, step_1: null, step_2: "" } }, + [] + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ a: 0, b: null, c: "" }); + }); + + test("false is preserved", async () => { + const flag_wf = workflow(async () => { + const val = await load_data("check"); + if (val) { + await send_alert("truthy"); + } + return { val }; + }); + // false should be treated as completed (key exists), not as missing + const result = await runWorkflow( + flag_wf, + { completed_steps: { step_0: false } }, + [] + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ val: false }); + }); +}); + +describe("inline step (step function)", () => { + const step_wf = workflow(async (x: number) => { + const ts = await step("timestamp", () => 1234567890); + const doubled = await double(x); + const rid = await step("random_id", () => "abc-123"); + return { ts, doubled, id: rid }; + }); + + test("first invocation returns inline_checkpoint", async () => { + const result = await runWorkflow(step_wf, {}, [7]); + expect(result.type).toBe("inline_checkpoint"); + expect(result.key).toBe("step_0"); + expect(result.result).toBe(1234567890); + }); + + test("step cached, dispatches task", async () => { + const result = await runWorkflow( + step_wf, + { completed_steps: { step_0: 1234567890 } }, + [7] + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("double"); + expect(result.steps[0].key).toBe("step_1"); + }); + + test("step + task cached, returns second inline step", async () => { + const result = await runWorkflow( + step_wf, + { completed_steps: { step_0: 1234567890, step_1: 14 } }, + [7] + ); + expect(result.type).toBe("inline_checkpoint"); + expect(result.key).toBe("step_2"); + expect(result.result).toBe("abc-123"); + }); + + test("all complete returns final result", async () => { + const result = await runWorkflow( + step_wf, + { + completed_steps: { + step_0: 1234567890, + step_1: 14, + step_2: "abc-123", + }, + }, + [7] + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ + ts: 1234567890, + doubled: 14, + id: "abc-123", + }); + }); +}); + +describe("unawaited tasks (flush pending)", () => { + test("single unawaited task at end is flushed", async () => { + const wf = workflow(async () => { + await extract_data("x"); + load_data("y"); // forgotten await + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: "raw" } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("sequential"); + expect(result.steps).toHaveLength(1); + expect(result.steps[0].name).toBe("load_data"); + }); + + test("multiple unawaited tasks flushed as parallel", async () => { + const wf = workflow(async () => { + await extract_data("x"); + clean_data("y"); // forgotten await + compute_stats("y"); // forgotten await + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: "raw" } }, + [] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].name).toBe("clean_data"); + expect(result.steps[1].name).toBe("compute_stats"); + }); + + test("no unawaited tasks means normal complete", async () => { + const wf = workflow(async () => { + const val = await double(5); + return val; + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 10 } }, + [] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(10); + }); +}); + +describe("child mode (_executingKey)", () => { + test("executes matching task directly", async () => { + const wf = workflow(async (x: number) => { + const val = await double(x); + return val; + }); + const result = await runWorkflow( + wf, + { completed_steps: {}, _executing_key: "step_0" }, + [7] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(14); // double(7) = 14 + }); + + test("replays cached steps before executing key", async () => { + const wf = workflow(async (x: number) => { + const doubled = await double(x); + const result = await add_one(doubled); + return result; + }); + const result = await runWorkflow( + wf, + { + completed_steps: { step_0: 10 }, + _executing_key: "step_1", + }, + [5] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(11); // add_one(10) = 11 + }); + + test("child mode with external path task", async () => { + const ext = task( + "f/external", + async function ext_task(x: number) { + return x * 3; + } + ); + const wf = workflow(async (x: number) => { + const result = await ext(x); + return result; + }); + const result = await runWorkflow( + wf, + { completed_steps: {}, _executing_key: "step_0" }, + [4] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe(12); // 4 * 3 + }); +}); + +describe("key determinism across replays", () => { + const det_wf = workflow(async (n: number) => { + const a = await double(n); + const b = await add_one(a); + const c = await double(b); + return c; + }); + + test("keys are consistent: step_0 always maps to first double", async () => { + // Empty checkpoint + const r1 = await runWorkflow(det_wf, {}, [3]); + expect(r1.steps[0].key).toBe("step_0"); + expect(r1.steps[0].name).toBe("double"); + + // With step_0 completed + const r2 = await runWorkflow( + det_wf, + { completed_steps: { step_0: 6 } }, + [3] + ); + expect(r2.steps[0].key).toBe("step_1"); + expect(r2.steps[0].name).toBe("add_one"); + + // With step_0 and step_1 completed + const r3 = await runWorkflow( + det_wf, + { completed_steps: { step_0: 6, step_1: 7 } }, + [3] + ); + expect(r3.steps[0].key).toBe("step_2"); + expect(r3.steps[0].name).toBe("double"); + }); +}); + +describe("parallel dispatch includes correct args from cached results", () => { + test("parallel steps receive cached parent result as args", async () => { + const wf = workflow(async (x: number) => { + const base = await double(x); + const [a, b] = await Promise.all([add_one(base), double(base)]); + return { a, b }; + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 20 } }, + [10] + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps[0].args).toEqual({ x: 20 }); + expect(result.steps[1].args).toEqual({ x: 20 }); + }); +}); + +describe("inline step with async function", () => { + test("async step function resolves correctly", async () => { + const wf = workflow(async () => { + const val = await step("async_step", async () => { + return 42; + }); + return val; + }); + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("inline_checkpoint"); + expect(result.key).toBe("step_0"); + expect(result.result).toBe(42); + }); +}); + +describe("workflow returning undefined", () => { + test("undefined return value is captured", async () => { + const wf = workflow(async () => { + await double(1); + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2 } }, + [] + ); + expect(result.type).toBe("complete"); + expect(result.result).toBeUndefined(); + }); +}); + +describe("large parallel group", () => { + test("dispatches 5 parallel steps at once", async () => { + const wf = workflow(async () => { + const results = await Promise.all([ + double(1), + double(2), + double(3), + double(4), + double(5), + ]); + return results; + }); + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(5); + for (let i = 0; i < 5; i++) { + expect(result.steps[i].key).toBe(`step_${i}`); + expect(result.steps[i].args).toEqual({ x: i + 1 }); + } + }); +}); + +describe("complex mixed workflow: seq → par → seq → par → seq", () => { + const complex_wf = workflow(async () => { + const init = await extract_data("start"); + const [a, b] = await Promise.all([double(1), double(2)]); + const mid = await load_data({ a, b }); + const [c, d] = await Promise.all([add_one(3), add_one(4)]); + const fin = await clean_data({ mid, c, d }); + return fin; + }); + + test("replay 0: dispatches extract_data", async () => { + const r = await runWorkflow(complex_wf, {}, []); + expect(r.steps[0].name).toBe("extract_data"); + }); + + test("replay 1: dispatches parallel [double, double]", async () => { + const r = await runWorkflow( + complex_wf, + { completed_steps: { step_0: "init" } }, + [] + ); + expect(r.mode).toBe("parallel"); + expect(r.steps).toHaveLength(2); + expect(r.steps[0].name).toBe("double"); + }); + + test("replay 2: dispatches load_data", async () => { + const r = await runWorkflow( + complex_wf, + { completed_steps: { step_0: "init", step_1: 2, step_2: 4 } }, + [] + ); + expect(r.mode).toBe("sequential"); + expect(r.steps[0].name).toBe("load_data"); + expect(r.steps[0].key).toBe("step_3"); + }); + + test("replay 3: dispatches parallel [add_one, add_one]", async () => { + const r = await runWorkflow( + complex_wf, + { + completed_steps: { + step_0: "init", + step_1: 2, + step_2: 4, + step_3: "mid", + }, + }, + [] + ); + expect(r.mode).toBe("parallel"); + expect(r.steps).toHaveLength(2); + expect(r.steps[0].name).toBe("add_one"); + }); + + test("replay 4: dispatches clean_data", async () => { + const r = await runWorkflow( + complex_wf, + { + completed_steps: { + step_0: "init", + step_1: 2, + step_2: 4, + step_3: "mid", + step_4: 4, + step_5: 5, + }, + }, + [] + ); + expect(r.mode).toBe("sequential"); + expect(r.steps[0].name).toBe("clean_data"); + expect(r.steps[0].key).toBe("step_6"); + }); + + test("replay 5: all complete", async () => { + const r = await runWorkflow( + complex_wf, + { + completed_steps: { + step_0: "init", + step_1: 2, + step_2: 4, + step_3: "mid", + step_4: 4, + step_5: 5, + step_6: "final", + }, + }, + [] + ); + expect(r.type).toBe("complete"); + expect(r.result).toBe("final"); + }); +}); + +// ===================================================================== +// ERROR PROPAGATION TESTS +// ===================================================================== + +describe("error propagation via __wmill_error marker", () => { + test("task error is thrown on replay", async () => { + const wf = workflow(async (x: number) => { + const result = await double(x); + return result; + }); + // Simulate child failure stored as __wmill_error marker + const checkpoint = { + completed_steps: { + step_0: { + __wmill_error: true, + message: "WAC task 'double' failed (child job abc-123)", + result: { message: "division by zero" }, + step_key: "double", + child_job_id: "abc-123", + }, + }, + }; + try { + await runWorkflow(wf, checkpoint, [5]); + expect(true).toBe(false); // should not reach here + } catch (e: any) { + expect(e.message).toContain("double"); + expect(e.result).toEqual({ message: "division by zero" }); + expect(e.child_job_id).toBe("abc-123"); + } + }); + + test("error is catchable with try/catch in workflow", async () => { + const wf = workflow(async (x: number) => { + try { + const result = await double(x); + return { success: true, result }; + } catch (e: any) { + return { success: false, error: e.message }; + } + }); + const checkpoint = { + completed_steps: { + step_0: { + __wmill_error: true, + message: "Task 'double' failed", + result: { message: "boom" }, + }, + }, + }; + const result = await runWorkflow(wf, checkpoint, [5]); + expect(result.type).toBe("complete"); + expect(result.result.success).toBe(false); + expect(result.result.error).toContain("double"); + }); + + test("error in parallel — one fails, caught by Promise.all reject", async () => { + const wf = workflow(async () => { + try { + const [a, b] = await Promise.all([double(1), add_one(2)]); + return { a, b }; + } catch (e: any) { + return { caught: true, error: e.message }; + } + }); + const checkpoint = { + completed_steps: { + step_0: { __wmill_error: true, message: "double failed", result: {} }, + step_1: 3, // add_one succeeded + }, + }; + const result = await runWorkflow(wf, checkpoint, []); + expect(result.type).toBe("complete"); + expect(result.result.caught).toBe(true); + }); + + test("retry pattern with try/catch + loop", async () => { + // Simulates: first attempt fails, second succeeds + const wf = workflow(async (x: number) => { + for (let i = 0; i < 3; i++) { + try { + const result = await double(x); + return { result, attempts: i + 1 }; + } catch (e) { + if (i === 2) throw e; + // retry on next iteration + } + } + }); + // First double (step_0) fails, second double (step_1) succeeds + const checkpoint = { + completed_steps: { + step_0: { __wmill_error: true, message: "temporary failure", result: {} }, + step_1: 10, + }, + }; + const result = await runWorkflow(wf, checkpoint, [5]); + expect(result.type).toBe("complete"); + expect(result.result.result).toBe(10); + expect(result.result.attempts).toBe(2); + }); + + test("inline step error is thrown", async () => { + const wf = workflow(async () => { + try { + const val = await step("risky", () => 42); + return { val }; + } catch (e: any) { + return { caught: e.message }; + } + }); + const checkpoint = { + completed_steps: { + step_0: { __wmill_error: true, message: "inline step failed", result: {} }, + }, + }; + const result = await runWorkflow(wf, checkpoint, []); + expect(result.type).toBe("complete"); + expect(result.result.caught).toContain("inline step failed"); + }); + + test("non-error object with __wmill_error field is NOT treated as error", async () => { + // An object with __wmill_error: false should be treated as a normal value + const wf = workflow(async () => { + const val = await double(5); + return val; + }); + const checkpoint = { + completed_steps: { + step_0: { __wmill_error: false, data: "not an error" }, + }, + }; + const result = await runWorkflow(wf, checkpoint, [5]); + expect(result.type).toBe("complete"); + expect(result.result).toEqual({ __wmill_error: false, data: "not an error" }); + }); +}); + +// ===================================================================== +// TASK OPTIONS TESTS +// ===================================================================== + +describe("task options", () => { + test("options are forwarded in dispatch step info", async () => { + const heavy = task( + async function heavy(x: number) { return x; }, + { timeout: 600, tag: "gpu", cache_ttl: 3600, priority: 10 }, + ); + const wf = workflow(async (x: number) => { + return await heavy(x); + }); + const result = await runWorkflow(wf, {}, [42]); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].timeout).toBe(600); + expect(result.steps[0].tag).toBe("gpu"); + expect(result.steps[0].cache_ttl).toBe(3600); + expect(result.steps[0].priority).toBe(10); + }); + + test("task without options has no extra fields", async () => { + const simple = task(async function simple(x: number) { return x; }); + const wf = workflow(async (x: number) => { + return await simple(x); + }); + const result = await runWorkflow(wf, {}, [1]); + expect(result.steps[0].timeout).toBeUndefined(); + expect(result.steps[0].tag).toBeUndefined(); + }); + + test("concurrency options forwarded", async () => { + const limited = task( + async function limited(x: number) { return x; }, + { concurrent_limit: 5, concurrency_key: "my-key", concurrency_time_window_s: 60 }, + ); + const wf = workflow(async (x: number) => { + return await limited(x); + }); + const result = await runWorkflow(wf, {}, [1]); + expect(result.steps[0].concurrent_limit).toBe(5); + expect(result.steps[0].concurrency_key).toBe("my-key"); + expect(result.steps[0].concurrency_time_window_s).toBe(60); + }); + + test("task with path and options", async () => { + const ext = task( + "f/gpu_script", + async function ext(x: number) { return x; }, + { timeout: 300, tag: "gpu" }, + ); + const wf = workflow(async (x: number) => { + return await ext(x); + }); + const result = await runWorkflow(wf, {}, [1]); + expect(result.steps[0].script).toBe("f/gpu_script"); + expect(result.steps[0].timeout).toBe(300); + expect(result.steps[0].tag).toBe("gpu"); + }); +}); + +// ===================================================================== +// SLEEP TESTS +// ===================================================================== + +describe("sleep", () => { + test("first invocation returns sleep output", async () => { + const wf = workflow(async () => { + await double(1); + await sleep(60); + await add_one(2); + return "done"; + }); + // step_0 (double) complete, step_1 is sleep + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2 } }, + [], + ); + expect(result.type).toBe("sleep"); + expect(result.key).toBe("step_1"); + expect(result.seconds).toBe(60); + }); + + test("sleep completes on replay when stored in checkpoint", async () => { + const wf = workflow(async () => { + await double(1); + await sleep(60); + await add_one(2); + return "done"; + }); + // step_0 (double) and step_1 (sleep) complete + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: true } }, + [], + ); + expect(result.type).toBe("dispatch"); + expect(result.steps[0].name).toBe("add_one"); + expect(result.steps[0].key).toBe("step_2"); + }); + + test("all steps including sleep complete returns result", async () => { + const wf = workflow(async () => { + await double(1); + await sleep(60); + await add_one(2); + return "done"; + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: true, step_2: 3 } }, + [], + ); + expect(result.type).toBe("complete"); + expect(result.result).toBe("done"); + }); + + test("sleep enforces minimum of 1 second", async () => { + const wf = workflow(async () => { + await sleep(0); + return "done"; + }); + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("sleep"); + expect(result.seconds).toBe(1); + }); + + test("sleep rounds to nearest integer", async () => { + const wf = workflow(async () => { + await sleep(3.7); + return "done"; + }); + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("sleep"); + expect(result.seconds).toBe(4); + }); +}); + +// ===================================================================== +// PARALLEL UTILITY TESTS +// ===================================================================== + +describe("parallel utility", () => { + test("processes all items with default concurrency", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3]; + const results = await parallel(items, double); + return results; + }); + // All 3 items dispatched in parallel: step_0, step_1, step_2 + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(3); + expect(result.steps[0].args).toEqual({ x: 1 }); + expect(result.steps[1].args).toEqual({ x: 2 }); + expect(result.steps[2].args).toEqual({ x: 3 }); + }); + + test("completes when all parallel items done", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3]; + const results = await parallel(items, double); + return results; + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: 4, step_2: 6 } }, + [], + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual([2, 4, 6]); + }); + + test("batched concurrency dispatches first batch", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3, 4, 5]; + const results = await parallel(items, double, { concurrency: 2 }); + return results; + }); + // First batch: items[0..2] → step_0, step_1 + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].args).toEqual({ x: 1 }); + expect(result.steps[1].args).toEqual({ x: 2 }); + }); + + test("batched concurrency dispatches second batch after first completes", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3, 4, 5]; + const results = await parallel(items, double, { concurrency: 2 }); + return results; + }); + // First batch done, second batch: items[2..4] → step_2, step_3 + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: 4 } }, + [], + ); + expect(result.type).toBe("dispatch"); + expect(result.mode).toBe("parallel"); + expect(result.steps).toHaveLength(2); + expect(result.steps[0].args).toEqual({ x: 3 }); + expect(result.steps[1].args).toEqual({ x: 4 }); + }); + + test("batched concurrency last batch may be smaller", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3, 4, 5]; + const results = await parallel(items, double, { concurrency: 2 }); + return results; + }); + // Two batches done, third batch: items[4..5] → step_4 + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: 4, step_2: 6, step_3: 8 } }, + [], + ); + expect(result.type).toBe("dispatch"); + expect(result.steps).toHaveLength(1); + expect(result.steps[0].args).toEqual({ x: 5 }); + }); + + test("batched concurrency completes with all results in order", async () => { + const wf = workflow(async () => { + const items = [1, 2, 3, 4, 5]; + const results = await parallel(items, double, { concurrency: 2 }); + return results; + }); + const result = await runWorkflow( + wf, + { completed_steps: { step_0: 2, step_1: 4, step_2: 6, step_3: 8, step_4: 10 } }, + [], + ); + expect(result.type).toBe("complete"); + expect(result.result).toEqual([2, 4, 6, 8, 10]); + }); + + test("empty items returns empty array", async () => { + const wf = workflow(async () => { + const results = await parallel([], double); + return results; + }); + const result = await runWorkflow(wf, {}, []); + expect(result.type).toBe("complete"); + expect(result.result).toEqual([]); + }); +}); From 61a1dfc1a872e86a73cc161f2daa7d23b743d617 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 9 Mar 2026 20:07:42 +0000 Subject: [PATCH 46/57] chore(main): release 1.652.0 (#8247) * chore(main): release 1.652.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 25 +++ backend/Cargo.lock | 180 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 130 insertions(+), 105 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fc48489883..26e8dbf644 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09) + + +### Features + +* add secretKeyRef support for package registry and storage credentials ([#8275](https://github.com/windmill-labs/windmill/issues/8275)) ([73d27e9](https://github.com/windmill-labs/windmill/commit/73d27e92dd6ced1602f6328f245fec0fa96860e1)) +* expose OTEL trace context as env vars in job execution ([#8277](https://github.com/windmill-labs/windmill/issues/8277)) ([93f75ad](https://github.com/windmill-labs/windmill/commit/93f75ada5e49036f0d998e3d3d53de4dc2c2e83f)) +* workflow-as-code (WAC) v2 ([#8172](https://github.com/windmill-labs/windmill/issues/8172)) ([a6d4390](https://github.com/windmill-labs/windmill/commit/a6d4390790d21d535df1e9d525bffd577c50d8dc)) + + +### Bug Fixes + +* cli: support deleting linked resources-variables without throwing ([#8248](https://github.com/windmill-labs/windmill/issues/8248)) ([7859bca](https://github.com/windmill-labs/windmill/commit/7859bca6ae80d32a73a46910960afc6812e64115)) +* Database studio fixes ([#8251](https://github.com/windmill-labs/windmill/issues/8251)) ([1d78589](https://github.com/windmill-labs/windmill/commit/1d785899404e8636a206cda9a2914df32a1a5269)) +* **frontend:** unsaved changes dialog when flow already saved ([#8259](https://github.com/windmill-labs/windmill/issues/8259)) ([0330993](https://github.com/windmill-labs/windmill/commit/0330993cb66cdabffcd6e552a0f85a9a3931c62d)) +* gracefully handle uninitialized OTEL tracing proxy port ([#8274](https://github.com/windmill-labs/windmill/issues/8274)) ([8b1fe8f](https://github.com/windmill-labs/windmill/commit/8b1fe8f9de7b0c03655558d0c46cfff71a4b2047)) +* guard iteration picker VirtualList against empty items array ([#8273](https://github.com/windmill-labs/windmill/issues/8273)) ([c97cf60](https://github.com/windmill-labs/windmill/commit/c97cf604ab4a902d89fe873b90dbeb9dabc940eb)), closes [#8272](https://github.com/windmill-labs/windmill/issues/8272) +* mask secrets in OAuth config debug/log output ([#8269](https://github.com/windmill-labs/windmill/issues/8269)) ([e75763d](https://github.com/windmill-labs/windmill/commit/e75763dbe5ffe08e6cde082203596d510c2c3b29)) +* parallel branchall hang on bad stop_after_all_iters_if + results.x.length null ([#8276](https://github.com/windmill-labs/windmill/issues/8276)) ([41e523f](https://github.com/windmill-labs/windmill/commit/41e523f827c4e3d5db525a1f14e24936b0b8af46)) +* redact secrets in set_global_setting log line ([#8270](https://github.com/windmill-labs/windmill/issues/8270)) ([6a0473c](https://github.com/windmill-labs/windmill/commit/6a0473c5783dc0fef2ae82dc5345a5f0596f124d)) +* remove $bindable() fallback values causing props_invalid_value error in oauth settings ([#8265](https://github.com/windmill-labs/windmill/issues/8265)) ([037035e](https://github.com/windmill-labs/windmill/commit/037035e094937827305dad29bd76a495d78bc46f)) +* skip down migrations in potentially_stale checksum comparison ([#8271](https://github.com/windmill-labs/windmill/issues/8271)) ([5ba4029](https://github.com/windmill-labs/windmill/commit/5ba4029d8692b2e6054fca7f45ed4cfded4738ef)) +* sql input horizontal scroll missing after switching flow steps ([#8249](https://github.com/windmill-labs/windmill/issues/8249)) ([ce8ac9c](https://github.com/windmill-labs/windmill/commit/ce8ac9cf52dc17061673b9b72556279c48c26f8e)) +* wmill workspace whoami output ([#8246](https://github.com/windmill-labs/windmill/issues/8246)) ([1ac391a](https://github.com/windmill-labs/windmill/commit/1ac391a795585747fe5911ac41b157556569fedb)) + ## [1.651.1](https://github.com/windmill-labs/windmill/compare/v1.651.0...v1.651.1) (2026-03-05) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1ab271d577..10589a44fc 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7103,7 +7103,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -7974,9 +7974,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" [[package]] name = "libffi" @@ -10664,7 +10664,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.35", - "socket2 0.6.2", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -10673,9 +10673,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", @@ -10702,7 +10702,7 @@ dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -12677,12 +12677,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -14462,7 +14462,7 @@ dependencies = [ "indexmap 2.11.1", "toml_datetime 0.7.0", "toml_parser", - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -14471,7 +14471,7 @@ version = "1.0.9+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4" dependencies = [ - "winnow 0.7.14", + "winnow 0.7.15", ] [[package]] @@ -15741,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-nats", @@ -15808,7 +15808,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15821,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "argon2", @@ -15961,7 +15961,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15984,7 +15984,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15997,7 +15997,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16023,7 +16023,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.651.1" +version = "1.652.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16033,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16050,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16073,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16096,7 +16096,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16112,7 +16112,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16132,7 +16132,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16152,7 +16152,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16166,7 +16166,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-nats", @@ -16193,7 +16193,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16218,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16236,7 +16236,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16257,7 +16257,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16277,7 +16277,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16307,7 +16307,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16334,7 +16334,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.651.1" +version = "1.652.0" dependencies = [ "lazy_static", "serde", @@ -16346,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.651.1" +version = "1.652.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16369,7 +16369,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16383,7 +16383,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.651.1" +version = "1.652.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16414,7 +16414,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.651.1" +version = "1.652.0" dependencies = [ "chrono", "lazy_static", @@ -16428,7 +16428,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16447,7 +16447,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.651.1" +version = "1.652.0" dependencies = [ "aes-gcm", "anyhow", @@ -16546,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.651.1" +version = "1.652.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16565,7 +16565,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.651.1" +version = "1.652.0" dependencies = [ "regex", "serde", @@ -16580,7 +16580,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16604,7 +16604,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "futures", @@ -16621,7 +16621,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.651.1" +version = "1.652.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16637,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -16658,7 +16658,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -16689,7 +16689,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-oauth2", @@ -16713,7 +16713,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-stream", @@ -16747,7 +16747,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "futures", @@ -16765,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.651.1" +version = "1.652.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16774,7 +16774,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "lazy_static", @@ -16786,7 +16786,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "serde_json", @@ -16798,7 +16798,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "gosyn", @@ -16810,7 +16810,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "lazy_static", @@ -16822,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "serde_json", @@ -16834,7 +16834,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "nu-parser", @@ -16845,7 +16845,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16856,7 +16856,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16869,7 +16869,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-recursion", @@ -16893,7 +16893,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "lazy_static", @@ -16907,7 +16907,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16924,7 +16924,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "lazy_static", @@ -16939,7 +16939,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "lazy_static", @@ -16958,7 +16958,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16974,7 +16974,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "serde", @@ -16985,7 +16985,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-recursion", @@ -17022,7 +17022,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "const_format", @@ -17060,7 +17060,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.651.1" +version = "1.652.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17071,7 +17071,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-recursion", @@ -17100,7 +17100,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17123,7 +17123,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17156,7 +17156,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17176,7 +17176,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17210,7 +17210,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17245,7 +17245,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17268,7 +17268,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17292,7 +17292,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-nats", @@ -17316,7 +17316,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17351,7 +17351,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17379,7 +17379,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-trait", @@ -17402,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17420,7 +17420,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.651.1" +version = "1.652.0" dependencies = [ "anyhow", "async-once-cell", @@ -17526,7 +17526,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.651.1" +version = "1.652.0" dependencies = [ "bytes", "futures", @@ -18126,9 +18126,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] @@ -18409,18 +18409,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5" +checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.40" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953" +checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0bbdc65bd7..fbff11ad2f 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.651.1" +version = "1.652.0" authors.workspace = true edition.workspace = true @@ -78,7 +78,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.651.1" +version = "1.652.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8960dfb2a1..d4f522eb1a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.651.1 + version: 1.652.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index a16ab8c6f6..0a816b8f13 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.651.1"; +export const VERSION = "v1.652.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index f03db28745..a3f87041e1 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.651.1"; +export const VERSION = "1.652.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6d7bb1a4fb..3eb871b624 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.651.1", + "version": "1.652.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.651.1", + "version": "1.652.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 744c4b2450..d17ff4e8a5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.651.1", + "version": "1.652.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 8783b3fa33..1087f52434 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.651.1" +wmill = ">=1.652.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 491b9434bf..68dc181dfa 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.651.1 + version: 1.652.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index e037ad67de..7248184a8b 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.651.1' + ModuleVersion = '1.652.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index cf3c7460a5..50ef005417 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.651.1" +version = "1.652.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 104171df6c..c82e6fb8a0 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.651.1", + "version": "1.652.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 75812526a2..37228629ba 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.651.1", + "version": "1.652.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index ab7ecc87ec..343d46525c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.651.1 +1.652.0 From badb6a669738377c0a41f402834255d5cfdfa7c7 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Mon, 9 Mar 2026 17:39:03 -0400 Subject: [PATCH 47/57] feat: add slack connection fields to workspace settings export/import (#8287) Co-authored-by: Claude Opus 4.6 --- backend/windmill-api/src/workspaces_export.rs | 26 +++++++++++++++++- cli/src/core/settings.ts | 27 ++++++++++++++++++- cli/test/settings_unit.test.ts | 16 +++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 091c465a61..cd32587648 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -282,6 +282,12 @@ struct SimplifiedSettings { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // V1 format: Legacy flat format for backward compatibility (matches main branch exactly) @@ -316,6 +322,12 @@ struct SimplifiedSettingsLegacy { color: Option, #[serde(skip_serializing_if = "Option::is_none")] operator_settings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_team_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + slack_command_script: Option, } // Internal struct for querying database @@ -335,6 +347,9 @@ struct SettingsRow { mute_critical_alerts: Option, color: Option, operator_settings: Option, + slack_team_id: Option, + slack_name: Option, + slack_command_script: Option, } pub(crate) async fn tarball_workspace( @@ -939,7 +954,10 @@ pub(crate) async fn tarball_workspace( workspace.name as name, mute_critical_alerts, color, - operator_settings + operator_settings, + slack_team_id, + slack_name, + slack_command_script FROM workspace_settings LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id WHERE workspace_id = $1"#, @@ -965,6 +983,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color.clone(), operator_settings: row.operator_settings.clone(), + slack_team_id: row.slack_team_id.clone(), + slack_name: row.slack_name.clone(), + slack_command_script: row.slack_command_script.clone(), }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) @@ -1024,6 +1045,9 @@ pub(crate) async fn tarball_workspace( mute_critical_alerts: row.mute_critical_alerts, color: row.color, operator_settings: row.operator_settings, + slack_team_id: row.slack_team_id, + slack_name: row.slack_name, + slack_command_script: row.slack_command_script, }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 3ffddc4837..0f7acce4a9 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -53,6 +53,9 @@ export interface SimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Legacy settings interface for reading old settings.yaml files @@ -77,6 +80,9 @@ interface LegacySimplifiedSettings { mute_critical_alerts?: boolean; color?: string; operator_settings?: any; + slack_team_id?: string; + slack_name?: string; + slack_command_script?: string; } // Helper to convert legacy flat settings to new grouped format @@ -94,6 +100,9 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings { if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts; if (settings.color !== undefined) result.color = settings.color; if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings; + if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id; + if (settings.slack_name !== undefined) result.slack_name = settings.slack_name; + if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script; // Handle auto_invite: check if already grouped or needs migration if (settings.auto_invite && typeof settings.auto_invite === "object") { @@ -183,12 +192,18 @@ export async function pushWorkspaceSettings( mute_critical_alerts: remoteSettings.mute_critical_alerts, color: remoteSettings.color, operator_settings: remoteSettings.operator_settings, + slack_team_id: remoteSettings.slack_team_id, + slack_name: remoteSettings.slack_name, + slack_command_script: remoteSettings.slack_command_script, }; } catch (err) { throw new Error(`Failed to get workspace settings: ${err}`); } - if (isSuperset(localSettings, settings)) { + // Exclude read-only fields from comparison (slack_team_id and slack_name are set via OAuth only) + const { slack_team_id: _lst, slack_name: _lsn, ...comparableLocal } = localSettings; + const { slack_team_id: _rst, slack_name: _rsn, ...comparableRemote } = settings; + if (isSuperset(comparableLocal, comparableRemote)) { log.debug(`Workspace settings are up to date`); return; } @@ -366,6 +381,16 @@ export async function pushWorkspaceSettings( requestBody: localSettings.operator_settings, }); } + + if (localSettings.slack_command_script != settings.slack_command_script) { + log.debug(`Updating slack command script...`); + await wmill.editSlackCommand({ + workspace, + requestBody: { + slack_command_script: localSettings.slack_command_script, + }, + }); + } } export async function pushWorkspaceKey( diff --git a/cli/test/settings_unit.test.ts b/cli/test/settings_unit.test.ts index 2d6a5249ef..014456cfc0 100644 --- a/cli/test/settings_unit.test.ts +++ b/cli/test/settings_unit.test.ts @@ -193,5 +193,21 @@ describe("migrateToGroupedFormat", () => { expect("webhook" in result).toBe(false); expect("deploy_to" in result).toBe(false); expect("color" in result).toBe(false); + expect("slack_team_id" in result).toBe(false); + expect("slack_name" in result).toBe(false); + expect("slack_command_script" in result).toBe(false); + }); + + test("copies slack fields through", () => { + const settings = { + name: "ws", + slack_team_id: "T12345", + slack_name: "my-team", + slack_command_script: "u/admin/slack_handler", + }; + const result = migrateToGroupedFormat(settings); + expect(result.slack_team_id).toBe("T12345"); + expect(result.slack_name).toBe("my-team"); + expect(result.slack_command_script).toBe("u/admin/slack_handler"); }); }); From 1e2413f02be1245ee177a8ce7f805a06de5589a9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 10 Mar 2026 04:59:56 +0000 Subject: [PATCH 48/57] perf: optimize job_stats storage for timestamps and zero-memory jobs (#8289) * perf: optimize job_stats storage for timestamps and zero-memory jobs Co-Authored-By: Claude Opus 4.6 * chore: update sqlx offline cache nullable metadata Co-Authored-By: Claude Opus 4.6 * refactor: use centisecond offsets for job_stats timestamps (~248 day range) Co-Authored-By: Claude Opus 4.6 * fix: update SELECT to use offsets_cs column name Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- ...e1b60caa3752669636e9fb0817a68121a9451.json | 17 +++++ ...f5747cb50d193475fff5fcfdde37d1bc74636.json | 17 ----- ...a861076a5fc4f7eefceb1c0de5cf55293f327.json | 17 +++++ ...4d3af288a346c50f29809dbc55a34088a0abc.json | 17 ----- ...000_optimize_job_stats_timestamps.down.sql | 2 + ...00000_optimize_job_stats_timestamps.up.sql | 5 ++ backend/windmill-api-jobs/src/job_metrics.rs | 67 +++++++++++-------- backend/windmill-common/src/job_metrics.rs | 62 +++++++++++------ backend/windmill-worker/src/handle_child.rs | 33 ++++----- 9 files changed, 138 insertions(+), 99 deletions(-) create mode 100644 backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json delete mode 100644 backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json create mode 100644 backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json delete mode 100644 backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json create mode 100644 backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql create mode 100644 backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql diff --git a/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json new file mode 100644 index 0000000000..afc74d8e7a --- /dev/null +++ b/backend/.sqlx/query-0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "0af0e0a1dddeee2021ba060e390e1b60caa3752669636e9fb0817a68121a9451" +} diff --git a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json b/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json deleted file mode 100644 index aaa1ae945b..0000000000 --- a/backend/.sqlx/query-1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Int4" - ] - }, - "nullable": [] - }, - "hash": "1db82007445ff5f644bb607aa28f5747cb50d193475fff5fcfdde37d1bc74636" -} diff --git a/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json new file mode 100644 index 0000000000..dc77dbc402 --- /dev/null +++ b/backend/.sqlx/query-a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Float4" + ] + }, + "nullable": [] + }, + "hash": "a837494a58ab58bfa18c0385350a861076a5fc4f7eefceb1c0de5cf55293f327" +} diff --git a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json b/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json deleted file mode 100644 index c071272ff9..0000000000 --- a/backend/.sqlx/query-d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text", - "Float4" - ] - }, - "nullable": [] - }, - "hash": "d44c37882150532383d1058639f4d3af288a346c50f29809dbc55a34088a0abc" -} diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql new file mode 100644 index 0000000000..e526b89b11 --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE job_stats DROP COLUMN IF EXISTS timeseries_start; +ALTER TABLE job_stats DROP COLUMN IF EXISTS offsets_cs; diff --git a/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql new file mode 100644 index 0000000000..f851c7887e --- /dev/null +++ b/backend/migrations/20260311000000_optimize_job_stats_timestamps.up.sql @@ -0,0 +1,5 @@ +-- Store timeseries timestamps as a start time + integer centisecond offsets +-- instead of full TIMESTAMPTZ[] arrays. Saves ~4 bytes per data point. +-- i32 centiseconds gives ~248 days of range with 10ms precision. +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS timeseries_start TIMESTAMPTZ; +ALTER TABLE job_stats ADD COLUMN IF NOT EXISTS offsets_cs INTEGER[]; diff --git a/backend/windmill-api-jobs/src/job_metrics.rs b/backend/windmill-api-jobs/src/job_metrics.rs index aba8a5f25c..ad316150c0 100644 --- a/backend/windmill-api-jobs/src/job_metrics.rs +++ b/backend/windmill-api-jobs/src/job_metrics.rs @@ -79,7 +79,7 @@ async fn get_job_metrics( >, ) -> error::JsonResult { let records = sqlx::query_as::<_, JobStatsRecord>( - "SELECT * FROM job_stats where workspace_id = $1 and job_id = $2", + "SELECT workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float, timeseries_start, offsets_cs FROM job_stats WHERE workspace_id = $1 AND job_id = $2", ) .bind(w_id) .bind(job_id) @@ -91,7 +91,7 @@ async fn get_job_metrics( let mut timeseries_metrics: Vec = vec![]; for record in records { - let metric_id = record.metric_id; + let metric_id = record.metric_id.clone(); match record.metric_kind { MetricKind::ScalarInt => { let value = record.scalar_int.unwrap_or_default() as f64; @@ -102,47 +102,43 @@ async fn get_job_metrics( scalar_metrics.push(ScalarMetric { metric_id: metric_id.clone(), value }); } MetricKind::TimeseriesInt => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_int) + let timestamps = resolve_timestamps(&record); + let timeseries_int = record.timeseries_int.unwrap_or_default(); + if timestamps.len() != timeseries_int.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_int.len()); } let (timestamps, timeseries_int) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_int.unwrap_or_default(), + timestamps, + timeseries_int, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_int.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_int.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } MetricKind::TimeseriesFloat => { - if record.timestamps.clone().unwrap_or_default().len() - != record.timeseries_int.clone().unwrap_or_default().len() - { - tracing::warn!("Timeseries metric {} has an invalid shape. It doesn't have one timestamp per measurement. (timestamps: {:?}, measurements: {:?})", metric_id, record.timestamps, record.timeseries_float) + let timestamps = resolve_timestamps(&record); + let timeseries_float = record.timeseries_float.unwrap_or_default(); + if timestamps.len() != timeseries_float.len() { + tracing::warn!("Timeseries metric {} has an invalid shape. timestamps: {}, measurements: {}", metric_id, timestamps.len(), timeseries_float.len()); } let (timestamps, timeseries_float) = timeseries_sample( from_timestamp, to_timestamp, timeseries_max_datapoints, - record.timestamps.unwrap_or_default(), - record.timeseries_float.unwrap_or_default(), + timestamps, + timeseries_float, ); - let mut values: Vec = vec![]; - for (idx, value) in timeseries_float.iter().enumerate() { - values.push(DataPoint { - timestamp: timestamps[idx], - value: value.to_owned() as f64, - }); - } + let values: Vec = timestamps + .iter() + .zip(timeseries_float.iter()) + .map(|(ts, v)| DataPoint { timestamp: *ts, value: *v as f64 }) + .collect(); timeseries_metrics.push(TimeseriesMetric { metric_id: metric_id.clone(), values }); } }; @@ -152,6 +148,21 @@ async fn get_job_metrics( let response = JobStatsResponse { metrics_metadata, scalar_metrics, timeseries_metrics }; Ok(Json(response)) } + +/// Reconstruct full timestamps from `timeseries_start` + `offsets_cs` if available, +/// otherwise fall back to legacy `timestamps` column. +fn resolve_timestamps(record: &JobStatsRecord) -> Vec> { + if let (Some(start), Some(offsets)) = (record.timeseries_start, &record.offsets_cs) { + if !offsets.is_empty() { + return offsets + .iter() + .map(|&cs| start + chrono::Duration::milliseconds(cs as i64 * 10)) + .collect(); + } + } + // Legacy fallback: use the full timestamps column + record.timestamps.clone().unwrap_or_default() +} #[derive(Deserialize)] struct JobProgressSetRequest { percent: i32, diff --git a/backend/windmill-common/src/job_metrics.rs b/backend/windmill-common/src/job_metrics.rs index d416e4d44b..c49185de72 100644 --- a/backend/windmill-common/src/job_metrics.rs +++ b/backend/windmill-common/src/job_metrics.rs @@ -15,9 +15,11 @@ pub struct JobStatsRecord { pub timestamps: Option>>, pub timeseries_int: Option>, pub timeseries_float: Option>, + pub timeseries_start: Option>, + pub offsets_cs: Option>, } -#[derive(sqlx::Type, Debug, PartialEq, Deserialize, Serialize)] +#[derive(sqlx::Type, Debug, Clone, PartialEq, Deserialize, Serialize)] #[sqlx(type_name = "METRIC_KIND", rename_all = "snake_case")] pub enum MetricKind { ScalarInt, @@ -52,29 +54,21 @@ pub async fn register_metric_for_job( return Ok(metric_id); } - let (scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) = match metric_kind - { + let is_timeseries = matches!( + metric_kind, + MetricKind::TimeseriesInt | MetricKind::TimeseriesFloat + ); + + let (scalar_int, scalar_float, timeseries_int, timeseries_float) = match metric_kind { MetricKind::ScalarInt | MetricKind::ScalarFloat => { - (None as Option, None as Option, None, None, None) + (None as Option, None as Option, None, None) } - MetricKind::TimeseriesInt => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - Some(&[] as &[i32]), - None, - ), - MetricKind::TimeseriesFloat => ( - None, - None, - Some(&[] as &[chrono::DateTime]), - None, - Some(&[] as &[f32]), - ), + MetricKind::TimeseriesInt => (None, None, Some(&[] as &[i32]), None), + MetricKind::TimeseriesFloat => (None, None, None, Some(&[] as &[f32])), }; sqlx::query( - "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timestamps, timeseries_int, timeseries_float) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", + "INSERT INTO job_stats (workspace_id, job_id, metric_id, metric_name, metric_kind, scalar_int, scalar_float, timeseries_int, timeseries_float, timeseries_start, offsets_cs) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, CASE WHEN $10 THEN now() ELSE NULL END, CASE WHEN $10 THEN ARRAY[]::int[] ELSE NULL END)", ) .bind(workspace_id) .bind(job_id) @@ -83,9 +77,9 @@ pub async fn register_metric_for_job( .bind(metric_kind) .bind(scalar_int) .bind(scalar_float) - .bind(timestamps) .bind(timeseries_int) .bind(timeseries_float) + .bind(is_timeseries) .execute(db) .warn_after_seconds(1) .await?; @@ -117,6 +111,30 @@ pub async fn record_metric( } let metric_kind = metric_kind_opt.unwrap(); + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +/// Record a timeseries metric value without the extra SELECT to look up metric_kind. +/// Use this when the caller already knows the metric kind (e.g. the worker that registered it). +pub async fn record_timeseries_value( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { + record_metric_impl(db, workspace_id, job_id, metric_id, value, metric_kind).await +} + +async fn record_metric_impl( + db: &DB, + workspace_id: String, + job_id: Uuid, + metric_id: String, + value: MetricNumericValue, + metric_kind: MetricKind, +) -> error::Result<()> { let (value_int, value_float) = match value { MetricNumericValue::Integer(val) => { if metric_kind != MetricKind::TimeseriesInt && metric_kind != MetricKind::ScalarInt { @@ -160,7 +178,7 @@ pub async fn record_metric( } MetricKind::TimeseriesInt => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_int = array_append(timeseries_int, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, @@ -169,7 +187,7 @@ pub async fn record_metric( } MetricKind::TimeseriesFloat => { sqlx::query!( - "UPDATE job_stats SET timestamps = array_append(timestamps, now()), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", + "UPDATE job_stats SET offsets_cs = array_append(offsets_cs, (EXTRACT(EPOCH FROM (now() - timeseries_start)) * 100)::int), timeseries_float = array_append(timeseries_float, $4) WHERE workspace_id = $1 AND job_id = $2 AND metric_id = $3", &workspace_id, &job_id, &metric_id, diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 1a107990dc..205da6700a 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -737,21 +737,24 @@ where let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0; if update_job_row && job_id != Uuid::nil() { if let Connection::Sql(ref db) = conn { - // tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs - if i == 2 { - memory_metric_id = job_metrics::register_metric_for_job( - &db, - w_id.to_string(), - job_id, - "memory_kb".to_string(), - job_metrics::MetricKind::TimeseriesInt, - Some("Job Memory Footprint (kB)".to_string()), - ) - .await; - } - if let Ok(ref metric_id) = memory_metric_id { - if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await { - tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + // Only track memory when it's non-zero (avoids storing all-zero timeseries for jobs that don't report memory) + if current_mem > 0 { + // Register on first non-zero reading (deferred from i==2 to avoid metric for jobs with no memory reporting) + if memory_metric_id.is_err() { + memory_metric_id = job_metrics::register_metric_for_job( + &db, + w_id.to_string(), + job_id, + "memory_kb".to_string(), + job_metrics::MetricKind::TimeseriesInt, + Some("Job Memory Footprint (kB)".to_string()), + ) + .await; + } + if let Ok(ref metric_id) = memory_metric_id { + if let Err(err) = job_metrics::record_timeseries_value(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem), job_metrics::MetricKind::TimeseriesInt).await { + tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); + } } } } From 718406537f9da960ad1e5917fea04bc7bb46aefe Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 10 Mar 2026 05:22:12 +0000 Subject: [PATCH 49/57] feat: add indexer time window setting (default 7 days) (#8290) * feat: add indexer time window setting (default 7 days) Co-Authored-By: Claude Opus 4.6 * feat: add time window note to search UIs Co-Authored-By: Claude Opus 4.6 * feat: fetch indexer time window from API in search UIs Co-Authored-By: Claude Opus 4.6 * chore: update ee-repo-ref to 9df755c57fbfc88f4a724e1ea51b1d5f5af4fe52 This commit updates the EE repository reference after PR #447 was merged in windmill-ee-private. Previous ee-repo-ref: c17f16bf45091272974e3aa8009cdf5cc15669bf New ee-repo-ref: 9df755c57fbfc88f4a724e1ea51b1d5f5af4fe52 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api/openapi.yaml | 3 ++ backend/windmill-common/src/indexer.rs | 10 ++++++ .../lib/components/ServiceLogsInner.svelte | 8 ++++- .../src/lib/components/instanceSettings.ts | 3 +- .../IndexerMemorySettings.svelte | 31 +++++++++++++++++++ .../lib/components/search/RunsSearch.svelte | 6 +++- 7 files changed, 59 insertions(+), 4 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 060c5666c7..75a2b893b2 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -716b350bce1730b302c66ea69df618fa40f2f16b +9df755c57fbfc88f4a724e1ea51b1d5f5af4fe52 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index d4f522eb1a..f94407e56a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16876,6 +16876,9 @@ paths: lost_lock_ownership: description: Is the current indexer service being replaced type: boolean + max_index_time_window_secs: + description: Maximum time window in seconds for indexing + type: number /srch/index/search/service_logs: get: diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 45a924748d..23bde13795 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -13,6 +13,7 @@ pub struct TantivyIndexerSettings { pub refresh_index_period: u64, pub refresh_log_index_period: u64, pub max_indexed_job_log_size: usize, + pub max_index_time_window_secs: i64, pub should_clear_job_index: bool, pub should_clear_log_index: bool, } @@ -26,6 +27,7 @@ impl Default for TantivyIndexerSettings { refresh_index_period: 300, refresh_log_index_period: 300, max_indexed_job_log_size: 1_000_000, + max_index_time_window_secs: 60 * 60 * 24 * 7, // 7 days should_clear_job_index: false, should_clear_log_index: false, } @@ -39,6 +41,7 @@ pub struct TantivyIndexerSettingsOpt { pub refresh_index_period: Option, pub refresh_log_index_period: Option, pub max_indexed_job_log_size: Option, + pub max_index_time_window_secs: Option, pub should_clear_job_index: Option, pub should_clear_log_index: Option, } @@ -58,6 +61,7 @@ pub async fn load_indexer_config(db: &DB) -> error::Result error::Result TantivyIndexerSettings { if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") { settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize; } + if let Some(b) = get_env_var("TANTIVY_MAX_INDEX_TIME_WINDOW__S") { + settings.max_index_time_window_secs = b as i64; + } settings } diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index abcffa32ef..df0298f579 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -523,7 +523,13 @@ {#if allLogs == undefined}
{:else if Object.keys(allLogs).length == 0} -
No logs
+
+ No logs + Search only covers a recent time window, configurable in instance settings + under Indexer. +
{:else if minTs && maxTs} {@const minTsN = new Date(minTs).getTime()} {@const maxTsN = new Date(maxTs).getTime()} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 29c730b736..5374f0c258 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -85,7 +85,8 @@ const indexerSettingsSchema = z refresh_index_period: positiveNumber.optional(), max_indexed_job_log_size: positiveNumber.optional(), commit_log_max_batch_size: positiveNumber.optional(), - refresh_log_index_period: positiveNumber.optional() + refresh_log_index_period: positiveNumber.optional(), + max_index_time_window_secs: positiveNumber.optional() }) .passthrough() diff --git a/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte b/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte index 5a9dfb496a..52bc3cd9bc 100644 --- a/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte +++ b/frontend/src/lib/components/instanceSettings/IndexerMemorySettings.svelte @@ -84,6 +84,37 @@ />
+
+ + { + if (v == null) { + const { max_index_time_window_secs: _, ...rest } = $values['indexer_settings'] + $values['indexer_settings'] = rest + } else { + $values['indexer_settings'] = { + ...$values['indexer_settings'], + max_index_time_window_secs: v * 86400 + } + } + }} + /> + +
{/if}
- Note that new runs might take a while to become searchable (by default ~5min) + Note that new runs might take a while to become searchable (by default ~5min). + {#if indexMetadata?.max_index_time_window_secs} + Search only covers the last {Math.round(indexMetadata.max_index_time_window_secs / 86400)} day(s), + configurable in instance settings under Indexer. + {/if}
{#if !$enterpriseLicense}
From bc9f235e1e906ce8199c79eee04fb7ada5e1b4a5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 10 Mar 2026 05:39:29 +0000 Subject: [PATCH 50/57] chore(main): release 1.653.0 (#8288) * chore(main): release 1.653.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 13 ++ backend/Cargo.lock | 148 +++++++++--------- backend/Cargo.toml | 4 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 15 files changed, 102 insertions(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26e8dbf644..feec672d41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## [1.653.0](https://github.com/windmill-labs/windmill/compare/v1.652.0...v1.653.0) (2026-03-10) + + +### Features + +* add indexer time window setting (default 7 days) ([#8290](https://github.com/windmill-labs/windmill/issues/8290)) ([0c4d72c](https://github.com/windmill-labs/windmill/commit/0c4d72cfe38d61cf3f6e9bc31056005f1adb494d)) +* add slack connection fields to workspace settings export/import ([#8287](https://github.com/windmill-labs/windmill/issues/8287)) ([39e77ec](https://github.com/windmill-labs/windmill/commit/39e77ecd002b41630fa8d146ee0f15369656acda)) + + +### Performance Improvements + +* optimize job_stats storage for timestamps and zero-memory jobs ([#8289](https://github.com/windmill-labs/windmill/issues/8289)) ([2d8335d](https://github.com/windmill-labs/windmill/commit/2d8335dc43a7cb182eb5a058119d8b0be067cdfd)) + ## [1.652.0](https://github.com/windmill-labs/windmill/compare/v1.651.1...v1.652.0) (2026-03-09) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 10589a44fc..64a04ce4ec 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8102,9 +8102,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.24" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4735e9cbde5aac84a5ce588f6b23a90b9b0b528f6c5a8db8a4aff300463a0839" +checksum = "d52f4c29e2a68ac30c9087e1b772dc9f44a2b66ed44edf2266cf2be9b03dafc1" dependencies = [ "cc", "libc", @@ -15741,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -15808,7 +15808,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15821,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "argon2", @@ -15961,7 +15961,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15984,7 +15984,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -15997,7 +15997,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16023,7 +16023,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.652.0" +version = "1.653.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16033,7 +16033,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16050,7 +16050,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "base64 0.22.1", @@ -16073,7 +16073,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16096,7 +16096,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16112,7 +16112,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16132,7 +16132,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16152,7 +16152,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16166,7 +16166,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -16193,7 +16193,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16218,7 +16218,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "flate2", @@ -16236,7 +16236,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16257,7 +16257,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16277,7 +16277,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16307,7 +16307,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16334,7 +16334,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.652.0" +version = "1.653.0" dependencies = [ "lazy_static", "serde", @@ -16346,7 +16346,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.652.0" +version = "1.653.0" dependencies = [ "argon2", "axum 0.7.9", @@ -16369,7 +16369,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16383,7 +16383,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.652.0" +version = "1.653.0" dependencies = [ "axum 0.7.9", "chrono", @@ -16414,7 +16414,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.652.0" +version = "1.653.0" dependencies = [ "chrono", "lazy_static", @@ -16428,7 +16428,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -16447,7 +16447,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.652.0" +version = "1.653.0" dependencies = [ "aes-gcm", "anyhow", @@ -16546,7 +16546,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.652.0" +version = "1.653.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16565,7 +16565,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.652.0" +version = "1.653.0" dependencies = [ "regex", "serde", @@ -16580,7 +16580,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16604,7 +16604,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "futures", @@ -16621,7 +16621,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.652.0" +version = "1.653.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16637,7 +16637,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -16658,7 +16658,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -16689,7 +16689,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-oauth2", @@ -16713,7 +16713,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-stream", @@ -16747,7 +16747,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "futures", @@ -16765,7 +16765,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.652.0" +version = "1.653.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -16774,7 +16774,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16786,7 +16786,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "serde_json", @@ -16798,7 +16798,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "gosyn", @@ -16810,7 +16810,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16822,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "serde_json", @@ -16834,7 +16834,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "nu-parser", @@ -16845,7 +16845,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16856,7 +16856,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16869,7 +16869,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -16893,7 +16893,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16907,7 +16907,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16924,7 +16924,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16939,7 +16939,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "lazy_static", @@ -16958,7 +16958,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16974,7 +16974,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "serde", @@ -16985,7 +16985,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -17022,7 +17022,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "const_format", @@ -17060,7 +17060,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.652.0" +version = "1.653.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17071,7 +17071,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-recursion", @@ -17100,7 +17100,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "axum 0.7.9", @@ -17123,7 +17123,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17156,7 +17156,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17176,7 +17176,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17210,7 +17210,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17245,7 +17245,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17268,7 +17268,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17292,7 +17292,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-nats", @@ -17316,7 +17316,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17351,7 +17351,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17379,7 +17379,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-trait", @@ -17402,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17420,7 +17420,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.652.0" +version = "1.653.0" dependencies = [ "anyhow", "async-once-cell", @@ -17526,7 +17526,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.652.0" +version = "1.653.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index fbff11ad2f..36eca6296b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.652.0" +version = "1.653.0" authors.workspace = true edition.workspace = true @@ -78,7 +78,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.652.0" +version = "1.653.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f94407e56a..fe28092e87 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.652.0 + version: 1.653.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 0a816b8f13..121d5b6c6d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.652.0"; +export const VERSION = "v1.653.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index a3f87041e1..6ca14e9975 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -67,7 +67,7 @@ export { workspaceAdd, }; -export const VERSION = "1.652.0"; +export const VERSION = "1.653.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3eb871b624..647db391cd 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.652.0", + "version": "1.653.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.652.0", + "version": "1.653.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index d17ff4e8a5..66e1da26b3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.652.0", + "version": "1.653.0", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 1087f52434..de69e3add0 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.652.0" +wmill = ">=1.653.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 68dc181dfa..d14454be0b 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.652.0 + version: 1.653.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 7248184a8b..f03a1970d3 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.652.0' + ModuleVersion = '1.653.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 50ef005417..29ea738024 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.652.0" +version = "1.653.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index c82e6fb8a0..7daf5bf8e9 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.652.0", + "version": "1.653.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 37228629ba..08101b7678 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.652.0", + "version": "1.653.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index 343d46525c..be0764647c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.652.0 +1.653.0 From 0fcb29cfad75bef7be676b58d15740f53bba4faf Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 10 Mar 2026 10:06:02 +0100 Subject: [PATCH 51/57] feat(frontend): replace flat sugiyama with recursive compound layout for flow graph (#8204) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(frontend): replace flat sugiyama with recursive compound layout for flow graph Co-Authored-By: Claude Opus 4.6 * fix(frontend): double forloop wrapper padding and include wrappers in bbox Co-Authored-By: Claude Opus 4.5 * fix(frontend): gate debug wrappers behind SHOW_DEBUG_WRAPPERS flag Remove all debug console.log calls from compoundLayout and gate WrapperInfo creation and wrapper node rendering behind an exported SHOW_DEBUG_WRAPPERS constant. Replace wrapper-based bbox computation with groupLayouts-based loop so no WrapperInfo is needed for correct layout. Add contentMinX to LayoutResult for the top-level minX shift. Co-Authored-By: Claude Opus 4.6 * fix(frontend): remove debug wrapper nodes from flow graph Remove WrapperInfo type, SHOW_DEBUG_WRAPPERS flag, buildDebugWrapperNodes helper, DebugWrapperNode component, and all related plumbing in FlowGraphV2. The bbox computation now uses groupLayouts directly, keeping layout correctness without any debug wrapper overhead. Co-Authored-By: Claude Opus 4.6 * perf(frontend): optimize compoundLayout recursive algorithm Co-Authored-By: Claude Opus 4.6 * refactor(frontend): remove dead offset plumbing from flow graph The old flat sugiyama layout used a CSS margin-left hack (offset) to indent loop bodies. The new recursive compound layout handles indentation natively via coordinates, making the entire offset pipeline dead code. Removes offset from 11 node type definitions, NodeLayout, addNode helper, processModules parameter, NodeWrapper prop, 9 node renderers, AssetNode x-position calculations, AIToolNode x-position calculations, DragGhost nodeOffset function, FlowGraphV2 layout pipeline, util.ts type signatures, noteUtils NodeDep type, and noteEditor function signature. Co-Authored-By: Claude Opus 4.6 * fix(frontend): remove unused lastXCenter variable Co-Authored-By: Claude Opus 4.6 * perf(frontend): optimize compoundLayout hot paths Replace O(N²) queue.shift() with index pointer in BFS, eliminate redundant groupOwnedIds double-build, use Set for parent dedup, track minY in existing bbox loop, and cache maxBranchHeight. Co-Authored-By: Claude Opus 4.6 * chore: remove debug artifacts from PR Remove elk_viewer test page, console log dumps, and layout screenshots that were used during development. Co-Authored-By: Claude Opus 4.6 * fix(frontend): guard data.module.value access in ModuleNode When rapidly clicking expand/collapse on a subflow, the graph rebuilds and data.module can be transiently undefined. Add optional chaining to prevent "Cannot read properties of undefined (reading 'value')" errors. Co-Authored-By: Claude Opus 4.6 * refactor(frontend): simplify CompoundGroup type to 'branch' | 'loop' The layout never distinguishes branchall/branchone or forloop/whileloop, so collapse to two variants that match the actual code paths. Co-Authored-By: Claude Opus 4.6 * fix(frontend): address PR review feedback on flow layout - Add max recursion depth guard (50) to layoutLevel to prevent stack overflow with malformed flow data - Log swallowed decrossOpt error as console.debug for debuggability - Initialize maxY to -Infinity for correctness with negative positions - Fix indentation artifacts in graphBuilder data objects Co-Authored-By: Claude Opus 4.6 * formatting * fix: remove offset field from asset node data in FlowGraphV2 Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 --- .../src/lib/components/graph/DragGhost.svelte | 8 +- .../lib/components/graph/FlowGraphV2.svelte | 84 +-- .../lib/components/graph/compoundLayout.ts | 555 ++++++++++++++++++ .../components/graph/graphBuilder.svelte.ts | 59 +- .../lib/components/graph/noteEditor.svelte.ts | 2 +- .../lib/components/graph/noteUtils.svelte.ts | 3 - .../graph/renderers/nodes/AIToolNode.svelte | 4 +- .../graph/renderers/nodes/AssetNode.svelte | 16 +- .../renderers/nodes/BranchAllEndNode.svelte | 2 +- .../renderers/nodes/BranchAllStart.svelte | 2 +- .../renderers/nodes/BranchOneStart.svelte | 2 +- .../renderers/nodes/ForLoopEndNode.svelte | 2 +- .../renderers/nodes/ForLoopStartNode.svelte | 2 +- .../graph/renderers/nodes/ModuleNode.svelte | 13 +- .../graph/renderers/nodes/NoBranchNode.svelte | 2 +- .../graph/renderers/nodes/NodeWrapper.svelte | 8 +- .../graph/renderers/nodes/SubflowBound.svelte | 2 +- .../renderers/nodes/branchOneEndNode.svelte | 2 +- frontend/src/lib/components/graph/util.ts | 13 +- 19 files changed, 623 insertions(+), 158 deletions(-) create mode 100644 frontend/src/lib/components/graph/compoundLayout.ts diff --git a/frontend/src/lib/components/graph/DragGhost.svelte b/frontend/src/lib/components/graph/DragGhost.svelte index 6d2e26e10e..0aefc5c9e8 100644 --- a/frontend/src/lib/components/graph/DragGhost.svelte +++ b/frontend/src/lib/components/graph/DragGhost.svelte @@ -18,10 +18,6 @@ /** Offset so the cursor indicator icon doesn't overlap the cursor tip */ const CURSOR_INDICATOR_OFFSET = 8 - function nodeOffset(n: Node): number { - return ((n.data as Record)?.offset as number) ?? 0 - } - function getSubflowNodesAndEdges( moduleId: string, allNodes: Node[], @@ -68,7 +64,7 @@ maxY = -Infinity for (const n of sfNodes) { const abs = absolutePosition(n, allNodes) - const x = abs.x + nodeOffset(n) + const x = abs.x const y = abs.y const w = n.measured?.width ?? NODE.width const h = n.measured?.height ?? NODE.height @@ -90,7 +86,7 @@ let offsetY = containerHeight / 2 if (mainNode) { const mainAbs = absolutePosition(mainNode, allNodes) - const mx = mainAbs.x + nodeOffset(mainNode) - minX + PADDING + const mx = mainAbs.x - minX + PADDING const my = mainAbs.y - minY + PADDING const mw = mainNode.measured?.width ?? NODE.width const mh = mainNode.measured?.height ?? NODE.height diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 96b50204ba..853dbf5c54 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -20,7 +20,6 @@ import { graphBuilder, isTriggerStep, - topologicalSort, type InlineScript, type InsertKind, type NodeLayout, @@ -36,7 +35,6 @@ import ResultNode from './renderers/nodes/ResultNode.svelte' import BaseEdge from './renderers/edges/BaseEdge.svelte' import EmptyEdge from './renderers/edges/EmptyEdge.svelte' - import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' import { Expand, MousePointer, Hand } from 'lucide-svelte' import Toggle from '../Toggle.svelte' import DataflowEdge from './renderers/edges/DataflowEdge.svelte' @@ -71,6 +69,7 @@ import type { MoveManager } from './moveManager.svelte' import DragCoordinator from './DragCoordinator.svelte' import type { ModulesTestStates } from '../modulesTest.svelte' + import { compoundLayout } from './compoundLayout' import { deepEqual } from 'fast-equals' import type { AssetWithAltAccessType } from '../assets/lib' import type { ModuleActionInfo } from '$lib/components/flows/flowDiff' @@ -333,7 +332,6 @@ type NodeDep = { id: string parentIds?: string[] - offset?: number data?: { assets?: AssetWithAltAccessType[] } } type NodePos = { position: { x: number; y: number } } @@ -354,59 +352,21 @@ seenId.push(n.id) } - let nodeWidths: Record = {} - const nodes2: (NodeDep & NodePos)[] = nodes.map((n) => { - return { ...n, position: { x: 0, y: 0 } } + // Run recursive compound layout + const { positions, bbox } = compoundLayout(nodes, { + nodeWidth: NODE.width, + nodeHeight: NODE.height, + gapH: NODE.gap.horizontal, + gapV: NODE.gap.vertical }) - for (const n of topologicalSort(nodes)) { - const endId = n.id + '-end' - if (nodeWidths[endId] != undefined) { - nodeWidths[n.id] = Math.max(nodeWidths[n.id] ?? 0, nodeWidths[endId]) - } - if (n.parentIds && n.parentIds?.length == 1) { - const parent = n.parentIds[0] - const nodeWidth = nodeWidths[n.id] ?? 1 - nodeWidths[parent] = (nodeWidths[parent] ?? 0) + nodeWidth - } - } - - const dag = dagStratify().id(({ id }: NodeDep & NodePos) => id)(nodes2) - - let boxSize: any - try { - const layout = sugiyama() - .decross(nodes.length > 20 ? decrossTwoLayer() : decrossOpt()) - .coord(coordCenter()) - .nodeSize((d) => { - return [ - (nodeWidths[d?.data?.['id'] ?? ''] ?? 1) * (NODE.width + NODE.gap.horizontal * 1), - NODE.height + NODE.gap.vertical - ] as readonly [number, number] - }) - boxSize = layout(dag as any) - } catch { - const layout = sugiyama() - .decross(decrossTwoLayer()) - .coord(coordCenter()) - .nodeSize(() => [NODE.width + NODE.gap.horizontal, NODE.height + NODE.gap.vertical]) - boxSize = layout(dag as any) - } - - const newNodes = dag.descendants().map((des) => ({ - id: des.data.id, + // Center horizontally + const xCenter = (fullSize ? fullWidth : width) / 2 - bbox.width / 2 - (width - fullWidth) / 2 + const newNodes = nodes.map((n) => ({ + id: n.id, position: { - x: des.x - ? // @ts-ignore - (des.data.offset ?? 0) + - // @ts-ignore - des.x + - (fullSize ? fullWidth : width) / 2 - - boxSize.width / 2 - - NODE.width / 2 - - (width - fullWidth) / 2 - : 0, - y: des.y || 0 + x: (positions.get(n.id)?.x ?? 0) + xCenter - NODE.width / 2, + y: positions.get(n.id)?.y ?? 0 } })) @@ -631,7 +591,6 @@ Object.values(graph.nodes).map((n) => ({ id: n.id, parentIds: n.parentIds, - offset: n.data.offset ?? 0, data: { assets: (n.data as any).assets } })) ) @@ -640,10 +599,7 @@ let assetNodesResult = $showAssets ? computeAssetNodes( newNodes.map((n) => ({ - data: { - assets: n.data?.assets as AssetWithAltAccessType[], - offset: n.data?.offset as number - }, + data: { assets: n.data?.assets as AssetWithAltAccessType[] }, id: n.id, position: n.position })) @@ -674,7 +630,6 @@ id: n.id, position: n.position, parentIds: n.parentIds, - offset: n.data?.offset ?? 0, data: { assets: (n.data as any)?.assets }, type: n.type })), @@ -1039,8 +994,8 @@ {#if multiSelectEnabled} - nodesWithOffset.some(n => n.id === id) + selectedNodes={selectionManager.selectedIds.filter((id) => + nodesWithOffset.some((n) => n.id === id) )} allNodes={nodesWithOffset as (Node & { type: string })[]} onDeleteSelected={() => onDeleteMultiple?.(resolvedModuleIds)} @@ -1061,7 +1016,12 @@ {@render leftHeader()}
{:else} - + n.type !== 'note') }} + > {#if multiSelectEnabled}
diff --git a/frontend/src/lib/components/graph/compoundLayout.ts b/frontend/src/lib/components/graph/compoundLayout.ts new file mode 100644 index 0000000000..4bf3ddebda --- /dev/null +++ b/frontend/src/lib/components/graph/compoundLayout.ts @@ -0,0 +1,555 @@ +import { sugiyama, dagStratify, coordCenter, decrossTwoLayer, decrossOpt } from 'd3-dag' +import { NODE } from './util' + +type LayoutNode = { + id: string + parentIds?: string[] +} + +type LayoutConstants = { + nodeWidth: number + nodeHeight: number + gapH: number + gapV: number +} + +type CompoundGroup = { + type: 'branch' | 'loop' + headId: string + endId: string + branches: { + labelId: string + innerIds: string[] + }[] +} + +type LayoutResult = { + positions: Map + bbox: { width: number; height: number } + contentMinX: number +} + +const LOOP_INDENT = 25 + +/** + * Detect compound groups from a flat list of node IDs. + * Uses ID naming conventions from graphBuilder: + * - BranchAll/BranchOne: node X has children X-branch-N and X-end + * - ForLoop/WhileLoop: node X has child X-start and X-end + */ +function detectGroups( + nodeIds: Set, + allNodes: Map, + childrenMap: Map +): CompoundGroup[] { + const groups: CompoundGroup[] = [] + + for (const id of nodeIds) { + if (!id.endsWith('-end')) continue + + // Extract base ID (everything before -end) + const baseId = id.slice(0, -4) + if (!nodeIds.has(baseId)) continue + + const baseNode = allNodes.get(baseId) + if (!baseNode) continue + + // Check for branch pattern: probe for baseId-branch-N nodes directly + const branchLabelIds: string[] = [] + if (nodeIds.has(`${baseId}-branch-default`)) { + branchLabelIds.push(`${baseId}-branch-default`) + } + for (let i = 0; nodeIds.has(`${baseId}-branch-${i}`); i++) { + branchLabelIds.push(`${baseId}-branch-${i}`) + } + + // Check for loop pattern: baseId-start node + const hasStart = nodeIds.has(`${baseId}-start`) + + if (branchLabelIds.length > 0) { + // Branches are already in correct order: default first, then 0, 1, 2... + const branches = branchLabelIds.map((labelId) => ({ + labelId, + innerIds: findInnerIds(labelId, id, nodeIds, childrenMap) + })) + + groups.push({ type: 'branch', headId: baseId, endId: id, branches }) + } else if (hasStart) { + const innerIds = findInnerIds(`${baseId}-start`, id, nodeIds, childrenMap) + + groups.push({ + type: 'loop', + headId: baseId, + endId: id, + branches: [{ labelId: `${baseId}-start`, innerIds }] + }) + } + } + + return groups +} + +/** + * Find inner node IDs between a label/start node and an end node. + * These are nodes that are reachable from the label node but not including + * the label or end node themselves. + */ +function findInnerIds( + labelId: string, + endId: string, + nodeIds: Set, + childrenMap: Map +): string[] { + const inner: string[] = [] + const visited = new Set() + + // BFS from label to find all reachable nodes before end + const queue = [labelId] + visited.add(labelId) + visited.add(endId) // Don't traverse past end + + let qi = 0 + while (qi < queue.length) { + const current = queue[qi++] + const kids = childrenMap.get(current) ?? [] + for (const kid of kids) { + if (visited.has(kid)) continue + if (!nodeIds.has(kid)) continue + visited.add(kid) + inner.push(kid) + queue.push(kid) + } + } + + return inner +} + +/** + * Run sugiyama layout on a set of nodes with parent relationships. + * Returns x,y positions for each node, centered at x=0. + */ +function runSugiyama( + nodes: { id: string; parentIds?: string[] }[], + constants: LayoutConstants, + nodeSizes?: Map +): { positions: Map; width: number; height: number } { + if (nodes.length === 0) { + return { positions: new Map(), width: 0, height: 0 } + } + + if (nodes.length === 1) { + const pos = new Map() + pos.set(nodes[0].id, { x: 0, y: 0 }) + const w = nodeSizes?.get(nodes[0].id)?.width ?? constants.nodeWidth + const h = nodeSizes?.get(nodes[0].id)?.height ?? constants.nodeHeight + return { positions: pos, width: w, height: h } + } + + const nodeIdSet = new Set(nodes.map((n) => n.id)) + const dagNodes = nodes.map((n) => ({ + id: n.id, + parentIds: (n.parentIds ?? []).filter((pid) => nodeIdSet.has(pid)) + })) + + const dag = dagStratify().id(({ id }: { id: string }) => id)(dagNodes) + + let boxSize: { width: number; height: number } + try { + const layout = sugiyama() + .decross(nodes.length > 20 ? decrossTwoLayer() : decrossOpt()) + .coord(coordCenter()) + .nodeSize((d: any) => { + const nodeId = d?.data?.id ?? '' + const size = nodeSizes?.get(nodeId) + const w = size?.width ?? constants.nodeWidth + const h = size?.height ?? constants.nodeHeight + return [w + constants.gapH, h + constants.gapV] as readonly [number, number] + }) + boxSize = layout(dag as any) as any + } catch (e) { + console.debug('[compoundLayout] decrossOpt failed, falling back to decrossTwoLayer:', e) + const layout = sugiyama() + .decross(decrossTwoLayer()) + .coord(coordCenter()) + .nodeSize((d: any) => { + const nodeId = d?.data?.id ?? '' + const size = nodeSizes?.get(nodeId) + const h = size?.height ?? constants.nodeHeight + return [constants.nodeWidth + constants.gapH, h + constants.gapV] + }) + boxSize = layout(dag as any) as any + } + + const positions = new Map() + for (const desc of dag.descendants()) { + const nodeId = desc.data.id + // sugiyama returns CENTER positions; convert to TOP by subtracting half the node's allocated height + const h = nodeSizes?.get(nodeId)?.height ?? constants.nodeHeight + const rawY = (desc as any).y ?? 0 + positions.set(nodeId, { + x: (desc as any).x ?? 0, + y: rawY - (h + constants.gapV) / 2 + }) + } + + // Normalize y so minimum = 0 + let minY = Infinity + for (const pos of positions.values()) { + minY = Math.min(minY, pos.y) + } + if (minY !== Infinity && minY !== 0) { + for (const pos of positions.values()) { + pos.y -= minY + } + } + + // Normalize x so center of bbox = 0 (important for nested branch placement) + let minX = Infinity + let maxX = -Infinity + for (const pos of positions.values()) { + minX = Math.min(minX, pos.x) + maxX = Math.max(maxX, pos.x) + } + if (minX !== Infinity) { + const centerX = (minX + maxX) / 2 + for (const pos of positions.values()) { + pos.x -= centerX + } + } + + return { positions, width: boxSize.width, height: boxSize.height } +} + +/** + * Recursive compound layout. + * + * 1. Detect compound groups at this level + * 2. For each group, recursively lay out each branch + * 3. Compute wrapper bbox for each group + * 4. Replace group nodes with a single wrapper pseudo-node + * 5. Run sugiyama on the simplified graph + * 6. Expand wrapper positions back to absolute positions + */ +const MAX_RECURSION_DEPTH = 50 + +function layoutLevel( + nodeIds: string[], + allNodes: Map, + constants: LayoutConstants, + childrenMap: Map, + depth: number = 0 +): LayoutResult { + const positions = new Map() + const nodeIdSet = new Set(nodeIds) + + if (nodeIds.length === 0) { + return { + positions, + bbox: { width: constants.nodeWidth, height: 0 }, + contentMinX: 0 + } + } + + if (depth >= MAX_RECURSION_DEPTH) { + console.warn('[compoundLayout] Max recursion depth reached, falling back to flat layout') + const flatNodes = nodeIds.map((id) => { + const n = allNodes.get(id)! + return { id, parentIds: (n.parentIds ?? []).filter((pid) => nodeIdSet.has(pid)) } + }) + const result = runSugiyama(flatNodes, constants) + for (const [id, pos] of result.positions) { + positions.set(id, pos) + } + return { positions, bbox: { width: result.width, height: result.height }, contentMinX: 0 } + } + + // Step 1: detect compound groups at this level + const groups = detectGroups(nodeIdSet, allNodes, childrenMap) + + // First pass: quick set of ALL group-owned inner IDs (just for filtering nested heads) + const allGroupOwnedIds = new Set() + for (const group of groups) { + if (!nodeIdSet.has(group.headId)) continue + for (const branch of group.branches) { + for (const innerId of branch.innerIds) { + allGroupOwnedIds.add(innerId) + } + } + } + + // Filter to top-level groups (head not owned by another group) + const topLevelGroups = groups.filter( + (g) => nodeIdSet.has(g.headId) && !allGroupOwnedIds.has(g.headId) + ) + + // Build final groupOwnedIds and groupByHeadId from top-level only + const groupOwnedIds = new Set() + const groupByHeadId = new Map() + for (const group of topLevelGroups) { + groupByHeadId.set(group.headId, group) + groupOwnedIds.add(group.endId) + for (const branch of group.branches) { + groupOwnedIds.add(branch.labelId) + for (const innerId of branch.innerIds) { + groupOwnedIds.add(innerId) + } + } + } + + // Step 2-3: Recursively lay out each group and compute wrapper sizes + type GroupLayout = { + group: CompoundGroup + branchLayouts: { + labelId: string + result: LayoutResult + bbox: { width: number; height: number } + }[] + branchWidths: number[] + totalWidth: number + wrapperWidth: number + wrapperHeight: number + maxBranchHeight: number + } + + const groupLayouts = new Map() + const wrapperSizes = new Map() + + for (const group of topLevelGroups) { + const branchLayouts: GroupLayout['branchLayouts'] = [] + const isBranch = group.type === 'branch' + + for (const branch of group.branches) { + const branchNodeIds = [branch.labelId, ...branch.innerIds] + + // Find sub-groups within this branch + const result = layoutLevel(branchNodeIds, allNodes, constants, childrenMap, depth + 1) + + branchLayouts.push({ + labelId: branch.labelId, + result, + bbox: result.bbox + }) + } + + // Compute wrapper dimensions + let wrapperWidth: number + let wrapperHeight: number + let branchWidths: number[] = [] + let totalWidth = 0 + let maxBranchHeight = 0 + const rowHeight = constants.nodeHeight + constants.gapV + + if (isBranch) { + // Place branches side by side horizontally + branchWidths = branchLayouts.map((bl) => Math.max(bl.bbox.width, constants.nodeWidth)) + const gaps = Math.max(0, branchWidths.length - 1) * constants.gapH + totalWidth = branchWidths.reduce((s, w) => s + w, 0) + gaps + wrapperWidth = Math.max(totalWidth, constants.nodeWidth) + + maxBranchHeight = Math.max(0, ...branchLayouts.map((bl) => bl.bbox.height)) + // head row + branch content + end row + wrapperHeight = rowHeight + maxBranchHeight + rowHeight + } else { + // Loop: body is indented + const bodyWidth = branchLayouts[0]?.bbox.width ?? constants.nodeWidth + const bodyHeight = branchLayouts[0]?.bbox.height ?? 0 + wrapperWidth = Math.max(bodyWidth + LOOP_INDENT * 2, constants.nodeWidth) + // head row + start row + body + end row + wrapperHeight = rowHeight + bodyHeight + rowHeight + } + groupLayouts.set(group.headId, { + group, + branchLayouts, + branchWidths, + totalWidth, + wrapperWidth, + wrapperHeight, + maxBranchHeight + }) + wrapperSizes.set(group.headId, { width: wrapperWidth, height: wrapperHeight }) + } + + // Step 4: Build flattened node list for sugiyama + // Merge into a single pass: create flatNode and compute final parentIds with end→head redirection + const endToHead = new Map() + for (const group of topLevelGroups) { + endToHead.set(group.endId, group.headId) + } + + const flatNodes: { id: string; parentIds?: string[] }[] = [] + for (const nid of nodeIds) { + if (groupOwnedIds.has(nid)) continue + + const originalNode = allNodes.get(nid)! + const seen = new Set() + const newParents: string[] = [] + for (const pid of originalNode.parentIds ?? []) { + if (!nodeIdSet.has(pid)) continue + const resolved = endToHead.get(pid) ?? (groupOwnedIds.has(pid) ? undefined : pid) + if (resolved && !seen.has(resolved)) { + seen.add(resolved) + newParents.push(resolved) + } + } + flatNodes.push({ id: nid, parentIds: newParents }) + } + + // Step 5: Run sugiyama on flattened nodes + const sugResult = runSugiyama(flatNodes, constants, wrapperSizes) + + // Step 6: Resolve absolute positions + // First, set positions for regular (non-group) nodes + for (const [nid, pos] of sugResult.positions) { + if (groupByHeadId.has(nid)) continue // Handle groups separately + positions.set(nid, { x: pos.x, y: pos.y }) + } + + // Now expand group wrappers into absolute positions + for (const [headId, gl] of groupLayouts) { + const wrapperPos = sugResult.positions.get(headId) + if (!wrapperPos) continue + + const rowHeight = constants.nodeHeight + constants.gapV + const isBranch = gl.group.type === 'branch' + + // Position the head node at the top-center of the wrapper + positions.set(headId, { x: wrapperPos.x, y: wrapperPos.y }) + + if (isBranch) { + // Reuse cached branchWidths and totalWidth + let currentX = wrapperPos.x - gl.totalWidth / 2 + + for (let bi = 0; bi < gl.branchLayouts.length; bi++) { + const bl = gl.branchLayouts[bi] + const bw = gl.branchWidths[bi] + const branchCenterX = currentX + bw / 2 + + // Offset all branch positions relative to the branch center + for (const [innerNodeId, innerPos] of bl.result.positions) { + positions.set(innerNodeId, { + x: branchCenterX + innerPos.x, + y: wrapperPos.y + rowHeight + innerPos.y + }) + } + + currentX += bw + constants.gapH + } + + // Position end node below all branches + const maxBranchHeight = gl.maxBranchHeight + positions.set(gl.group.endId, { + x: wrapperPos.x, + y: wrapperPos.y + rowHeight + maxBranchHeight + constants.gapV + }) + } else { + // Loop: position start, body, and end + const bl = gl.branchLayouts[0] + if (bl) { + // Position body nodes with indent + for (const [innerNodeId, innerPos] of bl.result.positions) { + positions.set(innerNodeId, { + x: wrapperPos.x + LOOP_INDENT + innerPos.x, + y: wrapperPos.y + rowHeight + innerPos.y + }) + } + } + + // Position end node below body + const bodyHeight = bl?.bbox.height ?? 0 + positions.set(gl.group.endId, { + x: wrapperPos.x, + y: wrapperPos.y + rowHeight + bodyHeight + constants.gapV + }) + } + } + + // Compute overall bbox (nodes + group wrapper extents) + let minX = Infinity + let maxX = -Infinity + let minY = Infinity + let maxY = -Infinity + for (const pos of positions.values()) { + minX = Math.min(minX, pos.x - constants.nodeWidth / 2) + maxX = Math.max(maxX, pos.x + constants.nodeWidth / 2) + minY = Math.min(minY, pos.y) + maxY = Math.max(maxY, pos.y + constants.nodeHeight) + } + // Account for group wrapper extents in bbox (e.g. LOOP_INDENT makes wrappers wider than nodes) + for (const [headId, gl] of groupLayouts) { + const pos = sugResult.positions.get(headId) + if (!pos) continue + minX = Math.min(minX, pos.x - gl.wrapperWidth / 2) + maxX = Math.max(maxX, pos.x + gl.wrapperWidth / 2) + maxY = Math.max(maxY, pos.y + gl.wrapperHeight) + } + + const contentMinX = minX === Infinity ? 0 : minX + + const bboxWidth = maxX - minX + const bboxHeight = maxY - (positions.size > 0 ? minY : 0) + + const finalBbox = { + width: Math.max(bboxWidth, constants.nodeWidth), + height: Math.max(bboxHeight, 0) + } + return { positions, bbox: finalBbox, contentMinX } +} + +/** + * Main entry point for compound layout. + * + * Takes the flat list of nodes and edges from graphBuilder and produces + * absolute positions that account for compound structure (branches, loops). + */ +export function compoundLayout( + nodes: { id: string; parentIds?: string[] }[], + constants?: Partial +): LayoutResult { + const c: LayoutConstants = { + nodeWidth: constants?.nodeWidth ?? NODE.width, + nodeHeight: constants?.nodeHeight ?? NODE.height, + gapH: constants?.gapH ?? NODE.gap.horizontal, + gapV: constants?.gapV ?? NODE.gap.vertical + } + + // Build node map + const allNodes = new Map() + for (const n of nodes) { + allNodes.set(n.id, n) + } + + // Build children map once (reverse of parentIds), shared across all recursion levels + const childrenMap = new Map() + for (const [nid, node] of allNodes) { + for (const pid of node.parentIds ?? []) { + if (!childrenMap.has(pid)) childrenMap.set(pid, []) + childrenMap.get(pid)!.push(nid) + } + } + + const nodeIds = nodes.map((n) => n.id) + const result = layoutLevel(nodeIds, allNodes, c, childrenMap) + + // Shift positions so minX=0 (left-aligned). + // FlowGraphV2 centers with: xCenter = viewport/2 - bbox.width/2 + // which assumes positions start at x=0. + if (result.positions.size > 0) { + const minX = result.contentMinX + if (minX !== 0 && minX !== Infinity) { + for (const pos of result.positions.values()) { + pos.x -= minX + } + } + } + + // Check for missing nodes + const missing = nodes.filter((n) => !result.positions.has(n.id)) + if (missing.length > 0) { + console.warn( + '[compoundLayout] MISSING positions for:', + missing.map((n) => n.id) + ) + } + + return result +} diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index 37b3103f5f..1d01b2e252 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -88,9 +88,7 @@ export function buildPrefix(prefix: string | undefined, id: string): string { export type NodeLayout = { id: string parentIds?: string[] - data: { - offset?: number - } + data: {} selectable?: boolean } & FlowNode @@ -138,7 +136,6 @@ export type InputN = { export type ModuleN = { type: 'module' data: { - offset: number module: FlowModule id: string parentIds: string[] @@ -157,7 +154,6 @@ export type ModuleN = { export type BranchAllStartN = { type: 'branchAllStart' data: { - offset: number label: string id: string branchIndex: number @@ -171,7 +167,6 @@ export type BranchAllStartN = { export type BranchAllEndN = { type: 'branchAllEnd' data: { - offset: number id: string eventHandlers: GraphEventHandlers flowModuleState: GraphModuleState | undefined @@ -181,7 +176,6 @@ export type BranchAllEndN = { export type ForLoopEndN = { type: 'forLoopEnd' data: { - offset: number id: string eventHandlers: GraphEventHandlers simplifiedTriggerView: boolean @@ -192,7 +186,6 @@ export type ForLoopEndN = { export type ForLoopStartN = { type: 'forLoopStart' data: { - offset: number id: string eventHandlers: GraphEventHandlers flowModuleState: GraphModuleState | undefined @@ -217,22 +210,18 @@ export type ResultN = { export type WhileLoopStartN = { type: 'whileLoopStart' data: { - offset: number eventHandlers: GraphEventHandlers } } export type WhileLoopEndN = { type: 'whileLoopEnd' - data: { - offset: number - } + data: {} } export type BranchOneStartN = { type: 'branchOneStart' data: { - offset: number id: string eventHandlers: GraphEventHandlers flowModuleState: GraphModuleState | undefined @@ -248,7 +237,6 @@ export type BranchOneStartN = { export type BranchOneEndN = { type: 'branchOneEnd' data: { - offset: number id: string eventHandlers: GraphEventHandlers flowModuleState: GraphModuleState | undefined @@ -258,7 +246,6 @@ export type BranchOneEndN = { export type SubflowBoundN = { type: 'subflowBound' data: { - offset: number id: string eventHandlers: GraphEventHandlers label: string @@ -271,7 +258,6 @@ export type SubflowBoundN = { export type NoBranchN = { type: 'noBranch' data: { - offset: number id: string eventHandlers: GraphEventHandlers flowModuleState: GraphModuleState | undefined @@ -416,7 +402,7 @@ export function graphBuilder( const nodes: NodeLayout[] = [] const edges: Edge[] = [] - function addNode(module: FlowModule, offset: number) { + function addNode(module: FlowModule) { const duplicated = nodes.find((n) => n.id === module.id) if (duplicated) { console.log('Duplicated node detected: ', module, duplicated) @@ -426,7 +412,6 @@ export function graphBuilder( nodes.push({ id: module.id, data: { - offset: offset, module: module, id: module.id, parentIds: [], @@ -611,7 +596,6 @@ export function graphBuilder( nextNode: NodeLayout | undefined, simplifiedTriggerView: boolean, prefix: string | undefined, - currentOffset = 0, disableMoveIds: string[] = [], parentIndex?: string ) { @@ -646,13 +630,12 @@ export function graphBuilder( if (module.value.type === 'branchall') { // Start - addNode(module, currentOffset) + addNode(module) // "Collect result of each branch" node const endNode: NodeLayout = { id: `${module.id}-end`, data: { - offset: currentOffset, id: module.id, eventHandlers: eventHandlers, flowModuleState: extra.flowModuleStates?.[module.id] @@ -667,7 +650,6 @@ export function graphBuilder( const startNode: NodeLayout = { id: `${module.id}-branch-0`, data: { - offset: currentOffset, id: module.id, branchIndex: -1, eventHandlers: eventHandlers, @@ -693,7 +675,6 @@ export function graphBuilder( const startNode: NodeLayout = { id: `${module.id}-branch-${branchIndex}`, data: { - offset: currentOffset, label: defaultIfEmptyString(branch.summary, `Branch ${branchIndex + 1}`), id: module.id, branchIndex: branchIndex, @@ -724,7 +705,6 @@ export function graphBuilder( endNode, false, prefix, - currentOffset, localDisableMoveIds, parentIndex ? `${parentIndex}-${index}-${branchIndex}` : `${index}-${branchIndex}` ) @@ -734,13 +714,12 @@ export function graphBuilder( previousId = endNode.id } else if (module.value.type === 'forloopflow') { if (!simplifiedTriggerView) { - addNode(module, currentOffset) + addNode(module) } const startNode: NodeLayout = { id: `${module.id}-start`, data: { - offset: currentOffset + 25, id: module.id, module: module, simplifiedTriggerView, @@ -765,7 +744,6 @@ export function graphBuilder( const endNode: NodeLayout = { id: `${module.id}-end`, data: { - offset: currentOffset, id: module.id, eventHandlers: eventHandlers, simplifiedTriggerView, @@ -786,7 +764,6 @@ export function graphBuilder( endNode, false, prefix, - currentOffset + 25, localDisableMoveIds, parentIndex ? `${parentIndex}-${index}-${selectedIterIndex ?? '?'}` @@ -795,12 +772,11 @@ export function graphBuilder( previousId = endNode.id } else if (module.value.type === 'whileloopflow') { - addNode(module, currentOffset) + addNode(module) const startNode: NodeLayout = { id: `${module.id}-start`, data: { - offset: currentOffset + 25, eventHandlers: eventHandlers }, type: 'whileLoopStart' @@ -811,7 +787,7 @@ export function graphBuilder( const endNode: NodeLayout = { id: `${module.id}-end`, - data: { offset: currentOffset, ...extra }, + data: { ...extra }, type: 'whileLoopEnd' } @@ -827,7 +803,6 @@ export function graphBuilder( endNode, false, prefix, - currentOffset + 25, localDisableMoveIds, parentIndex ? `${parentIndex}-${index}-${selectedIterIndex ?? '?'}` @@ -836,12 +811,11 @@ export function graphBuilder( previousId = endNode.id } else if (module.value.type === 'branchone') { - addNode(module, currentOffset) + addNode(module) const endNode: NodeLayout = { id: `${module.id}-end`, data: { - offset: currentOffset, eventHandlers: eventHandlers, flowModuleState: extra.flowModuleStates?.[module.id], id: module.id @@ -854,7 +828,7 @@ export function graphBuilder( // const defaultBranch: NodeLayout = { // id: `${module.id}-default`, // data: { - // offset: currentOffset, + // offset: 0, // label: 'Default', // id: module.id, // branchIndex: -1, @@ -868,7 +842,6 @@ export function graphBuilder( const defaultBranch: NodeLayout = { id: `${module.id}-branch-default`, data: { - offset: currentOffset, label: 'Default', id: module.id, branchIndex: -1, @@ -895,7 +868,6 @@ export function graphBuilder( endNode, false, prefix, - currentOffset, localDisableMoveIds, parentIndex ? `${parentIndex}-${index}` : index.toString() ) @@ -906,7 +878,6 @@ export function graphBuilder( const startNode: NodeLayout = { id: `${module.id}-branch-${branchIndex}`, data: { - offset: currentOffset, label: defaultIfEmptyString(branch.summary, 'Branch ' + (branchIndex + 1)), preLabel: branch.summary ? '' : branch.expr, id: module.id, @@ -933,7 +904,6 @@ export function graphBuilder( endNode, false, prefix, - currentOffset, localDisableMoveIds, parentIndex ? `${parentIndex}-${index}` : index.toString() ) @@ -951,7 +921,6 @@ export function graphBuilder( const startNode: NodeLayout = { id: startId, data: { - offset: currentOffset, label: `Start of subflow ${idWithoutPrefix}`, id: startId, subflowId: module.id, @@ -980,7 +949,6 @@ export function graphBuilder( const endNode: NodeLayout = { id: endId, data: { - offset: currentOffset, label: `End of subflow ${idWithoutPrefix}`, id: endId, subflowId: module.id, @@ -1000,13 +968,12 @@ export function graphBuilder( endNode, false, buildPrefix(prefix, module['oid'] ?? module.id), - currentOffset, localDisableMoveIds ) previousId = endNode.id } else { - addNode(module, currentOffset) + addNode(module) previousId = module.id } } @@ -1047,19 +1014,19 @@ export function graphBuilder( }) Object.entries(toAdd).forEach((x) => { - addNode({ ...failureModule, id: x[1] }, 0) + addNode({ ...failureModule, id: x[1] }) addEdge(x[0], x[1], undefined, undefined, { type: 'empty' }) }) } if (preprocessorModule) { - addNode(preprocessorModule, 0) + addNode(preprocessorModule) const id = JSON.parse(JSON.stringify(preprocessorModule.id)) addEdge(id, 'Input', undefined, undefined, { type: 'empty' }) } if (failureModule && !extra.flowModuleStates) { - addNode(failureModule, 0) + addNode(failureModule) } Object.keys(parents).forEach((key) => { diff --git a/frontend/src/lib/components/graph/noteEditor.svelte.ts b/frontend/src/lib/components/graph/noteEditor.svelte.ts index 4d59cfa480..e59e3be4f5 100644 --- a/frontend/src/lib/components/graph/noteEditor.svelte.ts +++ b/frontend/src/lib/components/graph/noteEditor.svelte.ts @@ -219,7 +219,7 @@ export class NoteEditor { /** * Clean up group notes using DAG path completion */ - cleanupGroupNotes(flowNodes: { id: string; parentIds?: string[]; offset?: number }[]): void { + cleanupGroupNotes(flowNodes: { id: string; parentIds?: string[] }[]): void { if (!this.isAvailable()) { return } diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts index d84817a693..3350e07ce8 100644 --- a/frontend/src/lib/components/graph/noteUtils.svelte.ts +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -14,7 +14,6 @@ export type NodeDep = { position: { x: number; y: number } data?: { assets?: AssetWithAltAccessType[] } parentIds?: string[] - offset?: number type?: string } @@ -207,7 +206,6 @@ function calculateGroupNoteLayout( nodes.map((n) => ({ id: n.id, position: n.position, - data: { offset: n.offset ?? 0 }, type: n.type ?? '' })) ) @@ -337,7 +335,6 @@ export function computeNoteNodes( return { ...n, data: origNode?.data, - offset: origNode?.offset, type: origNode?.type } }) diff --git a/frontend/src/lib/components/graph/renderers/nodes/AIToolNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AIToolNode.svelte index 242f9fac61..28e9d58fa4 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AIToolNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AIToolNode.svelte @@ -175,7 +175,7 @@ ? inputToolWidth + inputToolXGap : isLastRow && tools.length % 2 === 1 ? (ROW_WIDTH - inputToolWidth) / 2 - : 0) + node.data.offset, + : 0), y: baseOffset + rowOffset * @@ -207,7 +207,7 @@ parentId: node.id, width: NEW_TOOL_NODE_WIDTH, position: { - x: (ROW_WIDTH - NEW_TOOL_NODE_WIDTH) / 2 + node.data.offset, + x: (ROW_WIDTH - NEW_TOOL_NODE_WIDTH) / 2, y: baseOffset + rowOffset }, selectable: false diff --git a/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte index 0db4e92f3a..aec1fb03f8 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/AssetNode.svelte @@ -11,7 +11,7 @@ let computeAssetNodesCache: [NodeDep[], ReturnType] | undefined type NodeDep = { - data: object & { assets?: AssetWithAltAccessType[] | undefined; offset?: number } + data: object & { assets?: AssetWithAltAccessType[] | undefined } id: string position: { x: number; y: number } } @@ -78,14 +78,13 @@ width: inputAssetWidth, position: { x: - (node.data.offset ?? 0) + - (displayedInputAssets.length === 1 + displayedInputAssets.length === 1 ? (NODE.width - inputAssetWidth) / 2 - 10 // Ensure we see the edge : (inputAssetWidth + inputAssetXGap) * (i - displayedInputAssets.length / 2) + (NODE.width + inputAssetXGap) / 2 + (overflowedInputAssets.length ? (-ASSETS_OVERFLOWED_NODE_WIDTH - inputAssetXGap) / 2 - : 0)), + : 0), y: READ_ASSET_Y_OFFSET }, selectable: false @@ -116,14 +115,13 @@ width: outputAssetWidth, position: { x: - (node.data.offset ?? 0) + - (displayedOutputAssets.length === 1 + displayedOutputAssets.length === 1 ? (NODE.width - outputAssetWidth) / 2 - 10 // Ensure we see the edge : (outputAssetWidth + outputAssetXGap) * (i - displayedOutputAssets.length / 2) + (NODE.width + outputAssetXGap) / 2 + (overflowedOutputAssets.length ? (-ASSETS_OVERFLOWED_NODE_WIDTH - outputAssetXGap) / 2 - : 0)), + : 0), y: WRITE_ASSET_Y_OFFSET }, selectable: false @@ -157,7 +155,7 @@ parentId: node.id, width: ASSETS_OVERFLOWED_NODE_WIDTH, position: { - x: (node.data.offset ?? 0) + MAX_ASSET_ROW_WIDTH - ASSETS_OVERFLOWED_NODE_WIDTH - 14, + x: MAX_ASSET_ROW_WIDTH - ASSETS_OVERFLOWED_NODE_WIDTH - 14, y: READ_ASSET_Y_OFFSET } } satisfies Node & AssetsOverflowedN) @@ -176,7 +174,7 @@ parentId: node.id, width: ASSETS_OVERFLOWED_NODE_WIDTH, position: { - x: (node.data.offset ?? 0) + MAX_ASSET_ROW_WIDTH - ASSETS_OVERFLOWED_NODE_WIDTH - 14, + x: MAX_ASSET_ROW_WIDTH - ASSETS_OVERFLOWED_NODE_WIDTH - 14, y: WRITE_ASSET_Y_OFFSET } } satisfies Node & AssetsOverflowedN) diff --git a/frontend/src/lib/components/graph/renderers/nodes/BranchAllEndNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/BranchAllEndNode.svelte index 27058af860..3b7dc461b8 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/BranchAllEndNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/BranchAllEndNode.svelte @@ -13,7 +13,7 @@ const { selectionManager } = getGraphContext() - + {#snippet children({ darkMode })} - + {#snippet children({ darkMode })} - + {#snippet children({ darkMode })} - + {#snippet children({ darkMode })} {#if data.simplifiedTriggerView} - + {#snippet children({ darkMode })} - + {#snippet children({ darkMode })} = 0 ? (state?.selectedForloopIndex ?? 0) + 1 @@ -136,15 +136,18 @@ onEditInput={data.eventHandlers.editInput} flowJob={data.flowJob} isOwner={data.isOwner} - maximizeSubflow={data.module.value.type == 'flow' && 'path' in data.module.value + maximizeSubflow={data.module?.value?.type == 'flow' && 'path' in data.module.value ? () => { - data.eventHandlers.expandSubflow(data.id, data.module.value['path']) + const path = data.module?.value && 'path' in data.module.value ? data.module.value['path'] as string : undefined + if (path) { + data.eventHandlers.expandSubflow(data.id, path) + } } : undefined} />
- {#if (data.module.value.type === 'branchall' || data.module.value.type === 'branchone') && data.insertable} + {#if (data.module?.value?.type === 'branchall' || data.module?.value?.type === 'branchone') && data.insertable}
+
+ updateIncludeType('workspaceDependencies', e.detail)} + options={{ right: 'Workspace dependencies' }} + /> +
From 31fc213933248d96a9520df921cd196a42b1075e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 10 Mar 2026 12:38:43 +0000 Subject: [PATCH 56/57] fix: handle missing schema in RunnableByPath during wmill.d.ts generation (#8300) --- frontend/src/lib/components/raw_apps/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts index 077a9ddad8..06264d9c5f 100644 --- a/frontend/src/lib/components/raw_apps/utils.ts +++ b/frontend/src/lib/components/raw_apps/utils.ts @@ -107,7 +107,11 @@ function hiddenRunnableToTsType(runnable: Runnable) { return '{}' } } else if (isRunnableByPath(runnable)) { - return schemaToTsType(removeStaticFields(runnable?.schema, runnable?.fields ?? {})) + if (runnable?.schema) { + return schemaToTsType(removeStaticFields(runnable.schema, runnable?.fields ?? {})) + } else { + return '{}' + } } else { return '{}' } From 932eb0dc4a2ae09b381bd03210bd17223f3dc258 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 10 Mar 2026 13:38:59 +0100 Subject: [PATCH 57/57] fix: show meaningful error messages in database manager schema fetch (#8296) Co-authored-by: Claude Opus 4.6 --- .../components/apps/components/display/dbtable/metadata.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts index a44cab88fa..4d970b2303 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts @@ -16,6 +16,7 @@ import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores' import { stringifyGraphqlSchema, stringifySchema } from '$lib/components/copilot/lib' import type { DbType } from '$lib/components/dbTypes' import { getDatabaseArg } from '$lib/components/dbOps' +import { sendUserToast } from '$lib/toast' export async function loadTableMetaData( input: DbInput, @@ -95,7 +96,8 @@ export async function loadAllTablesMetaData( return map } catch (e) { - throw new Error('Error loading all tables metadata: ' + e) + sendUserToast('Error loading tables metadata: ' + e, 'error') + throw e } }