From 44105679774dd76c826b63fd5a09dfd9747926ff Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 4 Dec 2025 08:34:32 +0100 Subject: [PATCH 01/25] chore(cli): remove unnecessary indentations in claude.md #7292 --- cli/src/commands/init/init.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index 24b2fbfeb6..d4e13c65e6 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -262,15 +262,13 @@ async function initAction(opts: InitOptions) { await Deno.writeTextFile( "CLAUDE.md", ` - # Claude +You are a helpful assistant that can help with Windmill scripts and flows creation. - You are a helpful assistant that can help with Windmill scripts and flows creation. +## Script Guidance +${scriptGuidanceContent} - ## Script Guidance - ${scriptGuidanceContent} - - ## Flow Guidance - ${flowGuidanceContent} +## Flow Guidance +${flowGuidanceContent} ` ); log.info(colors.green("Created CLAUDE.md")); From fcde732485a90213f4be91a481b4f697ee4b9661 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 4 Dec 2025 08:22:28 +0000 Subject: [PATCH 02/25] nit warning concurrency limit for deps --- backend/windmill-worker/src/worker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0b40386e3d..31275acbc1 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2826,7 +2826,7 @@ pub async fn handle_queued_job( } #[cfg(not(feature = "enterprise"))] - if job.concurrent_limit.is_some() { + if job.concurrent_limit.is_some() && !job.kind.is_dependency() { logs.push_str("---\n"); logs.push_str("WARNING: This job has concurrency limits enabled. Concurrency limits are an EE feature and the setting is ignored.\n"); logs.push_str("---\n"); From e9f13065bfedd9af84da58b161349f8e79e72b2a Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Thu, 4 Dec 2025 09:50:07 +0000 Subject: [PATCH 03/25] fix: show related job when deployment is in progress (#7294) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: show related job when deployment is in progress - Added job_id column to deployment_metadata table to track current deployment jobs - Updated backend to store job_id when creating dependency jobs for scripts and flows - Modified deployment status API endpoints to include job_id in responses - Updated frontend to display clickable job link in "Deployment in progress" badge - Added OpenAPI schema updates for new job_id field Resolves #7293 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: windmill-internal-app[bot] * update --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: windmill-internal-app[bot] Co-authored-by: Ruben Fiszel --- ...b8f0e9012c265e150afd4aa41972bf7334ba2.json | 17 ++++++ ...1c42aefee683e14cc738a809ecca17370e6ee.json | 29 +++++++++++ ...47a9bbaa6967e36777454746b8817d68219a5.json | 35 +++++++++++++ ...83f97076b2cbe2f617f7af39550a0db157b2b.json | 17 ++++++ ...116cc87ed06c49dcca04360250ddfd89d7ab3.json | 23 -------- ...d3d84fd76e44f9240f4292c5c1208096b8563.json | 29 ----------- ...add_job_id_to_deployment_metadata.down.sql | 2 + ...0_add_job_id_to_deployment_metadata.up.sql | 2 + backend/windmill-api/openapi.yaml | 6 +++ backend/windmill-api/src/flows.rs | 52 +++++++++++++++++-- backend/windmill-api/src/scripts.rs | 38 +++++++++++--- backend/windmill-queue/src/jobs.rs | 2 +- .../(logged)/flows/get/[...path]/+page.svelte | 11 ++++ .../scripts/get/[...hash]/+page.svelte | 11 ++++ 14 files changed, 209 insertions(+), 65 deletions(-) create mode 100644 backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json create mode 100644 backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json create mode 100644 backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json create mode 100644 backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json delete mode 100644 backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json delete mode 100644 backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json create mode 100644 backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql create mode 100644 backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql diff --git a/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json b/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json new file mode 100644 index 0000000000..5dd30c1492 --- /dev/null +++ b/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL\n DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2" +} diff --git a/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json b/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json new file mode 100644 index 0000000000..56c538d440 --- /dev/null +++ b/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT f.lock_error_logs, dm.job_id\n FROM flow f\n LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version\n AND f.workspace_id = dm.workspace_id AND f.path = dm.path\n WHERE f.path = $1 AND f.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock_error_logs", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee" +} diff --git a/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json b/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json new file mode 100644 index 0000000000..31187af03a --- /dev/null +++ b/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.lock, s.lock_error_logs, dm.job_id\n FROM script s\n LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id\n WHERE s.hash = $1 AND s.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "lock_error_logs", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5" +} diff --git a/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json b/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json new file mode 100644 index 0000000000..571a374b96 --- /dev/null +++ b/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL\n DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b" +} diff --git a/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json b/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json deleted file mode 100644 index 739184045f..0000000000 --- a/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "lock_error_logs", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3" -} diff --git a/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json b/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json deleted file mode 100644 index e8cba862fe..0000000000 --- a/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "lock", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "lock_error_logs", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563" -} diff --git a/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql new file mode 100644 index 0000000000..0b04137182 --- /dev/null +++ b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql @@ -0,0 +1,2 @@ +-- Remove job_id column from deployment_metadata table +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS job_id; diff --git a/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql new file mode 100644 index 0000000000..8c54c702cd --- /dev/null +++ b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql @@ -0,0 +1,2 @@ +-- Add job_id column to deployment_metadata table to track the current deployment job +ALTER TABLE deployment_metadata ADD COLUMN IF NOT EXISTS job_id UUID; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 639b270bd0..131e8da69f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -5950,6 +5950,9 @@ paths: type: string lock_error_logs: type: string + job_id: + type: string + format: uuid /w/{workspace}/jobs/list_selected_job_groups: # We use post because sending a huge array as a query param can produce @@ -6836,6 +6839,9 @@ paths: properties: lock_error_logs: type: string + job_id: + type: string + format: uuid /w/{workspace}/flows/get_triggers_count/{path}: get: diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 88d4f08fe9..7495dbd75b 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -577,6 +577,20 @@ async fn create_flow( .execute(&mut *new_tx) .await?; + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await?; + new_tx.commit().await?; webhook.send_message( w_id.clone(), @@ -1115,6 +1129,25 @@ async fn update_flow( )) })?; + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating deployment_metadata with job_id: {e:#}" + )) + })?; + if let Some(old_dep_job) = old_dep_job { sqlx::query!( "UPDATE v2_job_queue SET @@ -1154,9 +1187,10 @@ async fn list_tokens( list_tokens_internal(&db, &w_id, &path, true).await } -#[derive(FromRow, Serialize)] +#[derive(Serialize)] struct DeploymentStatus { lock_error_logs: Option, + job_id: Option, } async fn get_deployment_status( Extension(db): Extension, @@ -1164,9 +1198,12 @@ async fn get_deployment_status( ) -> JsonResult { let path = path.to_path(); let mut tx = db.begin().await?; - let status_o: Option = sqlx::query_as!( - DeploymentStatus, - "SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2", + let status_o = sqlx::query!( + "SELECT f.lock_error_logs, dm.job_id + FROM flow f + LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version + AND f.workspace_id = dm.workspace_id AND f.path = dm.path + WHERE f.path = $1 AND f.workspace_id = $2", path, w_id, ) @@ -1175,8 +1212,13 @@ async fn get_deployment_status( let status = not_found_if_none(status_o, "DeploymentStatus", path)?; + let deployment_status = DeploymentStatus { + lock_error_logs: status.lock_error_logs, + job_id: status.job_id, + }; + tx.commit().await?; - Ok(Json(status)) + Ok(Json(deployment_status)) } async fn get_flow_by_path( diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 7cbcf8cc03..f71d6e00ed 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -998,14 +998,14 @@ async fn create_script_internal<'c>( } let tx = PushIsolationLevel::Transaction(tx); - let (_, new_tx) = windmill_queue::push( + let (job_id, mut new_tx) = windmill_queue::push( &db, tx, &w_id, JobPayload::Dependencies { hash, language: ns.language, - path: ns.path, + path: ns.path.clone(), dedicated_worker: ns.dedicated_worker, }, windmill_queue::PushArgs::from(&args), @@ -1034,6 +1034,21 @@ async fn create_script_internal<'c>( None, ) .await?; + + // Store the job_id in deployment_metadata for this script deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + ns.path, + hash.0, + job_id + ) + .execute(&mut *new_tx) + .await?; + Ok((hash, new_tx, None)) } else { if codebase.is_none() { @@ -1750,19 +1765,22 @@ async fn raw_script_by_hash( Ok(r.script.content) } -#[derive(FromRow, Serialize)] +#[derive(Serialize)] struct DeploymentStatus { lock: Option, lock_error_logs: Option, + job_id: Option, } async fn get_deployment_status( Extension(db): Extension, Path((w_id, hash)): Path<(String, ScriptHash)>, ) -> JsonResult { let mut tx = db.begin().await?; - let status_o: Option = sqlx::query_as!( - DeploymentStatus, - "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2", + let status_o = sqlx::query!( + "SELECT s.lock, s.lock_error_logs, dm.job_id + FROM script s + LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id + WHERE s.hash = $1 AND s.workspace_id = $2", hash.0, w_id, ) @@ -1771,8 +1789,14 @@ async fn get_deployment_status( let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; + let deployment_status = DeploymentStatus { + lock: status.lock, + lock_error_logs: status.lock_error_logs, + job_id: status.job_id, + }; + tx.commit().await?; - Ok(Json(status)) + Ok(Json(deployment_status)) } pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 25d10784bb..059bfc69b8 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3040,7 +3040,7 @@ pub async fn concurrency_key( .await .map(|x| { if x.is_none() { - tracing::info!("No concurrency key found for job {id}, defaulting to empty string"); + tracing::info!("No concurrency key found for job {id}"); } return x; }) diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 55c0b4f775..9a243495c8 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -75,6 +75,7 @@ let inputSelected: 'saved' | 'history' | undefined = $state(undefined) let jsonView = $state(false) let deploymentInProgress = $state(false) + let deploymentJobId: string | undefined = $state(undefined) let intervalId: number | undefined = undefined @@ -166,8 +167,11 @@ }) if (status.lock_error_logs == undefined || status.lock_error_logs != '') { deploymentInProgress = false + deploymentJobId = undefined flow.lock_error_logs = status.lock_error_logs clearInterval(intervalId) + } else if (status.job_id) { + deploymentJobId = status.job_id } } } @@ -541,6 +545,13 @@ Deployment in progress + {#if deploymentJobId} + view job + {/if} {/if} {#if flow.lock_error_logs && flow.lock_error_logs != ''} diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 0dbe481e9f..40a0a51d1d 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -89,6 +89,7 @@ let topHash: string | undefined = $state() let can_write = $state(false) let deploymentInProgress = $state(false) + let deploymentJobId: string | undefined = $state(undefined) let intervalId: number let shareModal: ShareModal | undefined = $state() let runForm: RunForm | undefined = $state() @@ -156,9 +157,12 @@ }) if (status.lock != undefined || status.lock_error_logs != undefined) { deploymentInProgress = false + deploymentJobId = undefined script.lock = status.lock script.lock_error_logs = status.lock_error_logs clearInterval(intervalId) + } else if (status.job_id) { + deploymentJobId = status.job_id } } } @@ -689,6 +693,13 @@ Deployment in progress + {#if deploymentJobId} + view job + {/if} {/if} From 1dfb0f0078792202bd4baa179516b7e3a237334c Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Thu, 4 Dec 2025 11:11:54 +0100 Subject: [PATCH 04/25] refactor: move concurrency and debouncing settings to their own structs (#7286) * move concurrency and debouncing settings to it's own structs Signed-off-by: pyranota * cleanup queue::jobs::pull Signed-off-by: pyranota * done Signed-off-by: pyranota * migrated the rest Signed-off-by: pyranota * implement todo! * impmenet todo! v2 * more cleanup * fix all Signed-off-by: pyranota * remove unrelated code Signed-off-by: pyranota * nits Signed-off-by: pyranota * do not serialize rawcode Signed-off-by: pyranota * cleanup Signed-off-by: pyranota * fixes Signed-off-by: pyranota * update ee repo ref Signed-off-by: pyranota * update ee repo Signed-off-by: pyranota * fix tests Signed-off-by: pyranota * Update ee-repo-ref.txt --------- Signed-off-by: pyranota Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/tests/common/mod.rs | 14 +- backend/tests/job_payload.rs | 22 +- backend/tests/python_jobs.rs | 40 +- backend/tests/worker.rs | 89 +- backend/windmill-api/src/flows.rs | 45 +- backend/windmill-api/src/jobs.rs | 101 +-- .../src/triggers/trigger_helpers.rs | 14 +- backend/windmill-common/src/flows.rs | 54 +- backend/windmill-common/src/jobs.rs | 190 ++++- backend/windmill-common/src/scripts.rs | 4 + backend/windmill-queue/src/jobs.rs | 791 +++++++----------- backend/windmill-queue/src/schedule.rs | 31 +- backend/windmill-worker/src/ai/tools.rs | 22 +- backend/windmill-worker/src/worker_flow.rs | 72 +- .../windmill-worker/src/worker_lockfiles.rs | 16 +- backend/windmill-worker/src/worker_utils.rs | 14 +- 17 files changed, 677 insertions(+), 844 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9cc9af9a72..d57da779de 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d54c99a1e4a08ff30c35737bd282d6d8d2b49e1f +937820a9e572c10c0ce050e1bd5eddb3180fe921 diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 129fd86d63..421d1dd8c3 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -684,16 +684,13 @@ pub async fn run_deployed_relative_imports( let job = RunJob::from(JobPayload::ScriptHash { path: "f/system/test_import".to_string(), hash: ScriptHash(script.hash), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, language, priority: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), }) .push(&db2) .await; @@ -739,13 +736,10 @@ pub async fn run_preview_relative_imports( path: Some("f/system/test_import".to_string()), language, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .push(&db2) .await; diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 22e713329b..5eabdc83ce 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -52,11 +52,8 @@ mod job_payload { let result = RunJob::from(JobPayload::ScriptHash { hash: ScriptHash(123412), path: "f/system/hello".to_string(), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, language: ScriptLang::Deno, @@ -86,16 +83,13 @@ mod job_payload { let job = RunJob::from(JobPayload::ScriptHash { hash: ScriptHash(123413), path: "f/system/hello_with_preprocessor".to_string(), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, language: ScriptLang::Deno, priority: None, apply_preprocessor: true, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -167,9 +161,7 @@ mod job_payload { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[0], language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), cache_ttl: None, dedicated_worker: None, path: "f/system/hello/test-0".into(), @@ -187,9 +179,7 @@ mod job_payload { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[1], language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), cache_ttl: None, dedicated_worker: None, path: "f/system/hello/test-0".into(), diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 549e052b65..f6fc35295c 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -25,7 +25,10 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# workspace-dependencies-mode: manual\n# py: 3.11.11","tiny==0.1.3"], + vec![ + "# workspace-dependencies-mode: manual\n# py: 3.11.11", + "tiny==0.1.3", + ], ) .await?; Ok(()) @@ -186,11 +189,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, }); @@ -237,11 +237,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, }); @@ -273,11 +270,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, }); @@ -314,11 +308,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, }); @@ -353,11 +344,8 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, }); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index bb18d6e397..d18869efb8 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -188,9 +188,8 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default() + .into(), is_trigger: None, assets: None, } @@ -234,9 +233,8 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: + windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, } @@ -368,9 +366,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, @@ -425,9 +421,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, }.into(), @@ -466,9 +460,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, @@ -534,9 +526,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, }.into(), @@ -864,13 +854,10 @@ func main(derp string) (string, error) { path: None, lock: None, language: ScriptLang::Go, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("derp", json!("world")) .run_until_complete(&db, false, port) @@ -903,13 +890,10 @@ fn main(world: String) -> Result { path: None, lock: None, language: ScriptLang::Rust, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, })) .arg("world", json!("Hyrule")) .run_until_complete(&db, false, port) @@ -981,13 +965,10 @@ echo "hello $msg" path: None, lock: None, language: ScriptLang::Bash, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1016,13 +997,10 @@ def main [ msg: string ] { path: None, lock: None, language: ScriptLang::Nu, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1071,13 +1049,10 @@ def main [ path: None, lock: None, language: ScriptLang::Nu, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("3")) .arg("b", json!("null")) @@ -1135,13 +1110,10 @@ public class Main { path: None, lock: None, language: ScriptLang::Java, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!(3)) .arg("b", json!(3.0)) @@ -1172,13 +1144,10 @@ export async function main(a: Date) { path: None, lock: None, language: ScriptLang::Bun, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1209,13 +1178,10 @@ export async function main(a: Date) { path: None, lock: None, language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1247,13 +1213,10 @@ def main(a: datetime, b: bytes): path: None, lock: None, language: ScriptLang::Python3, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .arg("b", json!("dGVzdA==")) diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 7495dbd75b..6ae85b7ad0 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -729,13 +729,12 @@ async fn get_flow_version_by_id( let mut tx = user_db.begin(&authed).await?; // First, fetch the path to perform authorization check early - let path: Option = sqlx::query_scalar( - "SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2", - ) - .bind(version) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2") + .bind(version) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; let path = not_found_if_none( path, @@ -802,13 +801,12 @@ async fn update_flow_history( let mut tx = user_db.begin(&authed).await?; // Fetch path and perform authorization check early - let path: Option = sqlx::query_scalar( - "SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2", - ) - .bind(&w_id) - .bind(version) - .fetch_optional(&mut *tx) - .await?; + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2") + .bind(&w_id) + .bind(version) + .fetch_optional(&mut *tx) + .await?; let path = not_found_if_none( path, @@ -1482,10 +1480,9 @@ async fn archive_flow_by_path( /// Validates that flow debouncing configuration is supported by all workers /// Returns an error if debouncing is configured but workers are behind required version async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> { - if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await && { - let flow_value = nf.parse_flow_value()?; - flow_value.debounce_key.is_some() || flow_value.debounce_delay_s.is_some() - } { + if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await + && !nf.parse_flow_value()?.debouncing_settings.is_default() + { tracing::warn!( "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" ); @@ -1604,6 +1601,7 @@ mod tests { ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, StopAfterIf, }, + jobs::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, scripts, }; @@ -1654,11 +1652,9 @@ mod tests { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, is_trigger: None, assets: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), @@ -1744,17 +1740,14 @@ mod tests { })), preprocessor_module: None, same_worker: false, - concurrent_limit: None, - concurrency_time_window_s: None, skip_expr: None, cache_ttl: None, priority: None, early_return: None, - concurrency_key: None, chat_input_enabled: None, flow_env: None, - debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }; let expect = serde_json::json!({ "modules": [ diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 011c55cee0..add81f0b03 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -35,7 +35,8 @@ use windmill_common::flow_conversations::add_message_to_conversation_tx; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ check_tag_available_for_workspace_internal, format_completed_job_result, format_result, - DynamicInput, JobTriggerKind, ENTRYPOINT_OVERRIDE, + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, DynamicInput, + JobTriggerKind, ENTRYPOINT_OVERRIDE, }; use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::utils::{RunnableKind, WarnAfterExt}; @@ -284,10 +285,7 @@ pub fn workspaced_service() -> Router { "/completed/import", post(crate::jobs_export::import_completed_jobs).layer(cors.clone()), ) - .route( - "/delete", - post(crate::jobs_export::delete_jobs), - ) + .route("/delete", post(crate::jobs_export::delete_jobs)) .route( "/completed/get/:id", get(get_completed_job).layer(cors.clone()), @@ -4588,16 +4586,17 @@ pub async fn run_workflow_as_code( path: job.script_path, language: job.language.unwrap_or_else(|| ScriptLang::Deno), lock: raw_lock, - custom_concurrency_key: windmill_queue::custom_concurrency_key(&db, &job.id) - .await - .map_err(to_anyhow)?, - concurrent_limit: job.concurrent_limit, - concurrency_time_window_s: job.concurrency_time_window_s, + concurrency_settings: windmill_common::jobs::ConcurrencySettingsWithCustom { + custom_concurrency_key: windmill_queue::custom_concurrency_key(&db, &job.id) + .await + .map_err(to_anyhow)?, + concurrent_limit: job.concurrent_limit, + concurrency_time_window_s: job.concurrency_time_window_s, + }, cache_ttl: job.cache_ttl, dedicated_worker: None, // TODO(debouncing): enable for this mode - custom_debounce_key: None, - debounce_delay_s: None, + debouncing_settings: DebouncingSettings::default(), }), Some(job.tag.clone()), None, @@ -5458,11 +5457,17 @@ pub async fn run_wait_result_script_by_hash( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key: concurrency_key, - concurrent_limit: concurrent_limit, - concurrency_time_window_s: concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + concurrency_settings: windmill_common::jobs::ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrent_limit: concurrent_limit, + concurrency_time_window_s: concurrency_time_window_s, + } + .into(), + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() // TODO + }, cache_ttl, language, dedicated_worker, @@ -5934,11 +5939,8 @@ async fn run_preview_script( path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), lock: preview.lock, - custom_concurrency_key: None, - concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here - concurrency_time_window_s: None, // TODO(gbouv): same as above - custom_debounce_key: None, // TODO(pyra): same as for concurrency limits. - debounce_delay_s: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here + debouncing_settings: DebouncingSettings::default(), // TODO(pyra): same as for concurrency limits. cache_ttl: None, dedicated_worker: preview.dedicated_worker, }), @@ -6056,13 +6058,10 @@ async fn run_bundle_preview_script( path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), lock: preview.lock, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: preview.dedicated_worker, - custom_concurrency_key: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&args), authed.display_username(), @@ -6466,19 +6465,19 @@ async fn add_batch_jobs( add_virtual_items_if_necessary(&mut value.modules); let flow_status = FlowStatus::new(&value); ( - None, // script_hash - path, // script_path - job_kind, // job_kind - None, // language - None, // dedicated_worker - value.concurrency_key.clone(), // custom_concurrency_key - value.concurrent_limit.clone(), // concurrent_limit - value.concurrency_time_window_s, // concurrency_time_window_s - None, // timeout - None, // raw_code - None, // raw_lock - Some(value), // raw_flow - Some(flow_status), // flow_status + None, // script_hash + path, // script_path + job_kind, // job_kind + None, // language + None, // dedicated_worker + value.concurrency_settings.concurrency_key.clone(), // custom_concurrency_key + value.concurrency_settings.concurrent_limit.clone(), // concurrent_limit + value.concurrency_settings.concurrency_time_window_s, // concurrency_time_window_s + None, // timeout + None, // raw_code + None, // raw_lock + Some(value), // raw_flow + Some(flow_status), // flow_status ) } "noop" => ( @@ -6847,13 +6846,10 @@ async fn run_dynamic_select( path: None, language: dynamic_input.x_windmill_dyn_select_lang, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default().into(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&request.args.unwrap_or_default()), authed.display_username(), @@ -6977,11 +6973,16 @@ pub async fn run_job_by_hash_inner( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key: concurrency_key, - concurrent_limit: concurrent_limit, - concurrency_time_window_s: concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + concurrency_settings: ConcurrencySettings { + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, cache_ttl, language, dedicated_worker, diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index e592c43bb3..e3ae82b172 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -844,11 +844,8 @@ async fn trigger_script_with_retry_and_error_handler( JobPayload::ScriptHash { hash, path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, cache_ttl, priority, apply_preprocessor, @@ -862,16 +859,13 @@ async fn trigger_script_with_retry_and_error_handler( error_handler_path, error_handler_args, skip_handler: None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, priority, tag_override: tag.clone(), apply_preprocessor, trigger_path: Some(trigger_path), - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, }, _ => { return Err(windmill_common::error::Error::internal_err(format!( diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 9134da3c41..2d49d708ab 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -24,6 +24,7 @@ use crate::{ cache, db::DB, error::{Error, Result as WindmillResult}, + jobs::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, worker::{to_raw_value, Connection}, @@ -171,18 +172,10 @@ pub struct FlowValue { #[serde(default)] #[serde(skip_serializing_if = "is_default")] pub same_worker: bool, - - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub debounce_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub debounce_delay_s: Option, - + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, #[serde(skip_serializing_if = "Option::is_none")] pub skip_expr: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -919,12 +912,8 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "is_none_or_empty")] tag: Option, language: ScriptLang, - #[serde(skip_serializing_if = "Option::is_none")] - custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrency_time_window_s: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "is_none_or_empty_vec")] @@ -945,12 +934,8 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "is_none_or_empty")] tag: Option, language: ScriptLang, - #[serde(skip_serializing_if = "Option::is_none")] - custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrency_time_window_s: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "is_none_or_empty_vec")] @@ -989,9 +974,6 @@ struct UntaggedFlowModuleValue { lock: Option, tag: Option, language: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, is_trigger: Option, id: Option, default_node: Option, @@ -1000,6 +982,8 @@ struct UntaggedFlowModuleValue { tools: Option>, pass_flow_input_directly: Option, squash: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -1074,9 +1058,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { language: untagged .language .ok_or_else(|| serde::de::Error::missing_field("language"))?, - custom_concurrency_key: untagged.custom_concurrency_key, - concurrent_limit: untagged.concurrent_limit, - concurrency_time_window_s: untagged.concurrency_time_window_s, + concurrency_settings: untagged.concurrency_settings, is_trigger: untagged.is_trigger, assets: untagged.assets, }), @@ -1089,9 +1071,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { language: untagged .language .ok_or_else(|| serde::de::Error::missing_field("language"))?, - custom_concurrency_key: untagged.custom_concurrency_key, - concurrent_limit: untagged.concurrent_limit, - concurrency_time_window_s: untagged.concurrency_time_window_s, + concurrency_settings: untagged.concurrency_settings, is_trigger: untagged.is_trigger, assets: untagged.assets, }), @@ -1234,11 +1214,9 @@ pub async fn resolve_module( id, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, } = std::mem::replace(&mut val, Identity) else { unreachable!() @@ -1258,11 +1236,9 @@ pub async fn resolve_module( path: None, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 7cd332cb7e..3abacd93fc 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -73,7 +73,7 @@ impl std::fmt::Display for JobTriggerKind { } } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] #[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] pub enum JobKind { @@ -88,6 +88,7 @@ pub enum JobKind { Identity, FlowDependencies, AppDependencies, + #[default] Noop, DeploymentCallback, FlowScript, @@ -344,23 +345,13 @@ pub enum JobPayload { ScriptHash { hash: ScriptHash, path: String, - /// Override default concurrency key - custom_concurrency_key: Option, - /// How many jobs can run at the same time - concurrent_limit: Option, - /// In seconds - concurrency_time_window_s: Option, - /// If not set, will be inferred from the hash(path + step_id + inputs) - custom_debounce_key: Option, - /// Debouncing delay will be determined by the first job with the key. - /// All subsequent jobs with Some will get debounced. - /// If the job has no delay, it will execute immediately, fully ignoring pending delays. - debounce_delay_s: Option, cache_ttl: Option, dedicated_worker: Option, language: ScriptLang, priority: Option, apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, }, /// Execute flow step (can be subflow only). @@ -372,13 +363,11 @@ pub enum JobPayload { /// Execute flow step FlowScript { id: FlowNodeId, // flow_node(id). + path: String, language: ScriptLang, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, cache_ttl: Option, dedicated_worker: Option, - path: String, + concurrency_settings: ConcurrencySettings, }, /// Inline App Script @@ -458,19 +447,17 @@ pub enum JobPayload { error_handler_path: Option, error_handler_args: Option>>, skip_handler: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, - custom_debounce_key: Option, - debounce_delay_s: Option, cache_ttl: Option, priority: Option, tag_override: Option, trigger_path: Option, apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, }, DeploymentCallback { path: String, + // debouncing_settings: Option, }, Identity, Noop, @@ -479,6 +466,108 @@ pub enum JobPayload { }, } +// TODO: Add validation logic. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct DebouncingSettings { + #[serde( + skip_serializing_if = "Option::is_none", + rename = "debounce_key", + alias = "custom_debounce_key" + )] + /// debounce key is usually stored in the db + /// including when: + /// + /// 1. User have created custom debounce key from ui or cli + /// 2. User used default one + /// + /// in either cases this argument serves as reactive way of overwriting debounce key from the backend. + /// Default: hash(path + step_id + inputs) + pub custom_key: Option, + + #[serde(skip_serializing_if = "Option::is_none", rename = "debounce_delay_s")] + /// Debouncing delay will be determined by the first job with the key. + /// All subsequent jobs with Some will get debounced. + /// If the job has no delay, it will execute immediately, fully ignoring pending delays. + pub delay_s: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "max_total_debouncing_time" + )] + pub max_total_time: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "max_total_debounces_amount" + )] + pub max_total_amount: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "debounce_args_to_accumulate" + )] + /// top level arguments to preserve + /// For every debounce selected arguments will be saved + /// in the end (when job finally starts) arguments will be appended and passed to runnable + /// + /// NOTE: selected args should be the lists. + pub args_to_accumulate: Option>, +} + +impl DebouncingSettings { + pub fn is_default(&self) -> bool { + self == &Self::default() + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, sqlx::FromRow)] +pub struct ConcurrencySettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] +pub struct ConcurrencySettingsWithCustom { + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +impl From for ConcurrencySettingsWithCustom { + fn from( + ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, + ) -> Self { + ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + +impl From for ConcurrencySettings { + fn from( + ConcurrencySettingsWithCustom { + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }: ConcurrencySettingsWithCustom, + ) -> Self { + ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + #[derive(Clone, Serialize, Deserialize, Debug)] pub struct SkipHandler { pub path: String, @@ -487,20 +576,49 @@ pub struct SkipHandler { pub stop_message: String, } -#[derive(Clone, Serialize, Deserialize, Debug, Default)] +#[derive(Clone, Deserialize, Debug, Default)] pub struct RawCode { pub content: String, pub path: Option, pub hash: Option, pub language: ScriptLang, pub lock: Option, - pub custom_concurrency_key: Option, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub custom_debounce_key: Option, - pub debounce_delay_s: Option, pub cache_ttl: Option, pub dedicated_worker: Option, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(flatten)] + // NOTE: Since we can only deserialize the struct, + // even though the older versions pass `custom_debounce_key` to RawCode, + // we can still have `debounce_key` in DebouncingSettings + // we just add alias `custom_debounce_key` + // however, serializing this settings will produce `debounce_key` + pub debouncing_settings: DebouncingSettings, +} + +impl JobPayload { + pub fn job_kind(&self) -> JobKind { + match self { + JobPayload::Noop => JobKind::Noop, + JobPayload::Identity => JobKind::Identity, + JobPayload::Code { .. } => JobKind::Preview, + JobPayload::AIAgent { .. } => JobKind::AIAgent, + JobPayload::FlowNode { .. } => JobKind::FlowNode, + JobPayload::ScriptHash { .. } => JobKind::Script, + JobPayload::AppScript { .. } => JobKind::AppScript, + JobPayload::RawFlow { .. } => JobKind::FlowPreview, + JobPayload::ScriptHub { .. } => JobKind::Script_Hub, + JobPayload::FlowScript { .. } => JobKind::FlowScript, + JobPayload::Dependencies { .. } => JobKind::Dependencies, + JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, + JobPayload::AppDependencies { .. } => JobKind::AppDependencies, + JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, + JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, + JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, + } + } } type Tag = String; @@ -597,17 +715,23 @@ pub async fn script_path_to_payload<'e>( JobPayload::ScriptHash { hash: ScriptHash(hash), path: script_path.to_owned(), - custom_concurrency_key: concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, cache_ttl, language, dedicated_worker, priority, apply_preprocessor: !skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), + concurrency_settings: ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrent_limit, + concurrency_time_window_s, + } + .into(), + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, }, tag, delete_after_use, diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 191fe7343a..25bde2804e 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -495,6 +495,10 @@ pub struct NewScript { pub tag: Option, pub draft_only: Option, pub envs: Option>, + // NOTE: concurrency and debounce data is inline, + // bc it was this before refactor + // and rust seems to hash it differently + // for backwards compat we keep them inline pub concurrency_key: Option, pub concurrent_limit: Option, pub concurrency_time_window_s: Option, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 059bfc69b8..8f7b2ed7a1 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -14,9 +14,9 @@ use async_recursion::async_recursion; use chrono::{DateTime, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; -use quick_cache::sync::Cache; #[cfg(feature = "prometheus")] use prometheus::IntCounter; +use quick_cache::sync::Cache; use regex::Regex; use reqwest::Client; use serde::Deserialize; @@ -38,7 +38,10 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; +use windmill_common::jobs::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, JobTriggerKind, + EMAIL_ERROR_HANDLER_USER_EMAIL, +}; use windmill_common::utils::{configure_client, now_from_db}; use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; @@ -168,7 +171,6 @@ pub async fn cancel_single_job<'c>( db: &Pool, force_cancel: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { - let id = job_running.id; if force_cancel || (job_running.parent_job.is_none() && !job_running.running) { let username = username.to_string(); @@ -236,7 +238,6 @@ pub async fn cancel_job<'c>( //TODO fetch mini completed job instead of QueuedJob let job = get_queued_job_v2(&mut *tx, &id).await?; - if job.is_none() { return Ok((tx, None)); } @@ -249,7 +250,6 @@ pub async fn cancel_job<'c>( )); } - if job.workspace_id != w_id { return Err(Error::BadRequest( "You are not authorized to cancel this job belonging to another workspace".to_string(), @@ -324,16 +324,8 @@ ORDER BY depth, id tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len()); } - let (ntx, _) = cancel_single_job( - username, - reason.clone(), - job, - w_id, - tx, - db, - force_cancel, - ) - .await?; + let (ntx, _) = + cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel).await?; tx = ntx; if !force_cancel { @@ -357,16 +349,9 @@ ORDER BY depth, id let job = get_queued_job_v2(&mut *tx, &job_id).await?; if let Some(job) = job { - let (ntx, _) = cancel_single_job( - username, - reason.clone(), - job, - w_id, - tx, - db, - force_cancel, - ) - .await?; + let (ntx, _) = + cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel) + .await?; tx = ntx; } } @@ -445,13 +430,10 @@ pub async fn push_init_job<'c>( path: Some(format!("{INIT_SCRIPT_PATH_PREFIX}{worker_name}")), language: ScriptLang::Bash, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&ehm), worker_name, @@ -504,13 +486,10 @@ pub async fn push_periodic_bash_job<'c>( )), language: ScriptLang::Bash, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, cache_ttl: None, dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&ehm), worker_name, @@ -867,7 +846,6 @@ pub async fn add_completed_job( restart_job_if_perpetual(db, completed_job, &canceled_by).await?; - // tracing::error!("4 {:?}", start.elapsed()); Ok((completed_job.id, duration)) @@ -1343,9 +1321,6 @@ async fn restart_job_if_perpetual_inner( JobPayload::ScriptHash { hash, path: queued_job.runnable_path.clone().unwrap_or_default(), - custom_concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: queued_job.cache_ttl, dedicated_worker: None, language: queued_job @@ -1354,9 +1329,13 @@ async fn restart_job_if_perpetual_inner( .unwrap_or_else(|| ScriptLang::Deno), priority: queued_job.priority, apply_preprocessor: false, + concurrency_settings: ConcurrencySettings { + concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, + concurrent_limit: None, + concurrency_time_window_s: None, + }, // TODO(debouncing): handle properly - custom_debounce_key: None, - debounce_delay_s: None, + debouncing_settings: DebouncingSettings::default(), }, PushArgs::from(&args.0), &queued_job.created_by, @@ -1442,7 +1421,6 @@ fn apply_completed_job_cloud_usage( tracing::error!("Failed to get team plan status to update usage for workspace {w_id}: {err:#}"); } }; - }).await; if let Err(_) = result { @@ -1571,8 +1549,9 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> (cached.0.clone(), cached.1.clone(), cached.2) } else { // Cache expired, fetch from database - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" + let row_result = + sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" SELECT error_handler, error_handler_extra_args, @@ -1582,25 +1561,33 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> WHERE workspace_id = $1 "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| { + Error::internal_err(format!("no workspace settings for id {w_id}")) + })?; // Update cache with 60s TTL let expiry = now + 60; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ( + row_result.0.clone(), + row_result.1.clone(), + row_result.2, + expiry, + ), ); row_result } } else { // Cache miss, fetch from database - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" + let row_result = + sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" SELECT error_handler, error_handler_extra_args, @@ -1610,18 +1597,25 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> WHERE workspace_id = $1 "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| { + Error::internal_err(format!("no workspace settings for id {w_id}")) + })?; // Store in cache with 60s TTL let expiry = now + 60; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ( + row_result.0.clone(), + row_result.1.clone(), + row_result.2, + expiry, + ), ); row_result }; @@ -1991,7 +1985,6 @@ pub struct MiniPulledJob { pub permissioned_as_end_user_email: Option, } - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MiniCompletedJob { @@ -2041,7 +2034,6 @@ impl From for MiniCompletedJob { concurrent_limit: job.concurrent_limit, tag: job.tag, cache_ttl: job.cache_ttl, - } } } @@ -2116,11 +2108,16 @@ impl MiniCompletedJob { pub fn is_dependency(&self) -> bool { self.kind.is_dependency() } - } -fn schedule_path(trigger_kind: &Option, trigger: &Option) -> Option { - if trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { +fn schedule_path( + trigger_kind: &Option, + trigger: &Option, +) -> Option { + if trigger_kind + .as_ref() + .is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) + { trigger.clone() } else { None @@ -2413,7 +2410,6 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } - pub struct QueuedJobV2 { pub id: Uuid, pub workspace_id: String, @@ -2449,7 +2445,9 @@ impl QueuedJobV2 { } pub async fn get_queued_job_v2<'c>( - e: impl PgExecutor<'c>, job_id: &Uuid) -> error::Result> { + e: impl PgExecutor<'c>, + job_id: &Uuid, +) -> error::Result> { let job = sqlx::query_as!( QueuedJobV2, "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", @@ -2469,7 +2467,7 @@ pub struct PulledJobResult { pub job: Option, pub suspended: bool, pub missing_concurrency_key: bool, - pub error_while_preprocessing: Option + pub error_while_preprocessing: Option, } #[derive(thiserror::Error, Debug)] @@ -2480,7 +2478,6 @@ pub enum PulledJobResultToJobErr { ErrorWhilePreprocessing(JobCompleted), } - impl PulledJobResult { pub fn to_pulled_job(self) -> Result, PulledJobResultToJobErr> { match self { @@ -2761,7 +2758,6 @@ impl PulledJobResult { } } else { tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); - } // This will unblock pusher. tx.commit().await?; @@ -2853,21 +2849,21 @@ pub async fn pull( // Concurrency limit is available for either enterprise job or dependency job && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING)) => { - crate::jobs_ee::apply_concurrency_limit( - db, - pull_loop_count, - suspended, - job, - ) - .await? - .unwrap_or(PulledJobResult { - job: None, - suspended, - missing_concurrency_key: false, - error_while_preprocessing: None, - }) + crate::jobs_ee::apply_concurrency_limit(db, pull_loop_count, suspended, job) + .await? + .unwrap_or(PulledJobResult { + job: None, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }) } - _ => PulledJobResult { job, suspended, missing_concurrency_key: false, error_while_preprocessing: None }, + _ => PulledJobResult { + job, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }, }; Ok::<_, Error>(pulled_job_result) @@ -2885,7 +2881,12 @@ pub async fn pull( ) .await?; let Some(job) = job else { - return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false, error_while_preprocessing: None }); + return Ok(PulledJobResult { + job: None, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }); }; let has_concurent_limit = job.concurrent_limit.is_some(); @@ -2896,7 +2897,10 @@ pub async fn pull( } #[cfg(not(feature = "enterprise"))] - let has_concurent_limit = job.is_dependency() && job.concurrent_limit.is_some() && cfg!(feature = "private") && !*WMDEBUG_NO_DJOB_DEBOUNCING; + let has_concurent_limit = job.is_dependency() + && job.concurrent_limit.is_some() + && cfg!(feature = "private") + && !*WMDEBUG_NO_DJOB_DEBOUNCING; // if we don't have private flag, we don't have concurrency limit // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue @@ -3593,13 +3597,13 @@ pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result + Send + 'a, ->(id: &'a Uuid, w_id: &'a str, db: A) -> impl Future>> + Send + 'a { - async move { - let mut conn = db.acquire().await?; +pub fn get_mini_completed_job<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a>( + id: &'a Uuid, + w_id: &'a str, + db: A, +) -> impl Future>> + Send + 'a { + async move { + let mut conn = db.acquire().await?; sqlx::query_as!( MiniCompletedJob, "SELECT @@ -3616,7 +3620,6 @@ A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, } } - pub enum PushIsolationLevel<'c> { IsolatedRoot(DB), Isolated(UserDB, Authed), @@ -3828,8 +3831,7 @@ pub async fn push<'c, 'd>( mut email: &str, mut permissioned_as: String, token_prefix: Option<&str>, - #[allow(unused_mut)] - mut scheduled_for_o: Option>, + #[allow(unused_mut)] mut scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, root_job: Option, @@ -4031,87 +4033,87 @@ pub async fn push<'c, 'd>( } } + #[derive(Default)] + struct JobPayloadUntagged { + runnable_id: Option, + runnable_path: Option, + raw_code_tuple: Option<(String, Option)>, // (content, lock) + job_kind: JobKind, + raw_flow: Option, + flow_status: Option, + language: Option, + cache_ttl: Option, + dedicated_worker: Option, + _low_level_priority: Option, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + } let mut preprocessed = None; - #[allow(unused)] - let ( - script_hash, - script_path, + #[allow(unused)] + let JobPayloadUntagged { + runnable_id, + runnable_path, raw_code_tuple, job_kind, raw_flow, flow_status, language, - mut custom_concurrency_key, - mut concurrent_limit, - concurrency_time_window_s, cache_ttl, dedicated_worker, _low_level_priority, - custom_debounce_key, - debounce_delay_s, - ) = match job_payload { + concurrency_settings: + ConcurrencySettings { + mut concurrency_key, + mut concurrent_limit, + concurrency_time_window_s, // + }, + debouncing_settings, + } = match job_payload { JobPayload::ScriptHash { hash, path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, language, dedicated_worker, priority, apply_preprocessor, - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, } => { if apply_preprocessor { preprocessed = Some(false); } - ( - Some(hash.0), - Some(path), - None, - JobKind::Script, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: Some(hash.0), + runnable_path: Some(path), + job_kind: JobKind::Script, + language: Some(language), + concurrency_settings, + debouncing_settings, cache_ttl, dedicated_worker, - priority, - custom_debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + ..Default::default() + } } JobPayload::FlowScript { id, // flow_node(id). language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, dedicated_worker, path, - } => ( - Some(id.0), - Some(path), - None, - JobKind::FlowScript, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, + } => JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: Some(path), + job_kind: JobKind::FlowScript, + language: Some(language), + concurrency_settings, cache_ttl, dedicated_worker, - None, - None, // custom_debounce_key removed for flow steps - None, // debounce_delay_s removed for flow steps - ), + ..Default::default() + }, JobPayload::FlowNode { id, path } => { let data = cache::flow::fetch_flow(_db, id).await?; let value = data.value(); @@ -4124,46 +4126,28 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - Some(id.0), - Some(path), - None, - JobKind::FlowNode, - value_o, - status, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ) + JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: Some(path), + job_kind: JobKind::FlowNode, + raw_flow: value_o, + flow_status: status, + ..Default::default() + } } JobPayload::AppScript { id, // app_script(id). path, language, cache_ttl, - } => ( - Some(id.0), - path, - None, - JobKind::AppScript, - None, - None, - Some(language), - None, - None, - None, + } => JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: path, + job_kind: JobKind::AppScript, + language: Some(language), cache_ttl, - None, - None, - None, - None, - ), + ..Default::default() + }, JobPayload::ScriptHub { path, apply_preprocessor } => { if path == "hub/7771/slack" || path == "hub/7836/slack" || path == "hub/9084/slack" { // these scripts send app reports to slack @@ -4180,24 +4164,12 @@ pub async fn push<'c, 'd>( get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(_db)) .await?; - ( - None, - Some(path), - None, - // Some((script.content, script.lockfile)), - JobKind::Script_Hub, - None, - None, - Some(hub_script.language), - None, - None, - None, - None, - None, - None, - None, - None, - ) + JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::Script_Hub, + language: Some(hub_script.language), + ..Default::default() + } } JobPayload::Code(RawCode { content, @@ -4205,85 +4177,49 @@ pub async fn push<'c, 'd>( hash, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, dedicated_worker, - custom_debounce_key, - debounce_delay_s, - }) => ( - hash, - path, - Some((content, lock)), - JobKind::Preview, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, + debouncing_settings, + }) => JobPayloadUntagged { + runnable_id: hash, + runnable_path: path, + raw_code_tuple: Some((content, lock)), + job_kind: JobKind::Preview, + language: Some(language), + concurrency_settings: concurrency_settings.into(), + debouncing_settings, cache_ttl, dedicated_worker, - None, - custom_debounce_key, - debounce_delay_s, - ), - JobPayload::Dependencies { hash, language, path, dedicated_worker } => ( - Some(hash.0), - Some(path.clone()), - None, - JobKind::Dependencies, - None, - None, - Some(language), - None, - None, - None, - None, + ..Default::default() + }, + JobPayload::Dependencies { hash, language, path, dedicated_worker } => JobPayloadUntagged { + runnable_id: Some(hash.0), + runnable_path: Some(path.clone()), + job_kind: JobKind::Dependencies, + language: Some(language), dedicated_worker, - None, - None, - None, - ), + ..Default::default() + }, // CLI usage, is not modifying db, no need for debouncing. - JobPayload::RawScriptDependencies { script_path, content, language } => ( - None, - Some(script_path), - Some((content, None)), - JobKind::Dependencies, - None, - None, - Some(language), - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::RawScriptDependencies { script_path, content, language } => { + JobPayloadUntagged { + runnable_path: Some(script_path), + raw_code_tuple: Some((content, None)), + job_kind: JobKind::Dependencies, + language: Some(language), + ..Default::default() + } + } // CLI usage, is not modifying db, no need for debouncing. - JobPayload::RawFlowDependencies { path, flow_value } => ( - None, - Some(path), - None, - JobKind::FlowDependencies, - Some(flow_value), - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::RawFlowDependencies { path, flow_value } => JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::FlowDependencies, + raw_flow: Some(flow_value), + ..Default::default() + }, JobPayload::FlowDependencies { path, dedicated_worker, version } => { #[cfg(test)] let skip_compat = args @@ -4305,41 +4241,21 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - Some(version), - Some(path.clone()), - None, - JobKind::FlowDependencies, - value_o, - None, - None, - None, - None, - None, - None, + JobPayloadUntagged { + runnable_id: Some(version), + runnable_path: Some(path.clone()), + job_kind: JobKind::FlowDependencies, + raw_flow: value_o, dedicated_worker, - None, - None, - None, - ) + ..Default::default() + } } - JobPayload::AppDependencies { path, version } => ( - Some(version), - Some(path.clone()), - None, - JobKind::AppDependencies, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::AppDependencies { path, version } => JobPayloadUntagged { + runnable_id: Some(version), + runnable_path: Some(path.clone()), + job_kind: JobKind::AppDependencies, + ..Default::default() + }, JobPayload::RawFlow { mut value, path, restarted_from } => { add_virtual_items_if_necessary(&mut value.modules); @@ -4386,30 +4302,19 @@ pub async fn push<'c, 'd>( FlowStatus::new(&value) } // this is a new flow being pushed, flow_status is set to flow_value }; - let concurrency_key = value.concurrency_key.clone(); - let concurrent_limit = value.concurrent_limit; - let concurrency_time_window_s = value.concurrency_time_window_s; - let debounce_key = value.debounce_key.clone(); - let debounce_delay_s = value.debounce_delay_s; let cache_ttl = value.cache_ttl.map(|x| x as i32); let priority = value.priority; - ( - None, - path, - None, - JobKind::FlowPreview, - Some(value), - Some(flow_status), - None, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_path: path, + job_kind: JobKind::FlowPreview, + flow_status: Some(flow_status), cache_ttl, - None, - priority, - debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + concurrency_settings: value.concurrency_settings.clone(), + debouncing_settings: value.debouncing_settings.clone(), + raw_flow: Some(value), + ..Default::default() + } } JobPayload::SingleStepFlow { path, @@ -4420,16 +4325,13 @@ pub async fn push<'c, 'd>( error_handler_args, skip_handler, args, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, priority, tag_override, trigger_path, apply_preprocessor, - custom_debounce_key, - debounce_delay_s, + debouncing_settings, + concurrency_settings, } => { // Determine if this is a flow or a script let is_flow = flow_version.is_some(); @@ -4563,13 +4465,10 @@ pub async fn push<'c, 'd>( let flow_value = FlowValue { modules, failure_module, - concurrency_time_window_s, - concurrent_limit, - debounce_key: custom_debounce_key.clone(), - debounce_delay_s, + concurrency_settings: concurrency_settings.clone(), + debouncing_settings: debouncing_settings.clone(), priority, cache_ttl: cache_ttl.map(|val| val as u32), - concurrency_key: custom_concurrency_key.clone(), same_worker: false, early_return: None, skip_expr: None, @@ -4579,23 +4478,17 @@ pub async fn push<'c, 'd>( }; // this is a new flow being pushed, flow_status is set to flow_value: let flow_status: FlowStatus = FlowStatus::new(&flow_value); - ( - None, // No version needed - flow is stored in raw_flow like FlowPreview - Some(path), - None, - JobKind::SingleStepFlow, - Some(flow_value), - Some(flow_status), - None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::SingleStepFlow, + raw_flow: Some(flow_value), + flow_status: Some(flow_status), cache_ttl, - None, - priority, - custom_debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => { let mut ntx = tx.into_tx().await?; @@ -4618,20 +4511,18 @@ pub async fn push<'c, 'd>( let mut value = data.value().clone(); let priority = value.priority; let cache_ttl = value.cache_ttl.map(|x| x as i32); - let custom_concurrency_key = value.concurrency_key.clone(); - let concurrency_time_window_s = value.concurrency_time_window_s; - let mut concurrent_limit = value.concurrent_limit; - - let custom_debounce_key = value.debounce_key.clone(); - let mut debounce_delay_s = value.debounce_delay_s; + let mut concurrency_settings = value.concurrency_settings.clone(); + let mut debouncing_settings = value.debouncing_settings.clone(); if !apply_preprocessor { value.preprocessor_module = None; } else { tag = None; - concurrent_limit = None; + + concurrency_settings.concurrent_limit = None; // TODO: May be re-enable? - debounce_delay_s = None; + debouncing_settings.delay_s = None; + preprocessed = Some(false); } @@ -4652,23 +4543,19 @@ pub async fn push<'c, 'd>( // by additional checks when handling the flow. None }; - ( - Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull. - Some(path), - None, - JobKind::Flow, - value_o, - Some(status), - None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull. + runnable_path: Some(path), + job_kind: JobKind::Flow, + raw_flow: value_o, + flow_status: Some(status), cache_ttl, dedicated_worker, - priority, - custom_debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } JobPayload::RestartedFlow { completed_job_id, step_id, branch_or_iteration_n } => { let ( @@ -4687,6 +4574,7 @@ pub async fn push<'c, 'd>( branch_or_iteration_n, ) .await?; + let restarted_flow_status = FlowStatus { step: step_n, modules: truncated_modules, @@ -4715,11 +4603,8 @@ pub async fn push<'c, 'd>( }; let value = flow_data.value(); let priority = value.priority; - let concurrency_key = value.concurrency_key.clone(); - let concurrent_limit = value.concurrent_limit; - let concurrency_time_window_s = value.concurrency_time_window_s; - let debounce_key = value.debounce_key.clone(); - let debounce_delay_s = value.debounce_delay_s; + let concurrency_settings = value.concurrency_settings.clone(); + let debouncing_settings = value.debouncing_settings.clone(); let cache_ttl = value.cache_ttl.map(|x| x as i32); // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. @@ -4729,92 +4614,38 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - version, - flow_path, - None, - JobKind::Flow, - value_o, - Some(restarted_flow_status), - None, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: version, + runnable_path: flow_path, + job_kind: JobKind::Flow, + raw_flow: value_o, + flow_status: Some(restarted_flow_status), cache_ttl, - None, - priority, - debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } - JobPayload::DeploymentCallback { path } => ( - None, - Some(path.clone()), - None, - JobKind::DeploymentCallback, - None, - None, - None, - Some(format!("{workspace_id}:git_sync")), - Some(1), - Some(0), - None, - None, - None, - None, - None, - ), - JobPayload::Identity => ( - None, - None, - None, - JobKind::Identity, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), - JobPayload::Noop => ( - None, - None, - None, - JobKind::Noop, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), - JobPayload::AIAgent { path } => ( - None, - Some(path), - None, - JobKind::AIAgent, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::DeploymentCallback { path } => JobPayloadUntagged { + runnable_path: Some(path.clone()), + job_kind: JobKind::DeploymentCallback, + concurrency_settings: ConcurrencySettings { + concurrency_key: Some(format!("{workspace_id}:git_sync")), + concurrent_limit: Some(1), + concurrency_time_window_s: Some(0), + }, + ..Default::default() + }, + JobPayload::Identity => { + JobPayloadUntagged { job_kind: JobKind::Identity, ..Default::default() } + } + JobPayload::Noop => JobPayloadUntagged { job_kind: JobKind::Noop, ..Default::default() }, + JobPayload::AIAgent { path } => JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::AIAgent, + ..Default::default() + }, }; // Enforce concurrency limit on all dependency jobs. @@ -4823,13 +4654,13 @@ pub async fn push<'c, 'd>( // // This is not the case for scripts, so we can potentially have multiple djobs for scripts at the same time. if let (Some(path), true) = ( - &script_path, + &runnable_path, cfg!(feature = "private") && job_kind.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING && *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, ) { - custom_concurrency_key = Some(format!("dependency:{workspace_id}/{path}")); + concurrency_key = Some(format!("dependency:{workspace_id}/{path}")); concurrent_limit = Some(1); } @@ -4895,7 +4726,7 @@ pub async fn push<'c, 'd>( } else { "" }, - script_path.clone().expect("dedicated script has a path") + runnable_path.clone().expect("dedicated script has a path") ) } else { if tag == Some("".to_string()) { @@ -4973,7 +4804,7 @@ pub async fn push<'c, 'd>( match ( scheduled_for_o.is_some(), job_kind.is_dependency(), - script_path.clone(), + runnable_path.clone(), *WMDEBUG_NO_DJOB_DEBOUNCING, *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, // We only do debouncing for jobs triggered by relative imports @@ -5155,10 +4986,10 @@ pub async fn push<'c, 'd>( if schedule_path.is_none() { if let Some(debounced_job_id) = crate::jobs_ee::maybe_apply_debouncing( &job_id, - debounce_delay_s, - custom_debounce_key, + debouncing_settings.delay_s, + debouncing_settings.custom_key, workspace_id, - script_path.clone(), + runnable_path.clone(), &job_kind, &args, &mut scheduled_for_o, @@ -5174,9 +5005,9 @@ pub async fn push<'c, 'd>( insert_concurrency_key( workspace_id, &args, - &script_path, + &runnable_path, job_kind, - custom_concurrency_key, + concurrency_key, &mut tx, job_id, ) @@ -5294,8 +5125,8 @@ pub async fn push<'c, 'd>( parent_job, user, permissioned_as, - script_hash, - script_path.clone(), + runnable_id, + runnable_path.clone(), Json(args) as Json, job_kind.clone() as JobKind, schedule_path, @@ -5384,7 +5215,7 @@ pub async fn push<'c, 'd>( let operation_name = match job_kind { JobKind::Preview => "jobs.run.preview", JobKind::Script => { - s = ScriptHash(script_hash.unwrap()).to_string(); + s = ScriptHash(runnable_id.unwrap()).to_string(); hm.insert("hash", s.as_str()); "jobs.run.script" } @@ -5430,7 +5261,7 @@ pub async fn push<'c, 'd>( operation_name, ActionKind::Execute, workspace_id, - script_path.as_ref().map(|x| x.as_str()), + runnable_path.as_ref().map(|x| x.as_str()), Some(hm), ) .warn_after_seconds(1) @@ -5721,8 +5552,6 @@ async fn restarted_flows_resolution( )) } - - // Wrapper struct to send both job and optional flow_runners to dedicated workers pub struct DedicatedWorkerJob { pub job: Arc, @@ -5740,7 +5569,11 @@ pub struct FlowRunners { impl Drop for FlowRunners { fn drop(&mut self) { let total_runners = self.handles.len(); - tracing::info!("dropping {} flow runners for job {}", total_runners, self.job_id); + tracing::info!( + "dropping {} flow runners for job {}", + total_runners, + self.job_id + ); // First, drop all senders to signal workers to stop gracefully self.runners.clear(); @@ -5756,12 +5589,17 @@ impl Drop for FlowRunners { // Wait up to 5 seconds for natural termination let timeout_result = tokio::time::timeout( tokio::time::Duration::from_secs(5), - futures::future::join_all(handles) - ).await; + futures::future::join_all(handles), + ) + .await; match timeout_result { Ok(_) => { - tracing::info!("all {} flow runners for job {} terminated gracefully", total_runners, job_id); + tracing::info!( + "all {} flow runners for job {} terminated gracefully", + total_runners, + job_id + ); } Err(_) => { // Timeout reached, abort only the handles that haven't finished @@ -5859,4 +5697,3 @@ pub async fn get_same_worker_job( )) }) } - diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 52a397ce33..4a74ed80d6 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -20,6 +20,8 @@ use windmill_common::flows::Retry; use windmill_common::get_flow_version_info_from_version; use windmill_common::get_latest_flow_version_id_for_path; use windmill_common::jobs::check_tag_available_for_workspace_internal; +use windmill_common::jobs::ConcurrencySettings; +use windmill_common::jobs::DebouncingSettings; use windmill_common::jobs::JobPayload; use windmill_common::schedule::schedule_to_user; use windmill_common::scripts::ScriptHash; @@ -258,16 +260,13 @@ pub async fn push_scheduled_job<'c>( stop_condition, stop_message, }), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, priority: None, tag_override: schedule.tag.clone(), trigger_path: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -360,16 +359,13 @@ pub async fn push_scheduled_job<'c>( error_handler_args: None, skip_handler: None, args: static_args, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl, priority, tag_override: schedule.tag.clone(), trigger_path: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -385,16 +381,21 @@ pub async fn push_scheduled_job<'c>( JobPayload::ScriptHash { hash, path: schedule.script_path.clone(), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, dedicated_worker, language, priority, apply_preprocessor: false, - custom_debounce_key, - debounce_delay_s, + debouncing_settings: DebouncingSettings { + custom_key: custom_debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, + concurrency_settings: ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 84304f232c..d0f50ec750 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -366,9 +366,7 @@ async fn execute_windmill_tool( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = path @@ -379,32 +377,20 @@ async fn execute_windmill_tool( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, tool_module, tag, tool_module.delete_after_use.unwrap_or(false), ) } - FlowModuleValue::FlowScript { - id, - language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - tag, - .. - } => { + FlowModuleValue::FlowScript { id, language, concurrency_settings, tag, .. } => { let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id); let payload = JobPayloadWithTag { payload: JobPayload::FlowScript { id, language, - custom_concurrency_key: custom_concurrency_key.clone(), - concurrent_limit, - concurrency_time_window_s, + concurrency_settings: concurrency_settings.into(), cache_ttl: tool_module.cache_ttl.map(|x| x as i32), dedicated_worker: None, path, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index dd45d3b786..427951f18b 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -43,7 +43,8 @@ use windmill_common::flow_status::{ }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; @@ -3956,9 +3957,7 @@ async fn compute_next_flow_transform( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = path.unwrap_or_else(|| get_path(flow_job, status, module)); @@ -3968,9 +3967,7 @@ async fn compute_next_flow_transform( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, module, tag, delete_after_use, @@ -3984,9 +3981,7 @@ async fn compute_next_flow_transform( id, // flow_node(id). tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = get_path(flow_job, status, module); @@ -3995,9 +3990,7 @@ async fn compute_next_flow_transform( payload: JobPayload::FlowScript { id, language, - custom_concurrency_key: custom_concurrency_key.clone(), - concurrent_limit, - concurrency_time_window_s, + concurrency_settings: concurrency_settings.into(), cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, path, @@ -4625,18 +4618,14 @@ async fn payload_from_simple_module( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => raw_script_to_payload( path.unwrap_or_else(|| inner_path), content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, module, tag, delete_after_use, @@ -4645,20 +4634,16 @@ async fn payload_from_simple_module( id, // flow_node(id). tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => JobPayloadWithTag { payload: JobPayload::FlowScript { id, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, path: inner_path, + concurrency_settings: concurrency_settings.into(), }, tag, delete_after_use, @@ -4674,9 +4659,7 @@ pub fn raw_script_to_payload( content: String, language: windmill_common::scripts::ScriptLang, lock: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, + concurrency_settings: ConcurrencySettingsWithCustom, module: &FlowModule, tag: Option, delete_after_use: bool, @@ -4688,13 +4671,11 @@ pub fn raw_script_to_payload( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings, + // TODO: Should this have debouncing? + debouncing_settings: DebouncingSettings::default(), }), tag, delete_after_use, @@ -4756,11 +4737,6 @@ pub async fn script_to_payload( let ScriptHashInfo { tag, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, - debounce_key, - debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -4769,6 +4745,11 @@ pub async fn script_to_payload( timeout, on_behalf_of_email, created_by, + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + debounce_key, + debounce_delay_s, .. } = get_script_info_for_hash(None, db, &flow_job.workspace_id, hash.0).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { @@ -4784,11 +4765,16 @@ pub async fn script_to_payload( JobPayload::ScriptHash { hash, path: script_path, - custom_concurrency_key: concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, + concurrency_settings: ConcurrencySettings { + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), language, dedicated_worker, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index b00d15f4eb..56a3c82fe2 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1171,11 +1171,9 @@ async fn lock_modules<'c>( mut language, input_transforms, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, } = e.get_value()? else { let mut nmodified_ids = Vec::new(); @@ -1541,11 +1539,9 @@ async fn lock_modules<'c>( content, language, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }); new_flow_modules.push(e); @@ -1737,11 +1733,9 @@ async fn reduce_flow<'c>( language, input_transforms, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, .. } = std::mem::replace(&mut val, Identity) else { @@ -1764,11 +1758,9 @@ async fn reduce_flow<'c>( id, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }; } ForloopFlow { modules, modules_node, .. } diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index f105d8c415..e92092480b 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -395,15 +395,19 @@ pub async fn get_tag_and_concurrency(job_id: &Uuid, db: &DB) -> Option cache::flow::fetch_version(db, version).await, }; let flow_value = flow.map(|f| f.value().clone()).ok(); + let concurrency_key = flow_value .as_ref() - .map(|fv| fv.concurrency_key.clone()) - .flatten(); - let concurrent_limit = flow_value.as_ref().map(|fv| fv.concurrent_limit).flatten(); + .and_then(|fv| fv.concurrency_settings.concurrency_key.to_owned()); + + let concurrent_limit = flow_value + .as_ref() + .and_then(|fv| fv.concurrency_settings.concurrent_limit); + let concurrent_time_window_s = flow_value .as_ref() - .map(|fv| fv.concurrency_time_window_s) - .flatten(); + .and_then(|fv| fv.concurrency_settings.concurrency_time_window_s); + Some(TagAndConcurrencyKey { tag: tag_and_concurrency_key.tag, concurrency_key, From 3757cbce87a2d32539efdac7493af2efec22cb7e Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 4 Dec 2025 16:30:38 +0100 Subject: [PATCH 05/25] fix: enable back gcp triggers in CLI (#7299) --- cli/src/commands/trigger/trigger.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index cd52265b52..ecad4ed35e 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -18,7 +18,10 @@ import { removeType, TRIGGER_TYPES, } from "../../types.ts"; -import { fromBranchSpecificPath, isBranchSpecificFile } from "../../core/specific_items.ts"; +import { + fromBranchSpecificPath, + isBranchSpecificFile, +} from "../../core/specific_items.ts"; import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; @@ -32,7 +35,7 @@ type Trigger = { mqtt: MqttTrigger; sqs: SqsTrigger; gcp: GcpTrigger; - email: EmailTrigger + email: EmailTrigger; }; type TriggerFile = Omit< @@ -95,9 +98,7 @@ async function updateTrigger( postgres: wmill.updatePostgresTrigger, mqtt: wmill.updateMqttTrigger, sqs: wmill.updateSqsTrigger, - gcp: async (args) => { - throw new Error("GCP triggers are not supported yet"); - }, + gcp: wmill.updateGcpTrigger, email: wmill.updateEmailTrigger, }; const triggerFunction = triggerFunctions[triggerType]; @@ -124,9 +125,7 @@ async function createTrigger( postgres: wmill.createPostgresTrigger, mqtt: wmill.createMqttTrigger, sqs: wmill.createSqsTrigger, - gcp: async (args) => { - throw new Error("GCP triggers are not supported yet"); - }, + gcp: wmill.createGcpTrigger, email: wmill.createEmailTrigger, }; const triggerFunction = triggerFunctions[triggerType]; From 0594257a25d1a64e620daf4ea0b106f47424ef87 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 4 Dec 2025 22:09:25 +0000 Subject: [PATCH 06/25] fix(frontend): update workers page ui (#7264) * Improve workers page * Update group config drawer * improve dirty workergroup config * Make layout reactive * fix section animation * prevent opening dropdown while clicking New group config * migrate workers page to svelte 5 * Open drawer upon adding a worker group * nit critical alert table * improve queue metrics drawer * improve agent worker drawer * harmonize copy icon * improve agent worker doc * improve layout * Improve autoscaling event list * Improve tags managment * Remove default tags * fix npm check * Add info for agent workers * improve agent worker jwt token creation * Improve token display * nit * improve tag display * create EE component * nit * harmonize tag overflow * handle permission better * improve env var presets * handle permission for config * nit alerts * nit * Improve custom tag creation in tag select * optimistic tag addition * nit * nit * fix typo * improve workers table * Group config tags * show mismatch * fix typo * optimistic update when adding tag * do not allow to create tag when picking a tag to watch in alerts --- .../lib/components/AssignableTagsInner.svelte | 154 +- .../components/AutoscalingConfigEditor.svelte | 771 +++++----- .../lib/components/AutoscalingEvents.svelte | 106 +- .../src/lib/components/CenteredPage.svelte | 17 +- .../src/lib/components/CollapseLink.svelte | 2 +- .../src/lib/components/DefaultTags.svelte | 39 - .../lib/components/DefaultTagsInner.svelte | 192 ++- .../src/lib/components/DropdownV2Inner.svelte | 8 + frontend/src/lib/components/EEOnly.svelte | 33 + .../src/lib/components/FlowViewerInner.svelte | 4 +- .../components/HttpAgentWorkerDrawer.svelte | 365 +++-- .../src/lib/components/InstanceSetting.svelte | 32 +- frontend/src/lib/components/Label.svelte | 9 + .../lib/components/ManageTagsDrawer.svelte | 71 + .../src/lib/components/OccupancyBars.svelte | 46 + .../src/lib/components/QueueAlerts.svelte | 572 +++++++ .../lib/components/QueueMetricsDrawer.svelte | 413 +----- .../components/QueueMetricsDrawerInner.svelte | 147 +- .../src/lib/components/ScriptBuilder.svelte | 9 +- .../src/lib/components/ScriptEditor.svelte | 4 +- frontend/src/lib/components/Section.svelte | 47 +- frontend/src/lib/components/Subsection.svelte | 8 +- frontend/src/lib/components/TagList.svelte | 55 + .../src/lib/components/TagsToListenTo.svelte | 68 +- .../src/lib/components/TooltipInner.svelte | 2 +- .../src/lib/components/WorkerGroup.svelte | 1315 +++++++++-------- .../apps/components/display/AppText.svelte | 4 +- .../apps/editor/AppEditorHeaderDeploy.svelte | 9 +- .../apps/editor/AppExportButton.svelte | 4 +- .../editor/componentsPanel/CssSettings.svelte | 8 +- .../lib/components/common/badge/Badge.svelte | 3 +- .../components/common/button/Button.svelte | 7 +- .../common/drawer/DrawerContent.svelte | 9 +- .../lib/components/common/table/AppRow.svelte | 4 +- .../components/common/table/FlowRow.svelte | 4 +- .../components/details/ClipboardPanel.svelte | 4 +- .../details/CopyableCodeBlock.svelte | 31 +- .../flows/common/FlowCardHeader.svelte | 2 +- .../content/FlowModuleWorkerTagSelect.svelte | 2 +- .../flows/content/FlowRetries.svelte | 8 +- .../flows/content/FlowSettings.svelte | 16 +- .../components/meltComponents/Popover.svelte | 25 + .../src/lib/components/runs/RunsQueue.svelte | 7 +- .../lib/components/select/MultiSelect.svelte | 11 +- .../components/settings/TokenDisplay.svelte | 128 +- .../sidebar/CriticalAlertTable.svelte | 8 +- .../components/sidebar/SidebarContent.svelte | 4 +- .../components/text_input/TextInput.svelte | 2 +- .../webhook/WebhooksConfigSection.svelte | 6 +- frontend/src/lib/utils.ts | 1 + .../(root)/(logged)/workers/+page.svelte | 895 ++++++----- 51 files changed, 3361 insertions(+), 2330 deletions(-) delete mode 100644 frontend/src/lib/components/DefaultTags.svelte create mode 100644 frontend/src/lib/components/EEOnly.svelte create mode 100644 frontend/src/lib/components/ManageTagsDrawer.svelte create mode 100644 frontend/src/lib/components/OccupancyBars.svelte create mode 100644 frontend/src/lib/components/QueueAlerts.svelte create mode 100644 frontend/src/lib/components/TagList.svelte diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index 4b59d56539..cfca54bc99 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -8,8 +8,16 @@ import { superadmin, devopsRole } from '$lib/stores' import NoWorkerWithTagWarning from './runs/NoWorkerWithTagWarning.svelte' import { CUSTOM_TAGS_SETTING } from '$lib/consts' - import { base } from '$lib/base' import { createEventDispatcher } from 'svelte' + import TextInput from './text_input/TextInput.svelte' + import { twMerge } from 'tailwind-merge' + import Badge from './common/badge/Badge.svelte' + + interface Props { + variant?: 'popover' | 'drawer' + } + + let { variant = 'popover' }: Props = $props() let newTag: string = $state('') let customTags: string[] | undefined = $state(undefined) @@ -58,16 +66,48 @@ }) loadCustomTags() + + function onKeyDown(e: KeyboardEvent) { + if (e.key === 'Enter' && newTag.trim() !== '' && tagEditor) { + e.stopPropagation() + e.preventDefault() + saveCustomTag(newTag) + } + } + + async function saveCustomTag(tag: string, restoreCustomTags: boolean = false) { + try { + await SettingService.setGlobal({ + key: CUSTOM_TAGS_SETTING, + requestBody: { value: [...(customTags ?? []), tag.trim().replaceAll(' ', '_')] } + }) + dispatch('refresh') + loadCustomTags() + sendUserToast(restoreCustomTags ? 'Tag restored' : 'Tag added') + if (!restoreCustomTags) { + newTag = '' + } + } catch (err) { + sendUserToast(`Could not ${restoreCustomTags ? 'restore' : 'save'} custom tag: ${err}`, true) + } + } -
+ + +
{#if customTags == undefined} {:else} -
+
{#each customTags as customTag} -
{customTag}
+ + {customTag} + {#if tagEditor} {/if} -
+ {/each}
- + +
+ + +
{#if extractedCustomTag} -
+
Workspace specific tag
Tag: @@ -140,55 +201,40 @@ {/if} {/if} - - Configure worker groups to listen to tags - For tags specific to some workspaces, use 
tag(workspace1+workspace2)
 - To exclude 'workspace1' and 'workspace2' from a tag, use 
tag(^workspace1^workspace2)
 - For tag(workspace1+workspace2) +
{#if variant !== 'drawer'}
{/if} + To exclude 'workspace1' and 'workspace2' from a tag, use + 
tag(^workspace1^workspace2)
+
{#if variant !== 'drawer'}
{/if} + For + dynamic tagsdynamic tags - based on the workspace, use 
$workspace
, e.g: - 
tag-$workspace
 - For $workspace, e.g: + 
tag-$workspace

+ {#if variant !== 'drawer'}
{/if} + + For + dynamic tagsdynamic tags - based on args input, use 
$args[a.b.c]
where - 
a.b.c
is the path to the value in the args object + based on args input, use 
$args[a.b.c]
where + 
a.b.c
is the path to the value in the args object. + {/if}
diff --git a/frontend/src/lib/components/AutoscalingConfigEditor.svelte b/frontend/src/lib/components/AutoscalingConfigEditor.svelte index 37e8dcf4c4..82f260f502 100644 --- a/frontend/src/lib/components/AutoscalingConfigEditor.svelte +++ b/frontend/src/lib/components/AutoscalingConfigEditor.svelte @@ -5,26 +5,35 @@ import Tooltip from './Tooltip.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { Button } from './common' + import { Alert, Button } from './common' import { ExternalLink } from 'lucide-svelte' - import { createEventDispatcher } from 'svelte' + import TextInput from './text_input/TextInput.svelte' import Label from './Label.svelte' import MultiSelect from './select/MultiSelect.svelte' import { safeSelectItems } from './select/utils.svelte' import { ConfigService } from '$lib/gen' + import Select from './select/Select.svelte' + import ScriptPicker from './ScriptPicker.svelte' + import Badge from './common/badge/Badge.svelte' interface Props { config: AutoscalingConfig | undefined worker_tags: string[] | undefined + disabled: boolean } - let { config = $bindable(), worker_tags }: Props = $props() - - const dispatch = createEventDispatcher() + let { config = $bindable(), worker_tags, disabled }: Props = $props() let test_input: number = $state(3) let healthCheckLoading: boolean = $state(false) let healthCheckResult: { success: boolean; error?: string } | null = $state(null) + function validateMinMax(): string | undefined { + if (config?.min_workers && config?.max_workers && config.min_workers > config.max_workers) { + return 'Minimum cannot be greater than maximum' + } + return undefined + } + async function checkKubernetesHealth() { if (!config?.integration || config.integration.type !== 'kubernetes') return @@ -43,366 +52,258 @@ healthCheckLoading = false } } + + let collapsed: boolean = $state(true) -
-
-
Rules
- { - dispatch('dirty') - if (e.detail) { - if (!config) { - config = { - enabled: true, - min_workers: 3, - max_workers: 10, - integration: { type: 'dryrun' } +
+ {#snippet labelExtra()} + Beta + {/snippet} + {#snippet header()} +
+ { + if (e.detail) { + collapsed = false + if (!config) { + config = { + enabled: true, + min_workers: 3, + max_workers: 10, + integration: { type: 'dryrun' } + } + } else { + config.enabled = true } } else { - config.enabled = true + config = { + ...(config ?? { + min_workers: 3, + max_workers: 10, + integration: { type: 'dryrun' } + }), + enabled: false + } } - } else { - config = { - ...(config ?? { - min_workers: 3, - max_workers: 10, - integration: { type: 'dryrun' } - }), - enabled: false - } - } - }} - /> - -
+ {/snippet} + +
+ - - + -
-
-
- - - - - - + +
- - - - - - - - +
-
-
-
-
Integration
- {#if config?.integration} - dispatch('dirty')} - bind:selected={config.integration.type} - class="mb-4 mt-2" - > - {#snippet children({ item })} - - - - - - {/snippet} - - {#if config.integration.type === 'script'} - - - - -
- - -
-
- -
- - workers -
-
- {/if} - - {#if config.integration.type === 'kubernetes'} -
- Kubernetes configuration is automatically inferred from the cluster environment. The - worker group name and namespace will be detected automatically. -
- -
-
- - -
- - {#if healthCheckResult !== null} -
- {#if healthCheckResult.success} - Kubernetes autoscaling is healthy - {:else} - {healthCheckResult.error} - {#if healthCheckResult.error?.includes('permissions') || healthCheckResult.error?.includes('role')} -
Please follow the setup guide above to configure proper RBAC permissions.Create from template + {disabled} + + {:else} + {/if} +
+ Script must be in the 'admins' workspace + + +
+
+ +
+ {:else if config.integration.type === 'kubernetes'} +
+
+ Kubernetes configuration is automatically inferred from the cluster environment. The + worker group name and namespace will be detected automatically. +
+ +
+
+
+ + +
+ +
+ +
+ + workers +
+
+
+ + {#if healthCheckResult !== null} + + {#if healthCheckResult.success} + Kubernetes autoscaling is healthy + {:else} + {healthCheckResult.error} + {#if healthCheckResult.error?.includes('permissions') || healthCheckResult.error?.includes('role')} +
Please follow the setup guide above to configure proper RBAC permissions. + {/if} + {/if} + {/if}
- {/if} - -
- -
- - workers -
-
- {/if} + {:else if config.integration.type === 'dryrun'} +
+ + In dry run mode, autoscaling will be simulated and events will be logged but no actual + scaling will be performed. + +
+ {/if} +
{:else} {#snippet children({ item })} @@ -414,10 +315,186 @@ {/snippet} - + {/if} -
-
+ + +
+ + + + + + + + +
+ + + +
+ + + + + +
+ diff --git a/frontend/src/lib/components/AutoscalingEvents.svelte b/frontend/src/lib/components/AutoscalingEvents.svelte index 7a9a6e313f..47626dbb44 100644 --- a/frontend/src/lib/components/AutoscalingEvents.svelte +++ b/frontend/src/lib/components/AutoscalingEvents.svelte @@ -1,11 +1,14 @@ -
-
Autoscaling events {#if $enterpriseLicense}(5 last) - -
+
+
+
+

Autoscaling events

+ {#if $enterpriseLicense && events && events.length > 0} + Showing last {Math.min(limit, events.length)} + {/if} +
+ {#if $enterpriseLicense} +
+ {#if !$enterpriseLicense} -
Autoscaling is an EE feature
+
Autoscaling is an EE feature
{:else if loading} {:else if events} {#if events.length == 0} -
No events, is autoscaling set in the worker group config?
- {:else} -
- {#each events as event} -
-
{event.event_type} to {event.desired_workers}
-
{event.reason}
-
-
- {/each} +
+ No events. Is autoscaling configured in the worker group config?
+ {:else} + + + + Event type + Desired workers + Reason + Time + + + + {#each events as event} + + {event.event_type ?? 'N/A'} + {event.desired_workers} + {event.reason ?? 'N/A'} + + + + + {/each} + + + + {#if events.length >= limit && limit < 100} +
+ +
+ {/if} {/if} -
- -
+ {#if limit > 50} -
- Note that autoscaling events are only stored for the last 30 days. +
+ Note: Autoscaling events are only stored for the last 30 days.
{/if} {/if} diff --git a/frontend/src/lib/components/CenteredPage.svelte b/frontend/src/lib/components/CenteredPage.svelte index b6b85933d6..79143e5266 100644 --- a/frontend/src/lib/components/CenteredPage.svelte +++ b/frontend/src/lib/components/CenteredPage.svelte @@ -1,9 +1,18 @@ -
-
- -
+
{@render children?.({ width })}
diff --git a/frontend/src/lib/components/CollapseLink.svelte b/frontend/src/lib/components/CollapseLink.svelte index a723eea703..09cf94f301 100644 --- a/frontend/src/lib/components/CollapseLink.svelte +++ b/frontend/src/lib/components/CollapseLink.svelte @@ -11,7 +11,7 @@
- {/snippet} - {#snippet content()} - - {/snippet} - diff --git a/frontend/src/lib/components/DefaultTagsInner.svelte b/frontend/src/lib/components/DefaultTagsInner.svelte index 610da15112..ab2d403ec8 100644 --- a/frontend/src/lib/components/DefaultTagsInner.svelte +++ b/frontend/src/lib/components/DefaultTagsInner.svelte @@ -1,21 +1,40 @@ -
- {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
- {/if} +
+
+ Jobs that have not been specifically assigned custom tags will use a default tags based on the language they are in or their kind. +
+ + {#snippet action()} + {#if !$enterpriseLicense} + Read only + {:else} + + {/if} + {/snippet} + {#if defaultTagPerWorkspace == undefined || defaultTags == undefined} - {:else} -
- {#each defaultTags.sort() as tag (tag)} -
{tag}
→ -
-
+ {:else if !$enterpriseLicense} + +
+ {#each $state.snapshot(defaultTags).sort() as tag (tag)} + {defaultTagPerWorkspace ? `${tag}-$workspace` : tag} {/each}
+ {:else} +
- +
+ +
{#if defaultTagPerWorkspace} - + {#if limitToWorkspaces} {/if} {/if}
- - When tags use 
$workspace
, the final tag has - 
$workspace
replaced with the workspace id, allowing multi-vpc setup with - more ease, without having to assign a specific tag each time. +
+
Job language or kind
+
+
Default tag
+
+ + +
+ {#each $state.snapshot(defaultTags).sort() as tag (tag)} +
+
+ {tag} +
+ +
→
+
+ {defaultTagPerWorkspace ? `${tag}-$workspace` : tag} +
+
+ {/each} +
{/if} -
+
diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index bbf97a35ec..ebb12077d1 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -4,6 +4,7 @@ import { twMerge } from 'tailwind-merge' import type { MenubarMenuElements } from '@melt-ui/svelte' import type { Item } from '$lib/utils' + import { Tooltip } from './meltComponents' interface Props { aiId?: string @@ -53,6 +54,13 @@ {item.displayName}

{@render item.extra?.()} + {#if item.tooltip} + + {#snippet text()} + {item.tooltip} + {/snippet} + + {/if} {/each}
diff --git a/frontend/src/lib/components/EEOnly.svelte b/frontend/src/lib/components/EEOnly.svelte new file mode 100644 index 0000000000..0bf54b7b68 --- /dev/null +++ b/frontend/src/lib/components/EEOnly.svelte @@ -0,0 +1,33 @@ + + + + + {#snippet text()} + {#if children} + {@render children()} + {:else} + Enterprise Edition only feature + {/if} + {/snippet} + diff --git a/frontend/src/lib/components/FlowViewerInner.svelte b/frontend/src/lib/components/FlowViewerInner.svelte index 7fab797408..07052a42d4 100644 --- a/frontend/src/lib/components/FlowViewerInner.svelte +++ b/frontend/src/lib/components/FlowViewerInner.svelte @@ -5,7 +5,7 @@ import { Tab, Tabs, Button } from './common' import { copyToClipboard } from '../utils' - import { ArrowDown, Clipboard } from 'lucide-svelte' + import { ArrowDown, Copy } from 'lucide-svelte' import YAML from 'yaml' import { yaml } from 'svelte-highlight/languages' import HighlightTheme from './HighlightTheme.svelte' @@ -80,7 +80,7 @@ color="light" variant="border" size="xs" - startIcon={{ icon: Clipboard }} + startIcon={{ icon: Copy }} btnClasses="absolute top-2 right-2 w-min z-20" iconOnly /> diff --git a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte index fd0a14dab7..ed6abfda70 100644 --- a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte +++ b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte @@ -1,12 +1,18 @@ {#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null) && !(setting.hiddenIfEmpty && !$values[setting.key])} {#if setting.ee_only != undefined && !$enterpriseLicense} -
- - EE only {#if setting.ee_only != ''}{setting.ee_only}{/if} -
+ + {#if setting.ee_only != ''}{setting.ee_only}{/if} + {/if} {#if setting.fieldType == 'select'}
@@ -571,13 +561,17 @@ {@const currentTeam = $values['critical_error_channels'][i]?.teams_channel ? { team_id: $values['critical_error_channels'][i]?.teams_channel?.team_id, - team_name: $values['critical_error_channels'][i]?.teams_channel?.team_name + team_name: + $values['critical_error_channels'][i]?.teams_channel?.team_name } : undefined} - {@const currentChannel = $values['critical_error_channels'][i]?.teams_channel?.channel_id + {@const currentChannel = $values['critical_error_channels'][i]?.teams_channel + ?.channel_id ? { - channel_id: $values['critical_error_channels'][i]?.teams_channel?.channel_id, - channel_name: $values['critical_error_channels'][i]?.teams_channel?.channel_name + channel_id: + $values['critical_error_channels'][i]?.teams_channel?.channel_id, + channel_name: + $values['critical_error_channels'][i]?.teams_channel?.channel_name } : undefined}
diff --git a/frontend/src/lib/components/Label.svelte b/frontend/src/lib/components/Label.svelte index 42474e5b77..e00ee53cdc 100644 --- a/frontend/src/lib/components/Label.svelte +++ b/frontend/src/lib/components/Label.svelte @@ -2,6 +2,8 @@ import { twMerge } from 'tailwind-merge' import Required from './Required.svelte' import Tooltip from './Tooltip.svelte' + import { enterpriseLicense } from '$lib/stores' + import EEOnly from './EEOnly.svelte' interface Props { label?: string | undefined @@ -12,6 +14,7 @@ class?: string | undefined for?: string | undefined tooltip?: string | undefined + eeOnly?: boolean header?: import('svelte').Snippet error?: import('svelte').Snippet action?: import('svelte').Snippet @@ -27,6 +30,7 @@ class: clazz = undefined, for: forAttr = undefined, tooltip = undefined, + eeOnly = false, header, error, action, @@ -49,6 +53,11 @@ {tooltip} {/if} + {#if eeOnly} + {#if !$enterpriseLicense} + + {/if} + {/if} {@render header?.()}
{/if} diff --git a/frontend/src/lib/components/ManageTagsDrawer.svelte b/frontend/src/lib/components/ManageTagsDrawer.svelte new file mode 100644 index 0000000000..f6fee2fe38 --- /dev/null +++ b/frontend/src/lib/components/ManageTagsDrawer.svelte @@ -0,0 +1,71 @@ + + + + drawer?.closeDrawer?.()}> +
+ +
+ Tags determine which worker group will execute a given job. Workers process only those jobs + whose tags match those defined in their worker group + configuration. +
+ + +
+ +
+ { + if (onRefresh) { + onRefresh() + } + }} + /> +
+ + + + + +
+
+
+ + diff --git a/frontend/src/lib/components/OccupancyBars.svelte b/frontend/src/lib/components/OccupancyBars.svelte new file mode 100644 index 0000000000..7efbf8433e --- /dev/null +++ b/frontend/src/lib/components/OccupancyBars.svelte @@ -0,0 +1,46 @@ + + +
+ {#each rates as rate} + +
+ {#if rate.value !== undefined && rate.value > 0} + {@const heightPercent = Math.min(rate.value * 100, 100)} + {@const minHeight = heightPercent > 0 && heightPercent < 3 ? 1 : heightPercent} +
+ {/if} +
+ {#snippet text()} + {rate.label}: {rate.value ? displayOccupancyRate(rate.value) : '--'} + {/snippet} + + {/each} +
diff --git a/frontend/src/lib/components/QueueAlerts.svelte b/frontend/src/lib/components/QueueAlerts.svelte new file mode 100644 index 0000000000..1f147f52dd --- /dev/null +++ b/frontend/src/lib/components/QueueAlerts.svelte @@ -0,0 +1,572 @@ + + +
+ {#snippet action()} + {#if $enterpriseLicense} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+

Add queue alert

+ +
+ + + Tags that identify which workers to monitor for this alert + +
+ + {#if newAlertForm.tags_to_monitor.length === 0} + + {/if} +
+ {#if formErrors.tags_to_monitor} + {formErrors.tags_to_monitor} + {/if} +
+ +
+ + + Trigger alert when queue exceeds this many jobs + + + {#if formErrors.jobs_num_threshold} + {formErrors.jobs_num_threshold} + {/if} +
+ +
+ + + Wait time between alerts for the same condition + + + {#if formErrors.alert_cooldown_seconds} + {formErrors.alert_cooldown_seconds} + {/if} +
+ +
+ + + How long the condition must persist before alerting + + + {#if formErrors.alert_time_threshold_seconds} + {formErrors.alert_time_threshold_seconds} + {/if} +
+ +
+ + +
+
+ {/snippet} + + {/if} + {/snippet} + + {#if !$enterpriseLicense} +
+ Queue Metric Alerts is an enterprise feature allowing you to monitor queues for waiting jobs. + Please upgrade to access this functionality. Learn more about our plans +
+ {:else if queueAlertConfig.length === 0} +
+

No queue alerts configured

+

Add your first alert to monitor queue conditions

+
+ {:else} +
+ + + + + + + + + + + + {#each queueAlertConfig as config, index} + + + + + + + + {/each} + +
+ + Worker Tags + Tags that identify which workers to monitor for this alert + + + + Jobs Threshold + Trigger alert when queue exceeds this many jobs + + + + Cooldown (s) + Wait time between alerts for the same condition + + + + Time Threshold (s) + How long the condition must persist before alerting + + Actions
+ {#if editingRowIndex === index} +
+ { + if (!editForm.tags_to_monitor.includes(tag)) { + editForm.tags_to_monitor = [...editForm.tags_to_monitor, tag] + } + }} + createText="Press Enter to add custom tag" + placeholder="Select or create tags..." + class="flex-1" + /> + {#if editForm.tags_to_monitor.length === 0} + + {/if} +
+ {:else} + {@const isExpanded = expandedTagRows.includes(index)} + {@const tagsToShow = + isExpanded || config.tags_to_monitor.length <= MAX_NUMBER_OF_TAGS_DISPLAYED + ? config.tags_to_monitor + : config.tags_to_monitor.slice(0, MAX_NUMBER_OF_TAGS_DISPLAYED)} + +
+ {#each tagsToShow as tag} + + {tag} + + {/each} + + {#if config.tags_to_monitor.length > MAX_NUMBER_OF_TAGS_DISPLAYED && !isExpanded} + { + expandedTagRows = [...expandedTagRows, index] + }} + > + + {config.tags_to_monitor.length} + + {/if} +
+ {/if} + 		+ {#if editingRowIndex === index} + + {:else} + {config.jobs_num_threshold} + {/if} + + {#if editingRowIndex === index} + + {:else} + {config.alert_cooldown_seconds} + {/if} + + {#if editingRowIndex === index} + + {:else} + {config.alert_time_threshold_seconds} + {/if} + + {#if editingRowIndex === index} +
+ + +
+ {:else} +
+ + +
+ {/if} +
+
+ {/if} +
diff --git a/frontend/src/lib/components/QueueMetricsDrawer.svelte b/frontend/src/lib/components/QueueMetricsDrawer.svelte index dd4ad426fa..4b05c41899 100644 --- a/frontend/src/lib/components/QueueMetricsDrawer.svelte +++ b/frontend/src/lib/components/QueueMetricsDrawer.svelte @@ -1,423 +1,26 @@ - + -
- {#if $enterpriseLicense} - {#if changesMade} -
Non applied changes
- {/if} -
- - -
+ - {#if alerts.length > 0} -
-
- - - - - - - - - - - - {#each alerts as alert, index} - - - - - - - - {/each} - -
- Queue Tags to Monitor - - - Jobs - - - Cooldown (s) - - - Time (s) - - Actions
- {#if editingIndex === index} -
- {#each alert.tags_to_monitor as tag} - - {tag} - - - {/each} -
-
- filterTags(e)} - disabled={workerTags.length === alert.tags_to_monitor.length} - class="p-1 flex-grow mr-1" - /> - -
- - - {#if filteredTags.length > 0} -
    - {#each filteredTags as tag} - {#if !alert.tags_to_monitor.includes(tag)} -
  • - -
    • - {/if} - {/each} -
- {/if} - {:else} -
- {#each alert.tags_to_monitor as tag} - {tag} - {/each} -
- {/if} - 		- {#if editingIndex === index} - - {:else} - {alert.jobs_num_threshold} - {/if} - - {#if editingIndex === index} - - {:else} - {alert.alert_cooldown_seconds} - {/if} - - {#if editingIndex === index} - - {:else} - {alert.alert_time_threshold_seconds} - {/if} - -
- {#if editingIndex === index} - - {:else} - - - {/if} -
-
-
-
- {/if} +
- -
- -
- {:else} -

- Queue Metric Alerts are an enterprise feature allowing you to monitor queues for waiting - jobs. Please upgrade to access this functionality. - Learn more about our plans. -

- {/if} -
-

Queue Metrics

-
- -
+ + +
diff --git a/frontend/src/lib/components/QueueMetricsDrawerInner.svelte b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte index 76e0b0c72b..f6cc720b88 100644 --- a/frontend/src/lib/components/QueueMetricsDrawerInner.svelte +++ b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte @@ -20,6 +20,7 @@ import Skeleton from './common/skeleton/Skeleton.svelte' import DarkModeObserver from './DarkModeObserver.svelte' import Alert from './common/alert/Alert.svelte' + import { Section } from './common' let loading: boolean = true @@ -186,87 +187,89 @@ -{#if loading} - -{:else if noMetrics} -

No jobs delayed by more than 3 seconds in the last 14 days

-{:else} -
- {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { +
+ {#if loading} + + {:else if noMetrics} +

No jobs delayed by more than 3 seconds in the last 14 days

+ {:else} +
+ {#if countData} + 3s)' } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + y: { + title: { + display: true, + text: 'count' } } } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', + }} + /> + {/if} + {#if delayData} + 3s)' }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) + tooltip: { + callbacks: { + label: function (context) { + // @ts-ignore + if (context.raw.y === 1) { + return context.dataset.label + ': 0' + } else { + // @ts-ignore + return context.dataset.label + ': ' + context.raw.y + } + } + } + } + }, + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + + y: { + type: 'logarithmic', + title: { + display: true, + text: 'delay (s)' + }, + ticks: { + callback: (value, _) => (value === 1 ? '0' : value) + } } } - } - }} - /> - {/if} - - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. - -
-{/if} + }} + /> + {/if} + + Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are + included in the graph. + +
+ {/if} + diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index fd2326cb23..8d2b022e3b 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -12,7 +12,7 @@ PostgresTriggerService, CaptureService, type ScriptLang, - WorkerService, + WorkerService } from '$lib/gen' import { inferArgs } from '$lib/infer' import { @@ -189,7 +189,7 @@ : undefined ) const simplifiedPoll = writable(false) - + export function setPrimarySchedule(schedule: ScheduleTrigger | undefined | false) { primaryScheduleStore.set(schedule) loadTriggers() @@ -1046,7 +1046,7 @@ {#snippet content()}
-
+
{#snippet action()} {#if customUi?.settingsPanel?.metadata?.disableMute !== true} @@ -1214,7 +1214,7 @@
-
+
{#snippet header()}
{/if} -
{#if $enterpriseLicense && initialPath != ''} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index e77d91ca63..d042f3ec03 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -23,7 +23,7 @@ import Modal from './common/modal/Modal.svelte' import DiffEditor from './DiffEditor.svelte' import { - Clipboard, + Copy, CornerDownLeft, ExternalLink, Github, @@ -556,7 +556,7 @@ {:else} {label} + {@render labelExtra?.()} {/if} {@render header?.()} @@ -79,10 +84,7 @@ {/if} {#if eeOnly} {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
+ {/if} {/if} @@ -94,10 +96,17 @@ {/if} {#if !collapsable || !collapsed}
- {@render children?.()} + {#if description} +
{description}
+ {/if} +
+
+ {@render children?.()} +
+
{/if}
diff --git a/frontend/src/lib/components/Subsection.svelte b/frontend/src/lib/components/Subsection.svelte index b40e78e83e..bee078cf44 100644 --- a/frontend/src/lib/components/Subsection.svelte +++ b/frontend/src/lib/components/Subsection.svelte @@ -1,8 +1,9 @@ + +{#if tags.length > 0} +
+ + {#each visibleTags as tag (tag)} + + {tag} + + {/each} + + + {#if hasExtraTags} + + {#snippet trigger()} + + +{extraTags.length} + + {/snippet} + {#snippet content()} +
+ {#each extraTags as tag (tag)} + {tag} + {/each} +
+ {/snippet} + + {/if} +
+{:else} + No tags +{/if} diff --git a/frontend/src/lib/components/TagsToListenTo.svelte b/frontend/src/lib/components/TagsToListenTo.svelte index e99c67223b..f90b045438 100644 --- a/frontend/src/lib/components/TagsToListenTo.svelte +++ b/frontend/src/lib/components/TagsToListenTo.svelte @@ -1,37 +1,75 @@ worker_tags, - (w) => ((worker_tags = w.map((s) => s.replaceAll(' ', '_'))), dispatch('dirty')) - } + bind:this={multiSelect} + bind:value={() => worker_tags, (w) => (worker_tags = w.map((s) => s.replaceAll(' ', '_')))} {disabled} - class={disabled ? 'border-0' : ''} + class={twMerge(disabled ? 'border-0' : '', clazz)} allowClear={!disabled} - onCreateItem={(c) => { - worker_tags.push(c) - dispatch('dirty') - }} - createText="Press Enter to use this tag" + onCreateItem={createCustomTag} + createText={searchText ? `Create custom tag: ${searchText}` : 'Create custom tag'} /> diff --git a/frontend/src/lib/components/TooltipInner.svelte b/frontend/src/lib/components/TooltipInner.svelte index b8c435f167..28dab0595f 100644 --- a/frontend/src/lib/components/TooltipInner.svelte +++ b/frontend/src/lib/components/TooltipInner.svelte @@ -8,7 +8,7 @@
{#if markdownTooltip}
diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index d22e8c8539..0626dffabc 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -1,17 +1,32 @@ drawer?.closeDrawer()} - title={$superadmin || $devopsRole ? `Edit worker config '${name}'` : `Worker config '${name}'`} + title={canEditConfig ? `Edit worker config '${name}'` : `Worker config '${name}'`} + eeOnly > - {#if !$enterpriseLicense} - + {#if !hasEnterpriseFeatures} + Workers can still have their WORKER_TAGS, INIT_SCRIPT and WHITELIST_ENVS passed as env. Dedicated workers are an enterprise only feature. -
+ {:else if !canEditConfig} + + Only superadmin or devops role can edit the worker config. + {/if} + - { - dirty = true - if (nconfig == undefined) { - nconfig = {} - } - if (e.detail == 'dedicated') { - nconfig.dedicated_worker = '' - nconfig.worker_tags = undefined - } else { - nconfig.dedicated_worker = undefined - nconfig.worker_tags = [] - } - }} - class="mb-4" - > - {#snippet children({ item })} - - - {/snippet} - +
{#if selected == 'normal'} -
- {#if nconfig?.worker_tags != undefined} - { - dirty = true - }} - on:deletePriorityTag={(e) => { - const tag = e.detail - if (nconfig.priority_tags) { - delete nconfig.priority_tags[tag] - } - }} - bind:worker_tags={nconfig.worker_tags} - {customTags} - /> - -
- - - - - {#if defaultTagPerWorkspace} - ({ value: w.id }))} + onCreateItem={(c) => (workspaceTag = c)} + placeholder="Workspace ID" + disabled={!canEditConfig} /> - - {/if} -
+ {/if} +
+ {/if} + {/snippet} + {#if nconfig?.worker_tags != undefined} + {/if} - + + + {#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0} +
+ + {/if} + {#if nconfig !== undefined}
-
+
+ {/if} {:else if selected == 'dedicated'}
@@ -429,17 +505,15 @@ {/if} {#if nconfig?.dedicated_worker != undefined}

Workers will get killed upon detecting changes. It is assumed they are in an environment where the supervisor will restart them.

{ - dirty = true - }} + onchange={() => {}} bind:value={nconfig.dedicated_worker} />
@@ -449,52 +523,51 @@
-
-
+
{#each customEnvVars as envvar, i}
{ - dirty = true - }} + onkeypress={(e) => {}} /> { - dirty = true if (e.detail === 'dynamic') { envvar.value = undefined } }} > {#snippet children({ item })} - - + + {/snippet} - - {#if $superadmin || $devopsRole} - + startIcon={{ icon: Trash }} + iconOnly + destructive + disabled={!canEditConfig} + /> {/if}
{/each} - {#if $superadmin || $devopsRole} -
+ {#if canEditConfig} +
+ + + Set up env variables for AWS or SSL using our + { + let updated = false + aws_env_vars_preset.forEach((envvar) => { + if (!customEnvVars.some((e) => e.key === envvar)) { + updated = true + customEnvVars.push({ + key: envvar, + type: 'dynamic', + value: undefined + }) + } + }) + if (updated) { + customEnvVars = [...customEnvVars] + } + } + }, + { + displayName: `SSL (${ssl_env_vars_preset.join(', ')})`, + action: () => { + let updated = false + ssl_env_vars_preset.forEach((envvar) => { + if (!customEnvVars.some((e) => e.key === envvar)) { + updated = true + customEnvVars.push({ + key: envvar, + type: 'dynamic', + value: undefined + }) + } + }) + if (updated) { + customEnvVars = [...customEnvVars] + } + } + } + ]} + > + {#snippet buttonReplacement()} + + {/snippet} + +
{/if}
- {#if !($superadmin || $devopsRole)} -
- + {/if} +
+ {/each} + {/if} + {#if canEditConfig} +
+ - +
+ {/if} + +
+ {/if} +
-
- -
- {#snippet header()} -
- Beta - {#if nconfig.autoscaling?.enabled} - Enabled - {/if} -
- {/snippet} - (dirty = true)} - worker_tags={config?.worker_tags} - bind:config={nconfig.autoscaling} - /> -
- -
-
-
- - {#if nconfig.additional_python_paths} - {#each nconfig.additional_python_paths as _, i} -
- - {#if $superadmin || $devopsRole} - - {/if} -
- {/each} - {/if} - {#if $superadmin || $devopsRole} -
- -
- {/if} - - - {#if nconfig.pip_local_dependencies} - {#each nconfig.pip_local_dependencies as _, i} -
- - {#if $superadmin || $devopsRole} - - {/if} -
- {/each} - {/if} - {#if $superadmin || $devopsRole} -
- -
- {/if} - -
- {#if $superadmin || $devopsRole} + {#if canEditConfig}
- + Workers will get killed upon detecting any changes in this section (scripts or interval). It is assumed they are in an environment where the supervisor will restart them. @@ -737,13 +795,14 @@
-
+
Run at start of the workers. More lightweight than requiring custom worker images.
- {#snippet header()}
@@ -755,7 +814,7 @@
{ if (config) { - dirty = true const code = e.detail if (code != '') { nconfig.init_bash = code?.replace(/\r\n/g, '\n') @@ -774,17 +832,18 @@ }} />
- +
-
+
Run periodically at configurable intervals. Useful for maintenance tasks like cleaning disk space.
- {#snippet header()}
@@ -795,24 +854,24 @@ {/snippet}
- - { - dirty = true - }} - /> - Minimum: 60 seconds +
{ if (config) { - dirty = true const code = e.detail if (code != '') { nconfig.periodic_script_bash = code?.replace(/\r\n/g, '\n') @@ -832,215 +890,282 @@ }} />
- +
{#snippet actions()} -
+
- {#if dirty} -
Non applied changes
- {/if} - + await ConfigService.updateConfig({ name: 'worker__' + name, requestBody: nconfig }) + sendUserToast('Configuration set') + dispatch('reload') + }} + disabled={(!hasChanges && nconfig?.dedicated_worker == undefined) || !canEditConfig} + > + Apply changes + + {:else} + Read only + {/if}
{/snippet} -
- {#if hasWorkersWithoutIsolation(workers)} - {@const unsafeWorkers = getWorkersWithoutIsolation(workers)} -
- - {#snippet trigger()} - - {/snippet} - {#snippet content()} -
-
Workers without job isolation
-

- {unsafeWorkers.length} - {unsafeWorkers.length === 1 ? 'worker' : 'workers'} in this group - {unsafeWorkers.length === 1 ? 'is' : 'are'} running without job isolation (nsjail/unshare). -

-
- {#each unsafeWorkers as worker} - - {worker.worker} - - {/each} -
- - Learn more about job isolation → - -
- {/snippet} - -
- {/if} +
+
+
+
+ {#if selectGroup} + {@render selectGroup()} + + {:else} + Worker group: + {name} - + {/if} + + {`${pluralize(activeWorkers, 'worker')} `} + {#snippet text()}Number of active workers of this group in the last 15 seconds{/snippet} +
-
-
{pluralize(activeWorkers, 'worker')} {#if vcpus_memory?.vcpus} - {(vcpus_memory?.vcpus / 100000).toFixed(2)} vCPUs{/if} {#if vcpus_memory?.memory} - - {((vcpus_memory?.memory * 1.0) / 1024 / 1024 / 1024).toFixed(2)} GB{/if}
-
- {#if $superadmin} - - - - - {#if config} - - {/if} + - - {:else if config} - + {#snippet text()} +
+
Workers without job isolation
+

+ {unsafeWorkers.length} + {unsafeWorkers.length === 1 ? 'worker' : 'workers'} in this group + {unsafeWorkers.length === 1 ? 'is' : 'are'} running without job isolation (nsjail/unshare). +

+
+ {#each unsafeWorkers as worker} + + {worker.worker} + + {/each} +
+ + Learn more about job isolation + +
+ {/snippet} + +
{/if}
+ {#if !isAgent} +
+ {#if canEditConfig} + {#if width > 1000} + {#if config} + + {/if} + + + + {#if config} + + {/if} + {:else} + { + navigator.clipboard.writeText(YAML.stringify({ name, ...config })) + sendUserToast('Worker config copied to clipboard as YAML') + }, + disabled: !config + }, + { + displayName: 'Clean cache', + action: () => { + loadNConfig() + openClean = true + }, + disabled: !config, + type: 'delete' + }, + { + displayName: 'Delete config', + action: () => { + if (!hasEnterpriseFeatures) { + sendUserToast('Worker Management UI is an EE feature', true) + } else { + openDelete = true + } + }, + disabled: !config, + type: 'delete' + } + ]} + /> + {/if} + + {:else if config} + + {/if} +
+ {/if}
+ + {#if config?.worker_tags && config.worker_tags.length > 0} +
+
Tags:
+ +
+ {:else if config?.dedicated_worker} +
+
Dedicated to:
+
+ {config.dedicated_worker} +
+
+ {/if}
+ +{#if isAgent} +
+ + {#snippet children()} + This group is formed with agent workers, there is no associated config. {#if $superadmin || $devopsRole} + To modify the tags, generate a new JWT token .{/if} + {/snippet} + +
+{/if} diff --git a/frontend/src/lib/components/apps/components/display/AppText.svelte b/frontend/src/lib/components/apps/components/display/AppText.svelte index 7d1e09bd06..a7f98089f8 100644 --- a/frontend/src/lib/components/apps/components/display/AppText.svelte +++ b/frontend/src/lib/components/apps/components/display/AppText.svelte @@ -3,7 +3,7 @@ const bubble = createBubbler() import Tooltip from '$lib/components/Tooltip.svelte' - import { Clipboard } from 'lucide-svelte' + import { Copy } from 'lucide-svelte' import { getContext, untrack } from 'svelte' import { twMerge } from 'tailwind-merge' import { copyToClipboard, isCodeInjection } from '../../../../utils' @@ -273,7 +273,7 @@ btnClasses="!p-1" on:click={() => copyToClipboard(result)} > - +
{/if} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 9b6f5a9b2f..81f59a5028 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -2,7 +2,7 @@ import { Alert } from '$lib/components/common' import Toggle from '$lib/components/Toggle.svelte' import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' - import { Loader2, AlertTriangle } from 'lucide-svelte' + import { Loader2 } from 'lucide-svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -13,6 +13,7 @@ import { computeSecretUrl } from './appDeploy.svelte' import { base } from '$lib/base' import { isCloudHosted } from '$lib/cloud' + import EEOnly from '$lib/components/EEOnly.svelte' let { policy, @@ -216,11 +217,9 @@
{/if} + {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
+ {/if} { diff --git a/frontend/src/lib/components/apps/editor/AppExportButton.svelte b/frontend/src/lib/components/apps/editor/AppExportButton.svelte index 0bc4cf4b10..2090323725 100644 --- a/frontend/src/lib/components/apps/editor/AppExportButton.svelte +++ b/frontend/src/lib/components/apps/editor/AppExportButton.svelte @@ -7,7 +7,7 @@ import { Highlight } from 'svelte-highlight' import json from 'svelte-highlight/languages/json' import { Button } from '../../common' - import { Clipboard } from 'lucide-svelte' + import { Copy } from 'lucide-svelte' import { yaml } from 'svelte-highlight/languages' import YAML from 'yaml' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -45,7 +45,7 @@ )} variant="accent" size="sm" - startIcon={{ icon: Clipboard }} + startIcon={{ icon: Copy }} btnClasses="absolute top-2 right-2 w-min z-20" iconOnly /> diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte index 6fd86c9ad7..d081549dbd 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte @@ -1,6 +1,6 @@
-
+
{tooltip} {/if} + {#if eeOnly && !$enterpriseLicense} + + {/if}
{#if actions}
diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index 6c184563df..08822ad433 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -23,7 +23,7 @@ Pen, Share, Trash, - Clipboard + Copy } from 'lucide-svelte' import { goto as gotoUrl } from '$app/navigation' import { page } from '$app/stores' @@ -218,7 +218,7 @@ }, { displayName: 'Copy path', - icon: Clipboard, + icon: Copy, action: () => { copyToClipboard(path) } diff --git a/frontend/src/lib/components/common/table/FlowRow.svelte b/frontend/src/lib/components/common/table/FlowRow.svelte index 755d311996..4ed3429618 100644 --- a/frontend/src/lib/components/common/table/FlowRow.svelte +++ b/frontend/src/lib/components/common/table/FlowRow.svelte @@ -28,7 +28,7 @@ Calendar, Share, Archive, - Clipboard, + Copy, Eye, HistoryIcon } from 'lucide-svelte' @@ -211,7 +211,7 @@ }, { displayName: 'Copy path', - icon: Clipboard, + icon: Copy, action: () => { copyToClipboard(path) } diff --git a/frontend/src/lib/components/details/ClipboardPanel.svelte b/frontend/src/lib/components/details/ClipboardPanel.svelte index cb5e0d535f..289c7da416 100644 --- a/frontend/src/lib/components/details/ClipboardPanel.svelte +++ b/frontend/src/lib/components/details/ClipboardPanel.svelte @@ -1,6 +1,6 @@ - - + +
{ + onclick={(e) => { if (disabled) { return } @@ -23,9 +28,13 @@ }} >
- +
-
- +
+
diff --git a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte index c245fd9b41..ad736e40e0 100644 --- a/frontend/src/lib/components/flows/common/FlowCardHeader.svelte +++ b/frontend/src/lib/components/flows/common/FlowCardHeader.svelte @@ -83,7 +83,7 @@
{#if flowModuleValue} diff --git a/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte b/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte index d724f241e8..6f1a1e2db0 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte @@ -42,7 +42,7 @@ /> {:else}
+ {/if} +
-
+ +
{#if mcpUrl} - - - + {:else} - - - + {/if}
-
-

+ +
+ +

{displayTitle}

- -
-
- - -
-
- - {tokenOrUrl} - - -
-
-
+ +
+ + + + +
{info} - {#if mcpUrl} -
-

- Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) to access your Windmill scripts and flows as tools. -

-
- {/if}
- + + {#if mcpUrl} + +
+

+ Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) + to access your Windmill scripts and flows as tools. +

+
+ {/if} +
-

\ No newline at end of file +
diff --git a/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte index edbb3329bd..468acfc423 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte @@ -63,9 +63,9 @@ Acked
{/key} {/key} @@ -452,7 +452,7 @@ done` }} > - +
{/key}{/key}{/key}{/key} {/key} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 14d3719463..853bd8332c 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1343,6 +1343,7 @@ export type Item = { hide?: boolean | undefined extra?: Snippet id?: string + tooltip?: string } export function isObjectTooBig(obj: any): boolean { diff --git a/frontend/src/routes/(root)/(logged)/workers/+page.svelte b/frontend/src/routes/(root)/(logged)/workers/+page.svelte index e127ba9a36..32625a5bd8 100644 --- a/frontend/src/routes/(root)/(logged)/workers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workers/+page.svelte @@ -1,14 +1,14 @@ {#if $superadmin || $devopsRole} + { + loadCustomTags() + }} + /> {/if} @@ -346,7 +448,12 @@ fixedOverflowWidgets={false} /> {#snippet actions()} - + {/snippet} @@ -385,25 +492,27 @@
{:else} - - {#if $superadmin || $devopsRole} -
-
- { - loadCustomTags() - }} + {#snippet children({ width })} + + {#if $superadmin || $devopsRole} +
+
-
- -
-
+ startIcon={{ + icon: Tags + }} + on:click={() => { + manageTagsDrawer?.openDrawer() + }} + > + Manage tags + + -
-
- -
-
- {/if} - - {#if workers != undefined} - {#if groupedWorkers.length == 0} -

No workers seem to be available

- {/if} - -

{groupWorkers?.length} Worker Groups Worker groups are groups of workers that share a config and are meant to be identical. - Worker groups are meant to be used with tags. Tags can be assigned to scripts and flows - and can be seen as dedicated queues. Only the corresponding -

-
- - {#if $superadmin || $devopsRole} -
{ + if ( + e.key === 'Enter' && + newConfigName && + newConfigName.trim() !== '' && + $enterpriseLicense + ) { + addConfig() + } + }} + onClose={() => { + newConfigName = '' + }} + bind:this={newGroupPopover} + targetId="new-group-popover-trigger" > - + {#snippet trigger()}
- - + {/snippet} + {#snippet content()}
{#if !$enterpriseLicense} -
- - EE only -
+ {/if}
- + {/snippet}
- {/if}
+ {/if} + - {#if (groupedWorkers ?? []).length > 5} -
-
Worker group:
- + {#if worker_group} + {@const config = (workerGroups ?? {})[worker_group[0]]} + {@const activeWorkers = worker_group?.[1].flatMap((x) => + x[1]?.filter((y) => (y.last_ping ?? 0) < 15) + )} + + { + loadWorkerGroups() + }} + activeWorkers={activeWorkers?.length ?? 0} + {defaultTagPerWorkspace} + {width} + shouldAutoOpenDrawer={shouldAutoOpenDrawer === worker_group[0]} + onDrawerOpened={() => { + shouldAutoOpenDrawer = '' + }} + onDeleted={handleWorkerGroupDeleted} + selectGroup={(groupedWorkers ?? []).length > 5 ? selectGroup : undefined} /> - -
- {#if worker_group?.[1].length == 0 && search} -
- No workers found. Reset the search to see all workers. + +
+ +
+
Active workers
+
+ + +
- {:else} + Worker - -
- Worker Tags - - If defined, the workers only pull jobs with the same corresponding tag - -
- - Last ping + {#if !config || !config.worker_tags || config.worker_tags.length === 0} + +
+ Worker tags + + If defined, the workers only pull jobs with the same corresponding tag + +
+ + {/if} Worker start Jobs ran {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} @@ -625,10 +724,10 @@ Memory usage
(Windmill) Limits Version - Liveness + Status {#if $superadmin || $devopsRole} - Live Shell + Repl

Open a live shell to execute bash commands on the machine where the worker @@ -639,173 +738,279 @@ {/if} - - {#each worker_group[1] as [section, workers]} - {@const hostname = section?.split(splitter)?.[0]} - - -

-
- Host: - {hostname} -
- IP: - {workers[0].ip} + + {#if worker_group?.[1].length > 0} + {#each worker_group[1] as [section, workers], groupIdx} + {@const hostname = section?.split(splitter)?.[0]} + + +
+
+ Host: + {hostname} +
+ IP: + {workers[0].ip} - {#if workers?.length > 1} - {workers?.length} Workers - {/if} + {#if workers?.length > 1} + {workers?.length} Workers + {/if} +
+ + + {#if workers} + {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }} + {@const isWorkerAlive = isWorkerMaybeAlive(last_ping)} + {@const tagMismatchInfo = getTagMismatchInfo( + custom_tags, + config?.worker_tags + )} + + + {@const underscorePos = worker.search('_')} +
+ + {#if underscorePos === -1} + {worker} + {:else} + {truncate(worker, underscorePos)} + {worker} + {/if} + + {#if config && tagMismatchInfo.hasMismatch} + + + + + + {#snippet text()} +
+
Tag configuration mismatch
+

This worker has tags that differ from the config. This + could be due to a recent update of the config or because the + worker has tags defined as environment variables.

+ + {#if tagMismatchInfo.added.length > 0} +
+
Additional tags:
+ +
+ {/if} + + {#if tagMismatchInfo.removed.length > 0} +
+
Missing tags:
+ +
+ {/if} +
+ {/snippet} + + {/if} +
+ + {#if !config || !config.worker_tags || config.worker_tags.length === 0} + + + + {/if} + {displayDate(started_at)} + {jobs_executed} + {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} + + {#if last_job_id} + + View last job + + +
+ {last_job_workspace_id} + {/if} + + + + + {/if} + +
+
+ {memory_usage + ? Math.round(memory_usage / 1024 / 1024) + 'MB' + : '--'} +
+
+ ({wm_memory_usage + ? Math.round(wm_memory_usage / 1024 / 1024) + 'MB' + : '--'}) +
+
+ + +
+
+ {vcpus ? (vcpus / 100000).toFixed(2) + ' vCPUs' : '--'} +
+
+ {memory ? Math.round(memory / 1024 / 1024) + 'MB' : '--'} +
+
+ + +
+ {wm_version.split('-')[0]} +
+ + + {@const pingText = + last_ping != undefined + ? `${last_ping + timeSinceLastPing}s ago` + : 'Unknown'} + {@const statusText = + isWorkerAlive != undefined + ? isWorkerAlive + ? 'Alive' + : 'Dead' + : 'Unknown'} +
+ + {statusText} + +
{pingText}
+
+ + {#if $superadmin || $devopsRole} + + + + {/if} + + {/each} + {/if} + {/each} + {:else if search} + + +
+ No active workers found matching the search query
- {#if workers} - {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }} - {@const isWorkerAlive = isWorkerMaybeAlive(last_ping)} - - - {@const underscorePos = worker.search('_')} - {#if underscorePos === -1} - {worker} - {:else} - {truncate(worker, underscorePos)} - {worker} - {/if} - - - {#if custom_tags && custom_tags?.length > 2} - {truncate(custom_tags?.join(', ') ?? '', 10)} - {custom_tags?.join(', ')} - {:else} - {custom_tags?.join(', ') ?? ''} - {/if} - - {last_ping != undefined ? last_ping + timeSinceLastPing : -1}s ago - {displayDate(started_at)} - {jobs_executed} - {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} - - {#if last_job_id} - - View last job - -
- (workspace {last_job_workspace_id}) - {/if} - - - {displayOccupancyRate(occupancy_rate_15s)}/{displayOccupancyRate( - occupancy_rate_5m - )}/{displayOccupancyRate(occupancy_rate_30m)}/{displayOccupancyRate( - occupancy_rate - )} - - {/if} - -
-
- {memory_usage ? Math.round(memory_usage / 1024 / 1024) + 'MB' : '--'} -
-
- ({wm_memory_usage - ? Math.round(wm_memory_usage / 1024 / 1024) + 'MB' - : '--'}) -
-
- - -
-
- {vcpus ? (vcpus / 100000).toFixed(2) + ' vCPUs' : '--'} -
-
- {memory ? Math.round(memory / 1024 / 1024) + 'MB' : '--'} -
-
- - -
- {wm_version.split('-')[0]}{wm_version} -
- - - - {isWorkerAlive != undefined - ? isWorkerAlive - ? 'Alive' - : 'Dead' - : 'Unknown'} - - - {#if $superadmin || $devopsRole} - - - - {/if} - - {/each} - {/if} - {/each} + {:else} + + +
+ No active workers found for the group '{worker_group[0]}' +
+ + + {/if} - {/if} - {:else} - {@const worker_group = Object.entries(workerGroups ?? {}) - .filter((x) => !groupedWorkers.some((y) => y[0] == x[0])) - .find((x) => x[0] == selectedTab)} + {:else} + {@const worker_group = Object.entries(workerGroups ?? {}) + .filter((x) => !groupedWorkers.some((y) => y[0] == x[0])) + .find((x) => x[0] == selectedTab)} - {#if worker_group} - { - loadWorkerGroups() - }} - name={worker_group[0]} - config={worker_group[1]} - activeWorkers={0} - /> -
No workers currently in this worker group
+ {#if worker_group} + { + loadWorkerGroups() + }} + name={worker_group[0]} + config={worker_group[1]} + activeWorkers={0} + {width} + shouldAutoOpenDrawer={shouldAutoOpenDrawer === worker_group[0]} + onDrawerOpened={() => { + shouldAutoOpenDrawer = '' + }} + onDeleted={handleWorkerGroupDeleted} + /> +
No workers currently in this worker group
+ {/if} {/if} +
+
+ + {#if worker_group?.[2] === false} + {/if} -
-
- - {:else} -
- {#each new Array(4) as _} - - {/each} -
- {/if} + {:else} +
+ {#each new Array(4) as _} + + {/each} +
+ {/if} + {/snippet} {/if} + +{#snippet selectGroup()} +
+
Worker group + Worker groups are groups of workers that share a config and are meant to be identical. + Worker groups are meant to be used with tags. Tags can be assigned to scripts and flows and + can be seen as dedicated queues. Only the corresponding + +
+ (value = i)} + placeholder="PostgreSQL database name" + items={onlySelectedTags} + disabled={!$isCustomInstanceDbEnabled} + itemButtonWrapperClasses="flex items-center justify-between gap-2" + > + {#snippet endSnippet({ item })} + {#if !customInstanceDbs.current?.[item.value]?.success} +
+ {:else} +
+ {/if} + {/snippet} + + + +
+ + (openedDbNameWizard ? { dbname: value ?? '', status: status! } : undefined), + (v) => !v && (openedDbNameWizard = false) + } +/> diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte new file mode 100644 index 0000000000..9a8a2d443f --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte @@ -0,0 +1,199 @@ + + + !!opened, (v) => !v && !preventClose && (opened = undefined)} + target="#content" + title={'Custom Instance Database Setup'} + contentClasses="flex flex-col" + fixedWidth="md" + fixedHeight="md" +> + {#if opened} + {@const status = opened?.status} + {@const dbname = opened?.dbname} + {@const enableManageButton = + status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'} +
+
+
+ {dbname} + + Custom instance databases are databases created in the Windmill PostgreSQL instance. + Their credentials are automatically managed by Windmill and are never exposed to users. + Only super admins can create them. + +
+
+ {#if bottomHint} +
+ {@render bottomHint()} +
+ {/if} + (opened = undefined)} + /> +
+
+
+
+ {#if status?.error} +
+ + {status.error} + +
+ {/if} + + +
+ +
+
+ {/if} + diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte new file mode 100644 index 0000000000..51e95bc7eb --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -0,0 +1,288 @@ + + + + +
+
+
Data tables
+ + Store relational data out of the box. Interact with a fully managed PostgreSQL database + directly from the Windmill SDK. + +
+
+ + + + + {#each tableHeadNames as name, i} + + {name} + {#if tableHeadTooltips[name]} + + {@html tableHeadTooltips[name]} + + {/if} + + {/each} + + + + {#if tempSettings.dataTables.length == 0} + + + No data table in this workspace yet + + + {/if} + {#each tempSettings.dataTables as dataTable, dataTableIndex} + + + + + +
+
+ {#if dataTable.database.resource_type === 'instance'} + + Use Windmill's PostgreSQL instance + + {/if} + (ducklake.catalog.resource_path = i)} - placeholder="PostgreSQL database name" - items={safeSelectItems(Object.keys(instanceCatalogStatuses.value ?? {}))} - disabled={!isInstanceCatalogEnabled} - /> - - - - - - - {@render instanceCatalogWizard(status, ducklake.catalog.resource_path ?? '')} - - + {#snippet wizardBottomHint()} + Note: this is different from the Manage Ducklake button. This will show you the + content of the PostgreSQL database used as a catalog, while the other button + shows you the content of the ducklake (the parquet files). + {/snippet} + {/if}
@@ -399,142 +367,3 @@ - -{#snippet instanceCatalogWizard( - status: DucklakeInstanceCatalogDbStatus | undefined, - dbname: string -)} - {@const showManageCatalogButton = - status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'} - {#if !status} -
- {dbname} needs to be configured in the Windmill postgres instance -
- {/if} - - {#if status?.error} -
- - {status.error} - -
- {/if} - - - {#if showManageCatalogButton} -
- Note: the 'Manage catalog' button below is different from the Manage Ducklake button. This - will show you the content of the PostgreSQL database used as a catalog, while the other button - shows you the actual content of the ducklake (the parquet files). -
- {/if} -
- - {#if showManageCatalogButton} - instanceCatalogPopover?.close()} - /> - {/if} -
-{/snippet} diff --git a/frontend/src/lib/components/workspaceSettings/utils.svelte.ts b/frontend/src/lib/components/workspaceSettings/utils.svelte.ts new file mode 100644 index 0000000000..3d5d70afec --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/utils.svelte.ts @@ -0,0 +1,9 @@ +import { isCloudHosted } from '$lib/cloud' + +import { superadmin } from '$lib/stores' +import { derived } from 'svelte/store' + +export let isCustomInstanceDbEnabled = derived( + [superadmin], + ([superadmin_]) => superadmin_ && !isCloudHosted() +) diff --git a/frontend/src/lib/svelte5Utils.svelte.ts b/frontend/src/lib/svelte5Utils.svelte.ts index adb09a4809..90de70ba6d 100644 --- a/frontend/src/lib/svelte5Utils.svelte.ts +++ b/frontend/src/lib/svelte5Utils.svelte.ts @@ -2,7 +2,7 @@ import { untrack } from 'svelte' import { deepEqual } from 'fast-equals' -import type { StateStore } from './utils' +import { type StateStore } from './utils' export function withProps(component: Component, props: Props) { const ret = $state({ @@ -39,6 +39,9 @@ export type UsePromiseOptions = { clearValueOnRefresh?: boolean } +/** + * @deprecated Use `resource` from `runed` instead + */ export function usePromise( createPromise: () => Promise, { loadInit = true, clearValueOnRefresh = true }: UsePromiseOptions = {} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 9294ddbf63..465cd78183 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -59,6 +59,10 @@ import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' import CollapseLink from '$lib/components/CollapseLink.svelte' + import DataTableSettings, { + convertDataTableSettingsFromBackend, + type DataTableSettingsType + } from '$lib/components/workspaceSettings/DataTableSettings.svelte' import WorkspaceDependenciesSettings from '$lib/components/workspaceSettings/WorkspaceDependenciesSettings.svelte' let slackInitialPath: string = $state('') @@ -111,9 +115,9 @@ secondaryStorage: undefined }) - let ducklakeSettings: DucklakeSettingsType = $state({ - ducklakes: [] - }) + let dataTableSettings: DataTableSettingsType = $state({ dataTables: [] }) + + let ducklakeSettings: DucklakeSettingsType = $state({ ducklakes: [] }) let ducklakeSavedSettings: DucklakeSettingsType = $state(untrack(() => ducklakeSettings)) let workspaceDefaultAppPath: string | undefined = $state(undefined) @@ -132,6 +136,7 @@ | 'deploy_to' | 'error_handler' | 'ai' + | 'windmill_data_tables' | 'windmill_lfs' | 'git_sync' | 'default_app' @@ -307,6 +312,7 @@ !!$enterpriseLicense ) initialS3ResourceSettings = clone(s3ResourceSettings) + dataTableSettings = convertDataTableSettingsFromBackend(settings.datatable) ducklakeSettings = convertDucklakeSettingsFromBackend(settings.ducklake) ducklakeSavedSettings = clone(ducklakeSettings) @@ -341,7 +347,9 @@ if (!$workspaceStore) return try { - const config = await WorkspaceService.getWorkspaceSlackOauthConfig({ workspace: $workspaceStore }) + const config = await WorkspaceService.getWorkspaceSlackOauthConfig({ + workspace: $workspaceStore + }) useCustomSlackApp = !!config.slack_oauth_client_id slackOAuthClientId = config.slack_oauth_client_id || '' slackOAuthClientSecret = config.slack_oauth_client_secret || '' @@ -664,6 +672,13 @@ aiDescription="Windmill AI workspace settings" label="Windmill AI" /> + {#snippet workspaceConfig()} -
- {#if slackOAuthConfigLoaded} - -
-
Workspace specific Slack app
-
-
-
- Client ID: - {slackOAuthClientId} -
- +
+
+ Client ID: + {slackOAuthClientId} +
+ +
+
-
+ {:else} + + + + {#if useCustomSlackApp} +
+ + + + + +
+ Create a Slack app at{' '} + + Slack API + . Set the redirect URI to:{' '} + + {window.location.origin}{base}/oauth/callback_slack + +
+ + +
+ +
+
+ {/if} + {/if}
- {:else} - - - - {#if useCustomSlackApp} -
- - - - - -
- Create a Slack app at{' '} - - Slack API - . Set the redirect URI to:{' '} - - {window.location.origin}{base}/oauth/callback_slack - -
- - -
- -
-
- {/if} - {/if} -
- {/snippet} - - {:else if slack_tabs === 'teams_commands'} + {/snippet} + + {:else if slack_tabs === 'teams_commands'} {#if !$enterpriseLicense}
@@ -1111,6 +1126,8 @@ initialMaxTokensPerModel = clone(maxTokensPerModel) }} /> + {:else if tab == 'windmill_data_tables'} + {:else if tab == 'windmill_lfs'} Any: + """Run a script on the current worker without creating a job""" + endpoint = f"/w/{self.workspace}/jobs/run_inline/preview" + body = { + "content": content, + "language": language, + "args": args or {}, + } + return self.post(endpoint, json=body).text + def wait_job( self, job_id, @@ -1000,6 +1015,13 @@ class Windmill: }, ) + def datatable(self, name: str = "main"): + return DataTableClient(self, name) + + def ducklake(self, name: str = "main"): + return DucklakeClient(self, name) + + def init_global_client(f): @functools.wraps(f) @@ -1527,6 +1549,18 @@ def run_script_by_hash( timeout=timeout, ) +@init_global_client +def run_inline_script_preview( + content: str, + language: str, + args: dict = None, +) -> Any: + """Run a script on the current worker without creating a job""" + return _client.run_inline_script_preview( + content=content, + language=language, + args=args, + ) @init_global_client def username_to_email(username: str) -> str: @@ -1538,6 +1572,14 @@ def username_to_email(username: str) -> str: return _client.username_to_email(username) +@init_global_client +def datatable(name: str = "main") -> DataTableClient: + return _client.datatable(name) + +@init_global_client +def ducklake(name: str = "main") -> DucklakeClient: + return _client.ducklake(name) + def task(*args, **kwargs): from inspect import signature @@ -1635,3 +1677,78 @@ def stream_result(stream) -> None: """ for text in stream: append_to_result_stream(text) + +class DataTableClient: + def __init__(self, client: Windmill, name: str): + self.client = client + self.name = name + def query(self, sql: str, *args): + args_dict = {} + args_def = "" + for i, arg in enumerate(args): + args_dict[f"arg{i+1}"] = arg + args_def += f"-- ${i+1} arg{i+1}\n" + sql = args_def + sql + return SqlQuery( + sql, + lambda sql: self.client.run_inline_script_preview( + content=sql, + language="postgresql", + args={"database": f"datatable://{self.name}", **args_dict}, + ) + ) + +class DucklakeClient: + def __init__(self, client: Windmill, name: str): + self.client = client + self.name = name + def query(self, sql: str, **kwargs): + args_dict = {} + args_def = "" + for key, value in kwargs.items(): + args_dict[key] = value + args_def += f"-- ${key} ({infer_sql_type(value)})\n" + attach = f"ATTACH 'ducklake://{self.name}' AS dl;USE dl;\n" + sql = args_def + attach + sql + return SqlQuery( + sql, + lambda sql: self.client.run_inline_script_preview( + content=sql, + language="duckdb", + args=args_dict, + ) + ) + +class SqlQuery: + def __init__(self, sql: str, fetch_fn): + self.sql = sql + self.fetch_fn = fetch_fn + def fetch(self, result_collection: str | None = None): + sql = self.sql + if result_collection is not None: + sql = f'-- result_collection={result_collection}\n{sql}' + return self.fetch_fn(sql) + def fetch_one(self): + return self.fetch(result_collection="last_statement_first_row") + +def infer_sql_type(value) -> str: + """ + DuckDB executor requires explicit argument types at declaration + These types exist in both DuckDB and Postgres + Check that the types exist if you plan to extend this function for other SQL engines. + """ + if isinstance(value, bool): + # Check bool before int since bool is a subclass of int in Python + return "BOOLEAN" + elif isinstance(value, int): + return "BIGINT" + elif isinstance(value, float): + return "DOUBLE PRECISION" + elif value is None: + return "TEXT" + elif isinstance(value, str): + return "TEXT" + elif isinstance(value, dict) or isinstance(value, list): + return "JSON" + else: + return "TEXT" diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 26fcd52ffe..4937c2b069 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -11,9 +11,10 @@ sed -i 's/get \[Symbol\.toStringTag\]() {/get \[Symbol\.toStringTag\]() : string cp "${script_dirpath}/client.ts" "${script_dirpath}/src/" cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" +cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index c17fb2a8c9..f8db442962 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -36,7 +36,8 @@ fi cp "${script_dirpath}/client.ts" "${script_dirpath}/src/" cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" +cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, type SqlTemplateFunction } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.d.ts b/typescript-client/client.d.ts index 41238ff787..1ad02efbbe 100644 --- a/typescript-client/client.d.ts +++ b/typescript-client/client.d.ts @@ -1,5 +1,20 @@ import { DenoS3LightClientSettings, type S3Object } from "./s3Types"; -export { AdminService, AuditService, FlowService, GranularAclService, GroupService, JobService, ResourceService, VariableService, ScriptService, ScheduleService, SettingsService, UserService, WorkspaceService, } from "./index"; +export { + AdminService, + AuditService, + FlowService, + GranularAclService, + GroupService, + JobService, + ResourceService, + VariableService, + ScriptService, + ScheduleService, + SettingsService, + UserService, + WorkspaceService, +} from "./index"; +export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; export type Sql = string; export type Email = string; export type Base64 = string; @@ -17,19 +32,32 @@ export declare function getWorkspace(): string; * @param undefinedIfEmpty if the resource does not exist, return undefined instead of throwing an error * @returns resource value */ -export declare function getResource(path?: string, undefinedIfEmpty?: boolean): Promise; +export declare function getResource( + path?: string, + undefinedIfEmpty?: boolean +): Promise; /** * Get a resource value by path * @param jobId job id to get the root job id from (default to current job) * @returns root job id */ export declare function getRootJobId(jobId?: string): Promise; -export declare function runScript(path?: string | null, hash_?: string | null, args?: Record | null, verbose?: boolean): Promise; +export declare function runScript( + path?: string | null, + hash_?: string | null, + args?: Record | null, + verbose?: boolean +): Promise; export declare function waitJob(jobId: string, verbose?: boolean): Promise; export declare function getResult(jobId: string): Promise; export declare function getResultMaybe(jobId: string): Promise; export declare function task(f: (_: P) => T): (_: P) => Promise; -export declare function runScriptAsync(path: string | null, hash_: string | null, args: Record | null, scheduledInSeconds?: number | null): Promise; +export declare function runScriptAsync( + path: string | null, + hash_: string | null, + args: Record | null, + scheduledInSeconds?: number | null +): Promise; /** * Resolve a resource value in case the default value was picked because the input payload was undefined * @param obj resource value or path of the resource under the format `$res:path` @@ -43,7 +71,11 @@ export declare function getStatePath(): string; * @param value new value of the resource to set * @param initializeToTypeIfNotExist if the resource does not exist, initialize it with this type */ -export declare function setResource(value: any, path?: string, initializeToTypeIfNotExist?: string): Promise; +export declare function setResource( + value: any, + path?: string, + initializeToTypeIfNotExist?: string +): Promise; /** * Set the state * @param state state to set @@ -61,13 +93,20 @@ export declare function setState(state: any): Promise; * @param value value of the state */ -export declare function setFlowUserState(key: string, value: any, errorIfNotPossible?: boolean): Promise; +export declare function setFlowUserState( + key: string, + value: any, + errorIfNotPossible?: boolean +): Promise; /** * Get a flow user state * @param path path of the variable */ -export declare function getFlowUserState(key: string, errorIfNotPossible?: boolean): Promise; +export declare function getFlowUserState( + key: string, + errorIfNotPossible?: boolean +): Promise; /** * Get the internal state * @deprecated use getState instead @@ -90,9 +129,16 @@ export declare function getVariable(path: string): Promise; * @param isSecretIfNotExist if the variable does not exist, create it as secret or not (default: false) * @param descriptionIfNotExist if the variable does not exist, create it with this description (default: "") */ -export declare function setVariable(path: string, value: string, isSecretIfNotExist?: boolean, descriptionIfNotExist?: string): Promise; +export declare function setVariable( + path: string, + value: string, + isSecretIfNotExist?: boolean, + descriptionIfNotExist?: string +): Promise; export declare function databaseUrlFromResource(path: string): Promise; -export declare function denoS3LightClientSettings(s3_resource_path: string | undefined): Promise; +export declare function denoS3LightClientSettings( + s3_resource_path: string | undefined +): Promise; /** * Load the content of a file stored in S3. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -103,7 +149,10 @@ export declare function denoS3LightClientSettings(s3_resource_path: string | und * console.log(text); * ``` */ -export declare function loadS3File(s3object: S3Object, s3ResourcePath?: string | undefined): Promise; +export declare function loadS3File( + s3object: S3Object, + s3ResourcePath?: string | undefined +): Promise; /** * Load the content of a file stored in S3 as a stream. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -113,7 +162,10 @@ export declare function loadS3File(s3object: S3Object, s3ResourcePath?: string | * console.log(await fileContentBlob.text()); * ``` */ -export declare function loadS3FileStream(s3object: S3Object, s3ResourcePath?: string | undefined): Promise; +export declare function loadS3FileStream( + s3object: S3Object, + s3ResourcePath?: string | undefined +): Promise; /** * Persist a file to the S3 bucket. If the s3ResourcePath is undefined, it will default to the workspace S3 resource. * @@ -123,24 +175,28 @@ export declare function loadS3FileStream(s3object: S3Object, s3ResourcePath?: st * console.log(fileContentAsUtf8Str) * ``` */ -export declare function writeS3File(s3object: S3Object | undefined, fileContent: string | Blob, s3ResourcePath?: string | undefined): Promise; +export declare function writeS3File( + s3object: S3Object | undefined, + fileContent: string | Blob, + s3ResourcePath?: string | undefined +): Promise; /** * Get URLs needed for resuming a flow after this step * @param approver approver name * @returns approval page UI URL, resume and cancel API URLs for resuming the flow */ export declare function getResumeUrls(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; + approvalPage: string; + resume: string; + cancel: string; }>; /** * @deprecated use getResumeUrls instead */ export declare function getResumeEndpoints(approver?: string): Promise<{ - approvalPage: string; - resume: string; - cancel: string; + approvalPage: string; + resume: string; + cancel: string; }>; /** * Get an OIDC jwt token for auth to external services (e.g: Vault, AWS) (ee only) diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 93fc66669d..ab1c232ca6 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -17,6 +17,8 @@ import { type S3Object, } from "./s3Types"; +export { datatable, ducklake, type SqlTemplateFunction } from "./sqlUtils"; + export { AdminService, AuditService, @@ -138,7 +140,9 @@ export async function runScript( args: Record | null = null, verbose: boolean = false ): Promise { - console.warn('runScript is deprecated. Use runScriptByPath or runScriptByHash instead.'); + console.warn( + "runScript is deprecated. Use runScriptByPath or runScriptByHash instead." + ); if (path && hash_) { throw new Error("path and hash_ are mutually exclusive"); } @@ -157,7 +161,10 @@ async function _runScriptInternal( if (path) { console.info(`running \`${path}\` synchronously with args:`, args); } else if (hash_) { - console.info(`running script with hash \`${hash_}\` synchronously with args:`, args); + console.info( + `running script with hash \`${hash_}\` synchronously with args:`, + args + ); } } @@ -185,9 +192,7 @@ export async function runScriptByHash( * Append a text to the result stream * @param text text to append to the result stream */ -export function appendToResultStream( - text: string -) { +export function appendToResultStream(text: string) { console.log("WM_STREAM: " + text.replace(/\n/g, "\\n")); } @@ -195,9 +200,7 @@ export function appendToResultStream( * Stream to the result stream * @param stream stream to stream to the result stream */ -export async function streamResult( - stream: AsyncIterable -) { +export async function streamResult(stream: AsyncIterable) { for await (const text of stream) { appendToResultStream(text); } @@ -310,7 +313,9 @@ export async function runScriptAsync( args: Record | null, scheduledInSeconds: number | null = null ): Promise { - console.warn('runScriptAsync is deprecated. Use runScriptByPathAsync or runScriptByHashAsync instead.'); + console.warn( + "runScriptAsync is deprecated. Use runScriptByPathAsync or runScriptByHashAsync instead." + ); // Create a script job and return its job id. if (path && hash_) { throw new Error("path and hash_ are mutually exclusive"); @@ -815,7 +820,8 @@ export async function loadS3FileStream( // We use raw fetch here b/c OpenAPI generated client doesn't handle Blobs nicely const response = await fetch( - `${OpenAPI.BASE + `${ + OpenAPI.BASE }/w/${getWorkspace()}/job_helpers/download_s3_file?${queryParams}`, { method: "GET", diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 66ba76441d..0893075673 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -3,7 +3,7 @@ "version": "1.589.3", "exports": "./src/index.ts", "publish": { - "exclude": ["!src", "./s3Types.ts", "./client.ts"] + "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] }, "license": "Apache-2.0" } diff --git a/typescript-client/publish.sh b/typescript-client/publish.sh index 99f88ab29a..ba5c09c82f 100755 --- a/typescript-client/publish.sh +++ b/typescript-client/publish.sh @@ -9,8 +9,10 @@ rm -rf "${script_dirpath}/dist" ${script_dirpath}/build.sh rm "${script_dirpath}/client.ts" rm "${script_dirpath}/s3Types.ts" +rm "${script_dirpath}/sqlUtils.ts" npm install tsc cp "${script_dirpath}/src/client.ts" ${script_dirpath} cp "${script_dirpath}/src/s3Types.ts" ${script_dirpath} +cp "${script_dirpath}/src/sqlUtils.ts" ${script_dirpath} npm publish ${args} diff --git a/typescript-client/sqlUtils.d.ts b/typescript-client/sqlUtils.d.ts new file mode 100644 index 0000000000..012ebcb38a --- /dev/null +++ b/typescript-client/sqlUtils.d.ts @@ -0,0 +1,53 @@ +type ResultCollection = + | "last_statement_all_rows" + | "last_statement_first_row" + | "last_statement_all_rows_scalar" + | "last_statement_first_row_scalar" + | "all_statements_all_rows" + | "all_statements_first_row" + | "all_statements_all_rows_scalar" + | "all_statements_first_row_scalar" + | "legacy"; + +type FetchParams = { + resultCollection?: ResultCollectionT; +}; + +type SqlResult = + ResultCollectionT extends "last_statement_first_row" + ? object + : ResultCollectionT extends "all_statements_first_row" + ? object[] + : ResultCollectionT extends "last_statement_all_rows" + ? object[] + : ResultCollectionT extends "all_statements_all_rows" + ? object[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? any[] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? any[][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? any + : ResultCollectionT extends "all_statements_first_row_scalar" + ? any[] + : unknown; +export type SqlStatement = { + content: string; + + args: Record; + + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; +}; + +export interface SqlTemplateFunction { + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; +} + +export declare function datatable(name: string): SqlTemplateFunction; +export declare function ducklake(name: string): SqlTemplateFunction; diff --git a/typescript-client/sqlUtils.ts b/typescript-client/sqlUtils.ts new file mode 100644 index 0000000000..a024992de6 --- /dev/null +++ b/typescript-client/sqlUtils.ts @@ -0,0 +1,218 @@ +import { getWorkspace, JobService } from "./client"; + +type ResultCollection = + | "last_statement_all_rows" + | "last_statement_first_row" + | "last_statement_all_rows_scalar" + | "last_statement_first_row_scalar" + | "all_statements_all_rows" + | "all_statements_first_row" + | "all_statements_all_rows_scalar" + | "all_statements_first_row_scalar" + | "legacy"; + +type FetchParams = { + resultCollection?: ResultCollectionT; +}; + +type SqlResult = + ResultCollectionT extends "last_statement_first_row" + ? object + : ResultCollectionT extends "all_statements_first_row" + ? object[] + : ResultCollectionT extends "last_statement_all_rows" + ? object[] + : ResultCollectionT extends "all_statements_all_rows" + ? object[][] + : ResultCollectionT extends "last_statement_all_rows_scalar" + ? any[] + : ResultCollectionT extends "all_statements_all_rows_scalar" + ? any[][] + : ResultCollectionT extends "last_statement_first_row_scalar" + ? any + : ResultCollectionT extends "all_statements_first_row_scalar" + ? any[] + : unknown; +export type SqlStatement = { + content: string; + + args: Record; + + fetch( + params?: FetchParams // The union is for auto-completion + ): Promise>; + + fetchOne( + params?: Omit, "resultCollection"> + ): Promise>; +}; + +export interface SqlTemplateFunction { + (strings: TemplateStringsArray, ...values: any[]): SqlStatement; +} + +/** + * @example + * let sql = wmill.datatable() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age}::int + * `.fetch() + */ +export function datatable(name: string = "main"): SqlTemplateFunction { + return sqlProviderImpl(name, "datatable"); +} + +/** + * @example + * let sql = wmill.ducklake() + * let name = 'Robin' + * let age = 21 + * await sql` + * SELECT * FROM friends + * WHERE name = ${name} AND age = ${age} + * `.fetch() + */ +export function ducklake(name: string = "main"): SqlTemplateFunction { + return sqlProviderImpl(name, "ducklake"); +} + +function sqlProviderImpl( + name: string, + provider: "datatable" | "ducklake" +): SqlTemplateFunction { + let sql: SqlTemplateFunction = ( + strings: TemplateStringsArray, + ...values: any[] + ) => { + let formatArgDecl = { + datatable: (i: number) => `-- $${i + 1} arg${i + 1}`, + ducklake: (i: number) => { + let argType = + parseTypeAnnotation(strings[i], strings[i + 1]) || + inferSqlType(values[i]); + return `-- $arg${i + 1} (${argType})`; + }, + }[provider]; + + let formatArgUsage = { + datatable: (i: number) => { + let argType = + parseTypeAnnotation(strings[i], strings[i + 1]) || + inferSqlType(values[i]); + return `$${i + 1}::${argType}`; + }, + ducklake: (i: number) => `$arg${i + 1}`, + }[provider]; + + let content = values.map((_, i) => formatArgDecl(i)).join("\n") + "\n"; + if (provider === "ducklake") + content += `ATTACH 'ducklake://${name}' AS dl;USE dl;\n`; + + let contentBody = ""; + for (let i = 0; i < strings.length; i++) { + contentBody += strings[i]; + if (i !== strings.length - 1) contentBody += formatArgUsage(i); + } + content += contentBody; + + const args = { + ...Object.fromEntries(values.map((v, i) => [`arg${i + 1}`, v])), + ...(provider === "datatable" ? { database: `datatable://${name}` } : {}), + }; + const language = { + datatable: "postgresql" as const, + ducklake: "duckdb" as const, + }[provider]; + + async function fetch({ + resultCollection, + }: FetchParams = {}) { + if (resultCollection) + content = `-- result_collection=${resultCollection}\n${content}`; + try { + let result = await JobService.runScriptPreviewInline({ + workspace: getWorkspace(), + requestBody: { args, content, language }, + }); + return result as SqlResult; + } catch (e: any) { + let err = e; + if ( + e && + typeof e.body == "string" && + e.statusText == "Internal Server Error" + ) { + let body = e.body; + if (body.startsWith("Internal:")) body = body.slice(9).trim(); + if (body.startsWith("Error:")) body = body.slice(6).trim(); + if (body.startsWith("datatable")) body = body.slice(9).trim(); + err = Error(`${provider} ${body}`); + err.query = contentBody; + err.request = e.request; + } + throw err; + } + } + + return { + content, + args, + fetch, + fetchOne: (params) => + fetch({ ...params, resultCollection: "last_statement_first_row" }), + } satisfies SqlStatement; + }; + return sql; +} + +// DuckDB executor requires explicit argument types at declaration +// And postgres at argument usage. +// These types exist in both DuckDB and Postgres +// Check that the types exist if you plan to extend this function for other SQL engines. +function inferSqlType(value: any): string { + if (typeof value === "number" || typeof value === "bigint") { + if (Number.isInteger(value)) return "BIGINT"; + return "DOUBLE PRECISION"; + } else if (value === null || value === undefined) { + return "TEXT"; + } else if (typeof value === "string") { + return "TEXT"; + } else if (typeof value === "object") { + return "JSON"; + } else if (typeof value === "boolean") { + return "BOOLEAN"; + } else { + return "TEXT"; + } +} + +// The goal is to detect if the user added a type annotation manually +// +// untyped : sql`SELECT ${x} = 0` => ['SELECT ', ' = 0'] +// typed : sql`SELECT ${x}::int = 0` => ['SELECT ', '::int = 0'] +// typed : sql`SELECT CAST ( ${x} AS int ) = 0` => ['SELECT CAST ( ', ' AS int ) = 0'] +function parseTypeAnnotation( + prevTemplateString: string | undefined, + nextTemplateString: string | undefined +): string | undefined { + if (!nextTemplateString) return; + nextTemplateString = nextTemplateString.trimStart(); + if (nextTemplateString.startsWith("::")) { + return nextTemplateString.substring(2).trimStart().split(/\s+/)[0]; + } + prevTemplateString = prevTemplateString?.trimEnd(); + if ( + prevTemplateString?.endsWith("(") && + prevTemplateString + .substring(0, prevTemplateString.length - 1) + .trim() + .toUpperCase() + .endsWith("CAST") && + nextTemplateString.toUpperCase().startsWith("AS ") + ) { + return nextTemplateString.substring(2).trimStart().split(/\s+/)[0]; + } +} From d46012e26e60126bd6d89bed60d22a2fe7d44221 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 6 Dec 2025 10:15:38 +0000 Subject: [PATCH 21/25] exclude vscode from optimize deps --- frontend/vite.config.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 0c746db886..37375cac06 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -85,7 +85,8 @@ const config = { include: ['highlight.js', 'highlight.js/lib/core', 'monaco-vim', 'monaco-editor-wrapper'], exclude: [ '@codingame/monaco-vscode-standalone-typescript-language-features', - '@codingame/monaco-vscode-standalone-languages' + '@codingame/monaco-vscode-standalone-languages', + 'vscode' ] }, worker: { From 61b412eef43558fcba3a2e5de4200014ec909642 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 6 Dec 2025 10:26:54 +0000 Subject: [PATCH 22/25] fallback for unknown asset kind --- frontend/src/lib/components/assets/lib.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/lib/components/assets/lib.ts b/frontend/src/lib/components/assets/lib.ts index 0d96e67c39..cb7b824a64 100644 --- a/frontend/src/lib/components/assets/lib.ts +++ b/frontend/src/lib/components/assets/lib.ts @@ -25,6 +25,7 @@ export function formatAsset(asset: Asset): string { case 'datatable': return `datatable://${asset.path}` } + return 'unknown' } export function getAssetUsagePageUri(usage: ListAssetsResponse[number]['usages'][number]) { From 0e63dd301f2b2b258225d19a2830ec35c325716a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 6 Dec 2025 17:06:30 +0100 Subject: [PATCH 23/25] fix: raw app v0.5 (#7310) --- backend/windmill-api/src/apps.rs | 6 +- cli/deps.ts | 2 +- cli/src/commands/app/app_metadata.ts | 164 ++-- cli/src/commands/app/apps.ts | 12 +- cli/src/commands/app/bundle.ts | 25 +- cli/src/commands/app/dev.ts | 88 +- cli/src/commands/app/lint.ts | 226 +++++ cli/src/commands/app/metadata.ts | 5 +- cli/src/commands/app/raw_apps.ts | 250 +++++- cli/src/commands/script/script.ts | 12 + cli/src/commands/sync/sync.ts | 71 +- .../src/path-utils/path-assigner.ts | 99 +- frontend/.gitignore | 3 +- frontend/serve_ui_builder.sh | 17 + .../sharedUtils/vite.sharedUtils.config.js | 2 +- frontend/src/lib/components/Editor.svelte | 43 +- .../src/lib/components/ScriptBuilder.svelte | 5 +- .../helpers/RunnableComponent.svelte | 16 +- .../lib/components/apps/editor/appUtilsS3.ts | 5 +- .../src/lib/components/copilot/CronGen.svelte | 39 +- .../lib/components/copilot/MetadataGen.svelte | 130 +-- .../copilot/chat/AIChatInlineWidget.svelte | 10 +- .../copilot/chat/AIChatInput.svelte | 4 +- .../copilot/chat/AIChatManager.svelte.ts | 68 +- .../copilot/chat/AiChatLayout.svelte | 2 +- .../lib/components/copilot/chat/anthropic.ts | 29 +- .../lib/components/copilot/chat/app/core.ts | 849 ++++++++++++++++++ .../src/lib/components/copilot/chat/shared.ts | 1 + frontend/src/lib/components/copilot/lib.ts | 3 +- .../flows/content/FlowSettings.svelte | 2 +- .../raw_apps/RawAppBackgroundRunner.svelte | 15 +- .../components/raw_apps/RawAppEditor.svelte | 239 ++++- .../raw_apps/RawAppInlineScriptEditor.svelte | 1 + .../RawAppInlineScriptPanelList.svelte | 87 +- .../components/raw_apps/RawAppSidebar.svelte | 2 + .../src/lib/components/raw_apps/lintStore.ts | 91 ++ .../workspaceSettings/AISettings.svelte | 25 +- frontend/src/lib/editorLangUtils.ts | 206 ++--- frontend/src/lib/scripts.ts | 9 + frontend/src/lib/utils.ts | 7 + .../(root)/(logged)/apps_raw/add/+page.svelte | 18 +- 41 files changed, 2504 insertions(+), 384 deletions(-) create mode 100644 cli/src/commands/app/lint.ts create mode 100755 frontend/serve_ui_builder.sh create mode 100644 frontend/src/lib/components/copilot/chat/app/core.ts create mode 100644 frontend/src/lib/components/raw_apps/lintStore.ts diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 4c35907d05..08aca365f2 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1770,12 +1770,12 @@ async fn execute_component( // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), - force_viewer_one_of_fields: Some(one_of_inputs), - force_viewer_allow_user_resources: Some(allow_user_resources), + force_viewer_one_of_fields, + force_viewer_allow_user_resources, .. } => ( &Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() }, - &PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources }, + &PolicyTriggerableInputs { static_inputs, one_of_inputs: force_viewer_one_of_fields.unwrap_or_default(), allow_user_resources: force_viewer_allow_user_resources.unwrap_or_default() }, ), // 2. "run" mode. _ => { diff --git a/cli/deps.ts b/cli/deps.ts index 28d8909d3c..f8ae3f0b65 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -57,7 +57,7 @@ export { WebSocketServer, WebSocket } from "npm:ws"; export * as getPort from "npm:get-port@7.1.0"; export * as open from "npm:open"; export * as esMain from "npm:es-main"; -export * as windmillUtils from "jsr:@windmill-labs/shared-utils@1.0.10"; +export * as windmillUtils from "jsr:@windmill-labs/shared-utils@1.0.11"; // needed for dnt transform import * as wsTypes from "npm:@types/ws"; diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 11388b0083..b1c5c97c1b 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -24,10 +24,15 @@ import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; -import { AppFile as RawAppFile } from "./raw_apps.ts"; +import { + AppFile as RawAppFile, + loadRunnablesFromBackend, + writeRunnableToBackend, +} from "./raw_apps.ts"; import { replaceInlineScripts, AppFile as NormalAppFile } from "./apps.ts"; import { newPathAssigner, + newRawAppPathAssigner, SupportedLanguage, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; @@ -163,21 +168,29 @@ export async function generateAppLocksInternal( ); if (rawApp) { - const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER) + SEP; + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) const rawAppFile = appFile as RawAppFile; + let runnables = await loadRunnablesFromBackend(runnablesPath); + if (Object.keys(runnables).length === 0 && rawAppFile.runnables) { + // Fall back to old format + runnables = rawAppFile.runnables; + } // Replace inline scripts for changed runnables - replaceInlineScripts(rawAppFile.runnables, runnablesPath, false); + replaceInlineScripts(runnables, runnablesPath + SEP, false); - // Update the app runnables with new locks - rawAppFile.runnables = await updateRawAppRunnables( + // Update the app runnables with new locks (writes to separate files) + await updateRawAppRunnables( workspace, - rawAppFile.runnables, + runnables, remote_path, appFolder, rawWorkspaceDependencies, opts.defaultTs ); + // Note: updateRawAppRunnables now writes each runnable to its own file } else { const normalAppFile = appFile as NormalAppFile; @@ -193,13 +206,13 @@ export async function generateAppLocksInternal( rawWorkspaceDependencies, opts.defaultTs ); - } - // Write the updated app file - writeIfChanged( - appFilePath, - yamlStringify(appFile as Record, yamlOptions) - ); + // Write the updated app file (only for normal apps, raw apps use separate files) + writeIfChanged( + appFilePath, + yamlStringify(appFile as Record, yamlOptions) + ); + } } else { log.info(colors.gray(`No scripts changed in ${appFolder}`)); } @@ -278,8 +291,9 @@ async function traverseAndProcessInlineScripts( } /** - * Updates locks for all runnables in a raw app, generating locks inline script by inline script - * Also writes content and locks back to the runnables folder + * Updates locks for all runnables in a raw app, generating locks inline script by inline script. + * Writes each runnable to its own YAML file in the backend folder (new format). + * Also writes content and lock files to the runnables folder. */ async function updateRawAppRunnables( workspace: Workspace, @@ -288,7 +302,7 @@ async function updateRawAppRunnables( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun" -): Promise> { +): Promise { const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); // Ensure runnables folder exists @@ -298,15 +312,16 @@ async function updateRawAppRunnables( // Folder may already exist } - const pathAssigner = newPathAssigner(defaultTs); - - // Process each runnable - const updatedRunnables: Record = {}; + const pathAssigner = newRawAppPathAssigner(defaultTs); for (const [runnableId, runnable] of Object.entries(runnables)) { - // Only process inline scripts (runnableByName with inlineScript) - if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) { - updatedRunnables[runnableId] = runnable; + // Only process inline scripts (runnableByName/inline with inlineScript) + if ( + (runnable?.type !== "runnableByName" && runnable?.type !== "inline") || + !runnable?.inlineScript + ) { + // Write non-inline runnables to their own file as-is + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } @@ -315,7 +330,7 @@ async function updateRawAppRunnables( const content = inlineScript.content; if (!content || !language) { - updatedRunnables[runnableId] = runnable; + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } @@ -326,13 +341,30 @@ async function updateRawAppRunnables( `Runnable ${runnableId} content is still an !inline reference, skipping` ) ); - updatedRunnables[runnableId] = runnable; + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } // Skip frontend scripts - they don't need locks if (language === "frontend") { - updatedRunnables[runnableId] = runnable; + // Still need to write the runnable YAML file + const [basePathO, ext] = pathAssigner.assignPath( + runnable.name ?? runnableId, + language + ); + const basePath = basePathO.replaceAll(SEP, "/"); + const contentPath = path.join(runnablesFolder, `${basePath}${ext}`); + writeIfChanged(contentPath, content); + + // Write simplified runnable YAML - just type: 'inline' plus metadata + // inlineScript is not needed since content/language can be derived from files + const simplifiedRunnable: Record = { type: "inline" }; + for (const [key, value] of Object.entries(runnable)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } + writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable); continue; } @@ -353,7 +385,10 @@ async function updateRawAppRunnables( ); // Determine file extension for this language - const [basePathO, ext] = pathAssigner.assignPath(runnable.name, language); + const [basePathO, ext] = pathAssigner.assignPath( + runnable.name ?? runnableId, + language + ); const basePath = basePathO.replaceAll(SEP, "/"); const contentPath = path.join(runnablesFolder, `${basePath}${ext}`); const lockPath = path.join(runnablesFolder, `${basePath}lock`); @@ -366,23 +401,21 @@ async function updateRawAppRunnables( writeIfChanged(lockPath, lock); } - // Update the runnable with !inline references (preserve existing schema) - const inlineContentRef = `!inline ${basePath}${ext}`; - const inlineLockRef = - lock && lock !== "" ? `!inline ${basePath}lock` : ""; + // Write simplified runnable YAML - just type: 'inline' plus metadata + // inlineScript is not needed since content/lock/language can be derived from files + const simplifiedRunnable: Record = { type: "inline" }; + for (const [key, value] of Object.entries(runnable)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } - updatedRunnables[runnableId] = { - ...runnable, - inlineScript: { - ...inlineScript, - content: inlineContentRef, - lock: inlineLockRef, - }, - }; + // Write the runnable to its own YAML file + writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable); log.info( colors.gray( - ` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` + ` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` ) ); } catch (error: any) { @@ -391,12 +424,10 @@ async function updateRawAppRunnables( `Failed to generate lock for runnable ${runnableId}: ${error.message}` ) ); - // Continue with other runnables even if one fails - updatedRunnables[runnableId] = runnable; + // Write the original runnable even if lock generation fails + writeRunnableToBackend(runnablesFolder, runnableId, runnable); } } - - return updatedRunnables; } /** @@ -585,7 +616,7 @@ export interface InferredSchemaResult { /** * Infers schema for a single runnable from its file content. * Used by dev server to update schema in memory (for wmill.d.ts generation). - * Does NOT write to raw_app.yaml - schema is kept in memory only. + * Does NOT write to the runnable YAML file - schema is kept in memory only. * * @param appFolder - The folder containing the raw app * @param runnableFilePath - The path to the changed runnable file (relative to runnables folder) @@ -595,33 +626,52 @@ export async function inferRunnableSchemaFromFile( appFolder: string, runnableFilePath: string ): Promise { - // Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable") + // Extract runnable ID from file path (e.g., "myRunnable.ts" -> "myRunnable") const fileName = path.basename(runnableFilePath); - // Skip lock files - if (fileName.endsWith(".lock")) { + // Skip lock files and yaml files (runnable metadata) + if (fileName.endsWith(".lock") || fileName.endsWith(".yaml")) { return undefined; } - // Match pattern: {runnableId}.inline_script.{ext} - const match = fileName.match(/^(.+)\.inline_script\.[^.]+$/); + // Match pattern: {runnableId}.{ext} - extract the runnable ID (everything before the last dot) + const match = fileName.match(/^(.+)\.[^.]+$/); if (!match) { return undefined; } const runnableId = match[1]; - // Read the app file to get the language - const appFilePath = path.join(appFolder, "raw_app.yaml"); - const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; + // Read the runnable from its separate YAML file (new format) + const runnableFilePath2 = path.join( + appFolder, + APP_BACKEND_FOLDER, + `${runnableId}.yaml` + ); - if (!appFile.runnables?.[runnableId]) { - log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`)); - return undefined; + let runnable: any; + try { + runnable = await yamlParseFile(runnableFilePath2); + } catch { + // Fall back to reading from raw_app.yaml (old format) + try { + const appFilePath = path.join(appFolder, "raw_app.yaml"); + const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; + if (!appFile.runnables?.[runnableId]) { + log.warn( + colors.yellow(`Runnable ${runnableId} not found in backend folder or raw_app.yaml`) + ); + return undefined; + } + runnable = appFile.runnables[runnableId]; + } catch { + log.warn( + colors.yellow(`Could not read runnable ${runnableId} from any source`) + ); + return undefined; + } } - const runnable = appFile.runnables[runnableId]; - // Only process inline scripts if (!runnable?.inlineScript) { return undefined; diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index e2b715b199..5892eb0d91 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -16,6 +16,7 @@ import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; +import lintCommand from "./lint.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; export interface AppFile { @@ -62,15 +63,7 @@ export function replaceInlineScripts( } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { - if (k == "runType") { - if (addType) { - if (isVersionsGeq1585()) { - rec["type"] = "path"; - } else { - rec["type"] = "runnableByPath"; - } - } - } else if (k == "inlineScript" && typeof v == "object") { + if (k == "inlineScript" && typeof v == "object") { if (addType) { if (isVersionsGeq1585()) { rec["type"] = "inline"; @@ -230,6 +223,7 @@ const command = new Command() .arguments(" ") .action(push as any) .command("dev", devCommand) + .command("lint", lintCommand) .command( "generate-locks", "re-generate the lockfiles for app runnables inline scripts that have changed" diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 8f45cb5819..3b8ebc03c5 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -206,16 +206,21 @@ export async function createBundle( const wmillPlugin = { name: "wmill-virtual", setup(build: any) { - - - // Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill - build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => { - log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); - return { - path: args.path, - namespace: "wmill-virtual", - }; - }); + // Intercept imports of wmill with various path formats: + // - wmill, wmill.ts (bare import) + // - /wmill, /wmill.ts (absolute) + // - ./wmill, ./wmill.ts (same directory) + // - ../wmill, ../../wmill, etc. (parent directories) + build.onResolve( + { filter: /^(\.\.\/)+wmill(\.ts)?$|^(\.\/|\/)?wmill(\.ts)?$/ }, + (args: any) => { + log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); + return { + path: args.path, + namespace: "wmill-virtual", + }; + } + ); // Provide the virtual module content build.onLoad({ filter: /.*/, namespace: "wmill-virtual" }, (args: any) => { diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 327f06864e..b5b6f22c31 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -34,6 +34,7 @@ import { APP_BACKEND_FOLDER, inferRunnableSchemaFromFile, } from "./app_metadata.ts"; +import { loadRunnablesFromBackend } from "./raw_apps.ts"; const DEFAULT_PORT = 4000; const DEFAULT_HOST = "localhost"; @@ -189,14 +190,21 @@ async function dev(opts: DevOptions) { const wmillPlugin = { name: "wmill-virtual", setup(build: any) { - // Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill - build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => { - log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); - return { - path: args.path, - namespace: "wmill-virtual", - }; - }); + // Intercept imports of wmill with various path formats: + // - wmill, wmill.ts (bare import) + // - /wmill, /wmill.ts (absolute) + // - ./wmill, ./wmill.ts (same directory) + // - ../wmill, ../../wmill, etc. (parent directories) + build.onResolve( + { filter: /^(\.\.\/)+wmill(\.ts)?$|^(\.\/|\/)?wmill(\.ts)?$/ }, + (args: any) => { + log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); + return { + path: args.path, + namespace: "wmill-virtual", + }; + } + ); // Provide the virtual module content build.onLoad( @@ -665,19 +673,35 @@ export default command; /** * Generates wmill.d.ts with type definitions for runnables. - * Merges in-memory inferred schemas with runnables from raw_app.yaml. + * Loads runnables from separate YAML files in the backend folder (new format) + * or falls back to raw_app.yaml (old format). + * Merges in-memory inferred schemas with runnables. * * @param schemaOverrides - In-memory schema overrides (runnableId -> schema) */ async function genRunnablesTs(schemaOverrides: Record = {}) { log.info(colors.blue("🔄 Generating wmill.d.ts...")); - const rawApp = (await yamlParseFile( - path.join(process.cwd(), "raw_app.yaml") - )) as any; - const runnables = rawApp?.["runnables"] as any; + + const localPath = process.cwd(); + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) + let runnables = await loadRunnablesFromBackend(backendPath); + + if (Object.keys(runnables).length === 0) { + // Fall back to old format + try { + const rawApp = (await yamlParseFile( + path.join(localPath, "raw_app.yaml") + )) as any; + runnables = rawApp?.["runnables"] ?? {}; + } catch { + runnables = {}; + } + } // Apply schema overrides from in-memory cache - if (runnables && Object.keys(schemaOverrides).length > 0) { + if (Object.keys(schemaOverrides).length > 0) { for (const [runnableId, schema] of Object.entries(schemaOverrides)) { if (runnables[runnableId]?.inlineScript) { runnables[runnableId].inlineScript.schema = schema; @@ -697,16 +721,22 @@ async function genRunnablesTs(schemaOverrides: Record = {}) { async function loadRunnables(): Promise> { try { const localPath = process.cwd(); - const rawApp = (await yamlParseFile( - path.join(localPath, "raw_app.yaml") - )) as any; - replaceInlineScripts( - rawApp.runnables, - path.join(localPath, APP_BACKEND_FOLDER) + SEP, - true - ); + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); - return rawApp?.runnables ?? {}; + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) + let runnables = await loadRunnablesFromBackend(backendPath); + + if (Object.keys(runnables).length === 0) { + // Fall back to old format + const rawApp = (await yamlParseFile( + path.join(localPath, "raw_app.yaml") + )) as any; + runnables = rawApp?.runnables ?? {}; + } + + replaceInlineScripts(runnables, backendPath + SEP, true); + + return runnables; } catch (error: any) { log.error(colors.red(`Failed to load runnables: ${error.message}`)); return {}; @@ -755,14 +785,12 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; - } else if ( - (runnable.type === "path" || runnable.type === "runnableByPath") && - runnable.path - ) { - const runType = runnable.runType ?? "script"; + } else if (runnable.type === "path" && runnable.runType && runnable.path) { + // Path-based runnables have type: "path" and runType: "script"|"hubscript"|"flow" + const prefix = runnable.runType; requestBody.path = - runType !== "hubscript" - ? `${runType}/${runnable.path}` + prefix !== "hubscript" + ? `${prefix}/${runnable.path}` : `script/${runnable.path}`; } diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts new file mode 100644 index 0000000000..a4d73c6983 --- /dev/null +++ b/cli/src/commands/app/lint.ts @@ -0,0 +1,226 @@ +// deno-lint-ignore-file no-explicit-any +import * as fs from "node:fs"; +import * as path from "node:path"; +import process from "node:process"; +import { Command, colors, log, yamlParseFile } from "../../../deps.ts"; +import { GlobalOptions } from "../../types.ts"; +import { createBundle } from "./bundle.ts"; +import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; +import { loadRunnablesFromBackend } from "./raw_apps.ts"; + +interface LintOptions extends GlobalOptions { + fix?: boolean; +} + +interface LintResult { + valid: boolean; + errors: string[]; + warnings: string[]; +} + +/** + * Validates the structure of raw_app.yaml + */ +function validateRawAppYaml(appData: any): { + errors: string[]; + warnings: string[]; +} { + const errors: string[] = []; + const warnings: string[] = []; + + // Check required fields + if (!appData.summary) { + errors.push("Missing required field: 'summary'"); + } else if (typeof appData.summary !== "string") { + errors.push("Field 'summary' must be a string"); + } + + // Note: 'runnables' is no longer required in raw_app.yaml + // Runnables can be stored in separate files in the backend folder + + return { errors, warnings }; +} + +/** + * Validates that runnables exist either in backend/*.yaml files or in raw_app.yaml + */ +async function validateRunnables( + appDir: string, + appData: any +): Promise<{ errors: string[]; warnings: string[] }> { + const errors: string[] = []; + const warnings: string[] = []; + + const backendPath = path.join(appDir, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) + const runnablesFromBackend = await loadRunnablesFromBackend(backendPath); + const hasBackendRunnables = Object.keys(runnablesFromBackend).length > 0; + + // Check for runnables in raw_app.yaml (old format) + const hasYamlRunnables = + appData.runnables && + typeof appData.runnables === "object" && + !Array.isArray(appData.runnables) && + Object.keys(appData.runnables).length > 0; + + if (!hasBackendRunnables && !hasYamlRunnables) { + errors.push( + "No runnables found. Expected either:\n" + + " - Runnable YAML files in the 'backend/' folder (e.g., backend/myRunnable.yaml)\n" + + " - Or a 'runnables' field in raw_app.yaml (legacy format)" + ); + } else if (hasBackendRunnables) { + log.info( + colors.gray( + ` Found ${Object.keys(runnablesFromBackend).length} runnable(s) in backend folder` + ) + ); + } else if (hasYamlRunnables) { + log.info( + colors.gray( + ` Found ${Object.keys(appData.runnables).length} runnable(s) in raw_app.yaml (legacy format)` + ) + ); + warnings.push( + "Using legacy format with runnables in raw_app.yaml. Consider migrating to separate files in backend/" + ); + } + + return { errors, warnings }; +} + +/** + * Checks if the app can be built successfully + */ +async function validateBuild( + appDir: string +): Promise<{ errors: string[]; warnings: string[] }> { + const errors: string[] = []; + const warnings: string[] = []; + + try { + log.info(colors.blue("🔨 Testing build...")); + + // Try to create a bundle - this will validate that all dependencies are in place + await createBundle({ + production: true, + minify: false, + }); + + log.info(colors.green("✅ Build successful")); + } catch (error: any) { + errors.push(`Build failed: ${error.message}`); + } + + return { errors, warnings }; +} + +/** + * Validates a raw app folder + */ +async function lintRawApp( + appDir: string, + opts: LintOptions +): Promise { + const errors: string[] = []; + const warnings: string[] = []; + + // Check if we're in a .raw_app folder + const currentDirName = path.basename(appDir); + if (!currentDirName.endsWith(".raw_app")) { + errors.push( + `Not a raw app folder: '${currentDirName}' does not end with '.raw_app'` + ); + return { valid: false, errors, warnings }; + } + + // Check if raw_app.yaml exists + const rawAppPath = path.join(appDir, "raw_app.yaml"); + if (!fs.existsSync(rawAppPath)) { + errors.push("Missing raw_app.yaml file"); + return { valid: false, errors, warnings }; + } + + log.info(colors.blue("📋 Validating raw_app.yaml structure...")); + + // Parse and validate raw_app.yaml + let appData: any; + try { + appData = await yamlParseFile(rawAppPath); + } catch (error: any) { + errors.push(`Failed to parse raw_app.yaml: ${error.message}`); + return { valid: false, errors, warnings }; + } + + const yamlValidation = validateRawAppYaml(appData); + errors.push(...yamlValidation.errors); + warnings.push(...yamlValidation.warnings); + + if (errors.length > 0) { + return { valid: false, errors, warnings }; + } + + log.info(colors.green("✅ raw_app.yaml structure is valid")); + + // Validate runnables (either in backend folder or in raw_app.yaml) + log.info(colors.blue("📋 Validating runnables...")); + const runnablesValidation = await validateRunnables(appDir, appData); + errors.push(...runnablesValidation.errors); + warnings.push(...runnablesValidation.warnings); + + if (errors.length > 0) { + return { valid: false, errors, warnings }; + } + + log.info(colors.green("✅ Runnables are valid")); + + // Validate build + const buildValidation = await validateBuild(appDir); + errors.push(...buildValidation.errors); + warnings.push(...buildValidation.warnings); + + return { + valid: errors.length === 0, + errors, + warnings, + }; +} + +/** + * Main lint command + */ +async function lint(opts: LintOptions, appFolder?: string) { + const targetDir = appFolder ?? process.cwd(); + + log.info(colors.bold.blue(`\n🔍 Linting raw app: ${targetDir}\n`)); + + const result = await lintRawApp(targetDir, opts); + + // Display results + if (result.warnings.length > 0) { + log.info(colors.yellow("\n⚠️ Warnings:")); + result.warnings.forEach((warning) => { + log.info(colors.yellow(` - ${warning}`)); + }); + } + + if (result.errors.length > 0) { + log.info(colors.red("\n❌ Errors:")); + result.errors.forEach((error) => { + log.info(colors.red(` - ${error}`)); + }); + log.info(colors.red("\n❌ Lint failed\n")); + Deno.exit(1); + } + + log.info(colors.green("\n✅ All checks passed\n")); +} + +const command = new Command() + .description("Lint a raw app folder to validate structure and buildability") + .arguments("[app_folder:string]") + .option("--fix", "Attempt to fix common issues (not implemented yet)") + .action(lint as any); + +export default command; diff --git a/cli/src/commands/app/metadata.ts b/cli/src/commands/app/metadata.ts index a9e9dee812..4f56ab4fb4 100644 --- a/cli/src/commands/app/metadata.ts +++ b/cli/src/commands/app/metadata.ts @@ -13,9 +13,8 @@ export type Runnable = fields?: Record; } | { - type: "runnableByPath" | "path"; + type: "path"; + runType: "script" | "hubscript" | "flow"; path: string; - runType?: "script" | "flow" | "hubscript"; fields?: Record; - schema?: any; }; diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 200b697024..bd2e89e2b8 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -7,24 +7,197 @@ import { SEP, windmillUtils, yamlParseFile, + yamlStringify, } from "../../../deps.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { Policy } from "../../../gen/types.gen.ts"; +import path from "node:path"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { replaceInlineScripts, repopulateFields } from "./apps.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; +import { writeIfChanged } from "../../utils/utils.ts"; +import { yamlOptions } from "../sync/sync.ts"; +import { + EXTENSION_TO_LANGUAGE, + getLanguageFromExtension, +} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; export interface AppFile { - runnables: any; + runnables?: any; custom_path: string; public?: boolean; summary: string; policy: Policy; } +/** + * Finds the content file for a runnable by looking for files matching the runnableId. + * Returns the file extension and content, or undefined if not found. + */ +async function findRunnableContentFile( + backendPath: string, + runnableId: string, + allFiles: string[] +): Promise<{ ext: string; content: string } | undefined> { + // Look for files matching pattern: {runnableId}.{ext} + // where ext is a known language extension + for (const fileName of allFiles) { + // Skip yaml and lock files + if (fileName.endsWith(".yaml") || fileName.endsWith(".lock")) { + continue; + } + + // Check if file starts with runnableId followed by a dot + if (!fileName.startsWith(runnableId + ".")) { + continue; + } + + // Extract extension (everything after the first dot following runnableId) + const ext = fileName.substring(runnableId.length + 1); + + // Check if this is a recognized extension + if (EXTENSION_TO_LANGUAGE[ext]) { + try { + const content = await Deno.readTextFile( + path.join(backendPath, fileName) + ); + return { ext, content }; + } catch { + continue; + } + } + } + return undefined; +} + +/** + * Loads all runnables from separate YAML files in the backend folder. + * Each runnable is stored in a file named `.yaml`. + * + * Converts from file format to API format: + * - For inline scripts (type: 'inline'): derives inlineScript from sibling files + * - For path-based runnables (type: 'script'|'hubscript'|'flow'): converts to API format + * e.g., { type: "script" } -> { type: "path", runType: "script" } + * + * Returns an empty object if the backend folder doesn't exist. + * + * @param backendPath - Path to the backend folder + * @param defaultTs - Default TypeScript runtime ("bun" or "deno") + */ +export async function loadRunnablesFromBackend( + backendPath: string, + defaultTs: "bun" | "deno" = "bun" +): Promise> { + const runnables: Record = {}; + + try { + // First, collect all files in the backend folder + const allFiles: string[] = []; + for await (const entry of Deno.readDir(backendPath)) { + if (entry.isFile) { + allFiles.push(entry.name); + } + } + + // Process YAML files (runnable metadata files) + for (const fileName of allFiles) { + if (!fileName.endsWith(".yaml")) { + continue; + } + + const runnableId = fileName.replace(".yaml", ""); + const filePath = path.join(backendPath, fileName); + const runnable = (await yamlParseFile(filePath)) as Record; + + // If this is an inline script (type: 'inline'), derive inlineScript from files + if (runnable?.type === "inline") { + const contentFile = await findRunnableContentFile( + backendPath, + runnableId, + allFiles + ); + + if (contentFile) { + const language = getLanguageFromExtension(contentFile.ext, defaultTs); + + // Try to load lock file + let lock: string | undefined; + try { + lock = await Deno.readTextFile( + path.join(backendPath, `${runnableId}.lock`) + ); + } catch { + // No lock file, that's fine + } + + // Reconstruct inlineScript object + runnable.inlineScript = { + content: contentFile.content, + language, + ...(lock ? { lock } : {}), + }; + } + } else if ( + runnable?.type === "script" || + runnable?.type === "hubscript" || + runnable?.type === "flow" + ) { + // For path-based runnables, convert from file format to API format + // { type: "script" } -> { type: "path", runType: "script" } + // { type: "hubscript" } -> { type: "path", runType: "hubscript" } + // { type: "flow" } -> { type: "path", runType: "flow" } + const { type, schema: _schema, ...rest } = runnable; + runnable.type = "path"; + runnable.runType = type; + // Remove schema if present + delete runnable.schema; + Object.assign(runnable, rest); + } + + runnables[runnableId] = runnable; + } + } catch (error: any) { + if (error.name !== "NotFound") { + throw error; + } + } + + return runnables; +} + +/** + * Writes a single runnable to its YAML file in the backend folder. + * The file will be named `.yaml`. + * + * Converts from API format to file format: + * - For inline scripts: keeps type: "inline" + * - For path-based runnables: converts { type: "path", runType: "script" } to { type: "script" } + * and removes schema field + */ +export function writeRunnableToBackend( + backendPath: string, + runnableId: string, + runnable: any +): void { + let runnableToWrite = { ...runnable }; + + // Convert path-based runnables from API format to file format + if (runnable.type === "path" && runnable.runType) { + // { type: "path", runType: "script" } -> { type: "script" } + const { type: _type, runType, schema: _schema, ...rest } = runnable; + runnableToWrite = { + type: runType, + ...rest, + }; + } + + const filePath = path.join(backendPath, `${runnableId}.yaml`); + writeIfChanged(filePath, yamlStringify(runnableToWrite, yamlOptions)); +} + const alreadySynced: string[] = []; async function collectAppFiles( @@ -50,10 +223,10 @@ async function collectAppFiles( await readDirRecursive(fullPath + SEP, relativePath + SEP); } else if (entry.isFile) { // Skip raw_app.yaml as it's metadata, not an app file - // Skip node_modules and package-lock.json as they are generated + // Skip package-lock.json as it's generated if ( - relativePath === "raw_app.yaml" || - relativePath === "package-lock.json" + entry.name === "raw_app.yaml" || + entry.name === "package-lock.json" ) { continue; } @@ -99,15 +272,48 @@ export async function pushRawApp( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const path = localPath + "raw_app.yaml"; - const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts( - localApp.runnables, - localPath + SEP + APP_BACKEND_FOLDER + SEP, - true + const appFilePath = localPath + "raw_app.yaml"; + const localApp = (await yamlParseFile(appFilePath)) as AppFile; + + // Load runnables from separate YAML files in the backend folder + // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); + const runnablesFromBackend = await loadRunnablesFromBackend(backendPath); + + let runnables: Record; + if (Object.keys(runnablesFromBackend).length > 0) { + // Use runnables from separate files (new format) + runnables = runnablesFromBackend; + log.info( + colors.gray( + `Loaded ${Object.keys(runnables).length} runnables from backend folder` + ) + ); + } else if (localApp.runnables) { + // Fall back to runnables from raw_app.yaml (old format) + runnables = localApp.runnables; + log.info( + colors.gray( + `Loaded ${ + Object.keys(runnables).length + } runnables from raw_app.yaml (legacy format)` + ) + ); + } else { + runnables = {}; + } + + replaceInlineScripts(runnables, backendPath + SEP, true); + repopulateFields(runnables); + + // Create a temporary app object for policy generation + const appForPolicy = { ...localApp, runnables }; + await generatingPolicy( + appForPolicy, + remotePath, + localApp?.["public"] ?? false ); - repopulateFields(localApp.runnables); - await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false); + const files = await collectAppFiles(localPath); async function createBundleRaw() { log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); @@ -123,7 +329,7 @@ export async function pushRawApp( }); } if (app) { - if (isSuperset(localApp, app)) { + if (isSuperset({ ...localApp, runnables }, app)) { log.info(colors.green(`App ${remotePath} is up to date`)); return; } @@ -134,10 +340,10 @@ export async function pushRawApp( path: remotePath, formData: { app: { - value: { runnables: localApp.runnables, files }, + value: { runnables, files }, path: remotePath, summary: localApp.summary, - policy: localApp.policy, + policy: appForPolicy.policy, deployment_message: message, custom_path: localApp.custom_path, }, @@ -151,10 +357,10 @@ export async function pushRawApp( workspace, formData: { app: { - value: { runnables: localApp.runnables, files }, + value: { runnables, files }, path: remotePath, summary: localApp.summary, - policy: localApp.policy, + policy: appForPolicy.policy, deployment_message: message, custom_path: localApp.custom_path, }, @@ -162,16 +368,6 @@ export async function pushRawApp( css, }, }); - // await wmill.createApp({ - // workspace, - // requestBody: { - // path: remotePath, - // deployment_message: message, - // value: { runnables: localApp.runnables, files }, - // summary: localApp.summary, - // policy: localApp.policy, - // }, - // }); } } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 97a4f085cc..d359d3884c 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -66,6 +66,17 @@ export interface ScriptFile { kind?: "script" | "failure" | "trigger" | "command" | "approval"; } +/** + * Checks if a path is inside a raw app backend folder. + * Matches patterns like: .../myApp.raw_app/backend/... + */ +export function isRawAppBackendPath(filePath: string): boolean { + // Normalize path separators for consistent matching + const normalizedPath = filePath.replaceAll(SEP, "/"); + // Check if path contains pattern: *.raw_app/backend/ + return /\.raw_app\/backend\//.test(normalizedPath); +} + type PushOptions = GlobalOptions; async function push(opts: PushOptions, filePath: string) { opts = await mergeConfigWithConfigFile(opts); @@ -199,6 +210,7 @@ export async function handleFile( ): Promise { if ( !path.includes(".inline_script.") && + !isRawAppBackendPath(path) && exts.some((exts) => path.endsWith(exts)) ) { if (alreadySynced.includes(path)) { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 57222f482b..40a0928f59 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -70,6 +70,7 @@ import { OpenFlow } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; import { newPathAssigner, + newRawAppPathAssigner, PathAssigner, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; @@ -284,10 +285,6 @@ function extractFields(fields: Record) { fields[k] = undefined; } } - // if (k == 'runType') { - // fields["type"] = undefined - // fields["schema"] = undefined - // } }); } @@ -311,11 +308,7 @@ export function extractInlineScriptsForApps( } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { - if (k == "runType") { - rec["type"] = undefined; - rec["schema"] = undefined; - return []; - } else if (k == "inlineScript" && typeof v == "object") { + if (k == "inlineScript" && typeof v == "object") { rec["type"] = undefined; const o: Record = v as any; const name = toId(key ?? "", rec); @@ -509,13 +502,14 @@ function ZipFSElement( rawApp.policy = undefined; let inlineScripts; const value = rawApp?.["value"]; + const runnables = value?.["runnables"] ?? {}; // console.log("FOOB", value?.["runnables"]) - extractFieldsForRawApps(value?.["runnables"]); + extractFieldsForRawApps(runnables); try { inlineScripts = extractInlineScriptsForApps( undefined, value, - newPathAssigner(defaultTs), + newRawAppPathAssigner(defaultTs), (key, val_) => key ); } catch (error) { @@ -549,6 +543,7 @@ function ZipFSElement( throw error; } + // Yield inline script content and lock files for (const s of inlineScripts) { yield { isDirectory: false, @@ -561,12 +556,58 @@ function ZipFSElement( }; } - const runnables = value?.["runnables"]; - if (runnables) { - rawApp.runnables = runnables; - delete rawApp?.["value"]; + // Yield each runnable as a separate YAML file in the backend folder + // For inline scripts, simplify the YAML - inlineScript is not needed since + // content/lock/language can be derived from sibling files + for (const [runnableId, runnable] of Object.entries(runnables)) { + const runnableObj = runnable as Record; + let simplifiedRunnable: Record; + + if (runnableObj.inlineScript) { + // For inline scripts, remove inlineScript and just keep type: 'inline' + // plus any other metadata (name, fields, etc.) + simplifiedRunnable = { type: "inline" }; + + // Copy over any other fields that aren't inlineScript or type + for (const [key, value] of Object.entries(runnableObj)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } + } else if (runnableObj.type === "path" && runnableObj.runType) { + // For path-based runnables, convert from API format to file format + // { type: "path", runType: "script" } -> { type: "script" } + // Also remove schema field + const { type: _type, runType, schema: _schema, ...rest } = + runnableObj; + simplifiedRunnable = { + type: runType, + ...rest, + }; + } else { + // For other runnables, keep as-is + simplifiedRunnable = runnableObj; + } + + yield { + isDirectory: false, + path: path.join( + finalPath, + APP_BACKEND_FOLDER, + `${runnableId}.yaml` + ), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return yamlStringify(simplifiedRunnable, yamlOptions); + }, + }; } + // Remove runnables and value from raw_app.yaml - they are now in separate files + delete rawApp?.["value"]; + // Don't include runnables in raw_app.yaml anymore + yield { isDirectory: false, path: path.join(finalPath, "raw_app.yaml"), diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index d1a79a3a32..d34dcb35db 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -40,13 +40,13 @@ export const LANGUAGE_EXTENSIONS: Record = { /** * Gets the appropriate file extension for a given programming language. * Handles special cases for TypeScript variants based on the default runtime. - * + * * @param language - The programming language to get extension for * @param defaultTs - Default TypeScript runtime ("bun" or "deno") * @returns File extension string (without the dot) */ export function getLanguageExtension( - language: SupportedLanguage, + language: SupportedLanguage, defaultTs: "bun" | "deno" = "bun" ): string { if (language === defaultTs || language === "bunnative") { @@ -55,13 +55,67 @@ export function getLanguageExtension( return LANGUAGE_EXTENSIONS[language] || "no_ext"; } +/** + * Reverse mapping from file extensions to languages. + * Used when deriving language from file extension. + */ +export const EXTENSION_TO_LANGUAGE: Record = { + "py": "python3", + "bun.ts": "bun", + "deno.ts": "deno", + "go": "go", + "sh": "bash", + "ps1": "powershell", + "pg.sql": "postgresql", + "my.sql": "mysql", + "bq.sql": "bigquery", + "odb.sql": "oracledb", + "sf.sql": "snowflake", + "ms.sql": "mssql", + "gql": "graphql", + "native.ts": "nativets", + "frontend.js": "frontend", + "php": "php", + "rs": "rust", + "cs": "csharp", + "nu": "nu", + "playbook.yml": "ansible", + "java": "java", + "duckdb.sql": "duckdb", + // Plain .ts defaults to bun (will be overridden by defaultTs setting) + "ts": "bun", +}; + +/** + * Gets the language from a file extension. + * + * @param ext - File extension (e.g., "py", "ts", "bun.ts") + * @param defaultTs - Default TypeScript runtime for plain .ts files + * @returns The language, or undefined if not recognized + */ +export function getLanguageFromExtension( + ext: string, + defaultTs: "bun" | "deno" = "bun" +): SupportedLanguage | undefined { + // Check for compound extensions first (e.g., "bun.ts", "pg.sql") + const lang = EXTENSION_TO_LANGUAGE[ext]; + if (lang) { + // For plain .ts, return the default TypeScript runtime + if (ext === "ts") { + return defaultTs; + } + return lang; + } + return undefined; +} + export interface PathAssigner { assignPath(summary: string | undefined, language: SupportedLanguage): [string, string]; } /** * Creates a new path assigner for inline scripts. - * + * * @param defaultTs - Default TypeScript runtime ("bun" or "deno") * @returns Path assigner function */ @@ -94,4 +148,43 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { return [`${name}.inline_script.`, ext]; } return { assignPath }; +} + +/** + * Creates a new path assigner for raw app runnables. + * Unlike newPathAssigner, this does NOT add ".inline_script." prefix since + * everything in raw_app/backend/ is already known to be for inline scripts. + * + * @param defaultTs - Default TypeScript runtime ("bun" or "deno") + * @returns Path assigner function + */ +export function newRawAppPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { + let counter = 0; + const seen_names = new Set(); + function assignPath( + summary: string | undefined, + language: SupportedLanguage + ): [string, string] { + let name; + + name = summary?.toLowerCase()?.replaceAll(" ", "_") ?? ""; + + let original_name = name; + + if (name == "") { + original_name = "runnable"; + name = `runnable_0`; + } + + while (seen_names.has(name)) { + counter++; + name = `${original_name}_${counter}`; + } + seen_names.add(name); + + const ext = getLanguageExtension(language, defaultTs); + + return [`${name}.`, ext]; + } + return { assignPath }; } \ No newline at end of file diff --git a/frontend/.gitignore b/frontend/.gitignore index c1addf7ed8..bc35cf2ec3 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -12,4 +12,5 @@ storageState.json dist/ static/tsdocs/ static/ui_builder/ -ui_builder.tar.gz \ No newline at end of file +ui_builder.tar.gz +ui_builder_serve/ \ No newline at end of file diff --git a/frontend/serve_ui_builder.sh b/frontend/serve_ui_builder.sh new file mode 100755 index 0000000000..59eb6c3bd3 --- /dev/null +++ b/frontend/serve_ui_builder.sh @@ -0,0 +1,17 @@ +node ./scripts/untar_ui_builder.js + +mkdir ui_builder_serve || true +cp -r static/ui_builder ui_builder_serve/ui_builder || true +rm -rf static/ui_builder || true +python3 -c " +import os +os.chdir('ui_builder_serve') +from http.server import HTTPServer, SimpleHTTPRequestHandler +class H(SimpleHTTPRequestHandler): + def end_headers(self): + self.send_header('Cross-Origin-Opener-Policy', 'same-origin') + self.send_header('Cross-Origin-Embedder-Policy', 'require-corp') + self.send_header('Cross-Origin-Resource-Policy', 'cross-origin') + super().end_headers() +HTTPServer(('', 4000), H).serve_forever() +" \ No newline at end of file diff --git a/frontend/sharedUtils/vite.sharedUtils.config.js b/frontend/sharedUtils/vite.sharedUtils.config.js index 5bf4bcd688..257e103c9f 100644 --- a/frontend/sharedUtils/vite.sharedUtils.config.js +++ b/frontend/sharedUtils/vite.sharedUtils.config.js @@ -5,7 +5,7 @@ import { exec } from 'child_process' import { promisify } from 'util' const execAsync = promisify(exec) -const VERSION = '1.0.10' +const VERSION = '1.0.11' export default defineConfig({ build: { diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 92718fe175..67549a812b 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -101,6 +101,8 @@ import { setMonacoTypescriptOptions } from './monacoLanguagesOptions' import { copilotInfo } from '$lib/aiStore' import { getDbSchemas } from './apps/components/display/dbtable/metadata' + import { rawAppLintStore, type MonacoLintError } from './raw_apps/lintStore' + import { MarkerSeverity } from 'monaco-editor' import { resource, watch } from 'runed' // import EditorTheme from './EditorTheme.svelte' @@ -133,6 +135,8 @@ class?: string | undefined moduleId?: string enablePreprocessorSnippet?: boolean + /** When set, enables raw app lint collection mode and reports Monaco markers to the lint store under this key */ + rawAppRunnableKey?: string | undefined } let { @@ -160,7 +164,8 @@ key = undefined, class: clazz = undefined, moduleId = undefined, - enablePreprocessorSnippet = false + enablePreprocessorSnippet = false, + rawAppRunnableKey = undefined }: Props = $props() $effect.pre(() => { @@ -353,6 +358,25 @@ editor.pushUndoStop() } } + // Update lint diagnostics after code change + updateRawAppLintDiagnostics() + } + + /** Collect Monaco markers and update the raw app lint store */ + function updateRawAppLintDiagnostics(): void { + if (!rawAppRunnableKey || !model) return + const markers = meditor.getModelMarkers({ resource: model.uri }) + const lintErrors: MonacoLintError[] = markers + .filter((m) => m.severity === MarkerSeverity.Error || m.severity === MarkerSeverity.Warning) + .map((m) => ({ + message: m.message, + severity: m.severity === MarkerSeverity.Error ? 'error' : 'warning', + startLineNumber: m.startLineNumber, + startColumn: m.startColumn, + endLineNumber: m.endLineNumber, + endColumn: m.endColumn + })) + rawAppLintStore.setDiagnostics(rawAppRunnableKey, lintErrors) } function updateCode() { @@ -1331,6 +1355,20 @@ // updateEditorKeybindingsMode(editor, 'vim', undefined) + // Raw app lint collection: listen for marker changes and report to store + let markerChangeDisposable: IDisposable | undefined = undefined + if (rawAppRunnableKey && model) { + markerChangeDisposable = meditor.onDidChangeMarkers((uris) => { + if (!model || !rawAppRunnableKey) return + const modelUri = model.uri.toString() + if (uris.some((u) => u.toString() === modelUri)) { + updateRawAppLintDiagnostics() + } + }) + // Initial lint diagnostics collection + updateRawAppLintDiagnostics() + } + let ataModel: number | undefined = undefined editor?.onDidChangeModelContent((event) => { @@ -1450,6 +1488,9 @@ closeWebsockets() vimDisposable?.dispose() closeAIInlineWidget() + markerChangeDisposable?.dispose() + // Note: We don't clear lint diagnostics on dispose - they persist across runnable switches + // Diagnostics are only updated when Monaco reports new markers for this runnable console.log('disposing editor') model?.dispose() editor && editor.dispose() diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index cb700e2676..e4f8655157 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -1067,13 +1067,11 @@ onSummaryChange(script.summary)} + onChange={() => onSummaryChange(script.summary)} elementProps={{ type: 'text', placeholder: 'Short summary to be displayed when listed' @@ -1103,7 +1101,6 @@