diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d9a24e861..d5b943d7f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.589.3](https://github.com/windmill-labs/windmill/compare/v1.589.2...v1.589.3) (2025-12-05) + + +### Bug Fixes + +* **app:** fix appdatetimeinput in lists ([409c342](https://github.com/windmill-labs/windmill/commit/409c342ffd4499c5ab67b3b5acca8977753ded00)) +* **cli:** properly handle frontend scripts for app generate-locks ([af70eed](https://github.com/windmill-labs/windmill/commit/af70eed58d9b38f48891e14632660e3aa10ee35e)) +* linked secret in resources must be of type string ([9746030](https://github.com/windmill-labs/windmill/commit/97460304e9ed54b9a04e055b884e2fb997c5ab2c)) + +## [1.589.2](https://github.com/windmill-labs/windmill/compare/v1.589.1...v1.589.2) (2025-12-05) + + +### Bug Fixes + +* enable back gcp triggers in CLI ([#7299](https://github.com/windmill-labs/windmill/issues/7299)) ([3757cbc](https://github.com/windmill-labs/windmill/commit/3757cbce87a2d32539efdac7493af2efec22cb7e)) +* **flow:** fix chat mode modal + toggle ([#7296](https://github.com/windmill-labs/windmill/issues/7296)) ([f0ff6f4](https://github.com/windmill-labs/windmill/commit/f0ff6f405dcffa9b70d1a14334c647c29fbf1ed4)) +* **frontend:** fix saved/past inputs loading when in json mode on the flow detail page ([#7300](https://github.com/windmill-labs/windmill/issues/7300)) ([c3044a5](https://github.com/windmill-labs/windmill/commit/c3044a5a9be709807719201cde5eeeb0e70d7133)) +* **frontend:** update workers page ui ([#7264](https://github.com/windmill-labs/windmill/issues/7264)) ([0594257](https://github.com/windmill-labs/windmill/commit/0594257a25d1a64e620daf4ea0b106f47424ef87)) +* show related job when deployment is in progress ([#7294](https://github.com/windmill-labs/windmill/issues/7294)) ([e9f1306](https://github.com/windmill-labs/windmill/commit/e9f13065bfedd9af84da58b161349f8e79e72b2a)) + ## [1.589.1](https://github.com/windmill-labs/windmill/compare/v1.589.0...v1.589.1) (2025-12-03) diff --git a/backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json b/backend/.sqlx/query-05b69dcef0f4f649513e186e73089979c49b4b8113ee832ea7539b56a0415f32.json similarity index 87% rename from backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json rename to backend/.sqlx/query-05b69dcef0f4f649513e186e73089979c49b4b8113ee832ea7539b56a0415f32.json index 4e7b02738e..f221e8da2f 100644 --- a/backend/.sqlx/query-f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf.json +++ b/backend/.sqlx/query-05b69dcef0f4f649513e186e73089979c49b4b8113ee832ea7539b56a0415f32.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -45,6 +45,11 @@ }, { "ordinal": 8, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 9, "name": "language: ScriptLang", "type_info": { "Custom": { @@ -80,42 +85,42 @@ } }, { - "ordinal": 9, + "ordinal": 10, "name": "dedicated_worker", "type_info": "Bool" }, { - "ordinal": 10, + "ordinal": 11, "name": "priority", "type_info": "Int2" }, { - "ordinal": 11, + "ordinal": 12, "name": "delete_after_use", "type_info": "Bool" }, { - "ordinal": 12, + "ordinal": 13, "name": "timeout", "type_info": "Int4" }, { - "ordinal": 13, + "ordinal": 14, "name": "has_preprocessor", "type_info": "Bool" }, { - "ordinal": 14, + "ordinal": 15, "name": "on_behalf_of_email", "type_info": "Text" }, { - "ordinal": 15, + "ordinal": 16, "name": "created_by", "type_info": "Varchar" }, { - "ordinal": 16, + "ordinal": 17, "name": "path", "type_info": "Varchar" } @@ -135,6 +140,7 @@ true, true, true, + true, false, true, true, @@ -146,5 +152,5 @@ false ] }, - "hash": "f06ab5e0369b35694fa02c3aac685bd547a1d271eb7401df57fe1774de3211bf" + "hash": "05b69dcef0f4f649513e186e73089979c49b4b8113ee832ea7539b56a0415f32" } diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json b/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json new file mode 100644 index 0000000000..5dd30c1492 --- /dev/null +++ b/backend/.sqlx/query-0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL\n DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "0e621bba5913482b8235d7d8442b8f0e9012c265e150afd4aa41972bf7334ba2" +} diff --git a/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json b/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json new file mode 100644 index 0000000000..56c538d440 --- /dev/null +++ b/backend/.sqlx/query-1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT f.lock_error_logs, dm.job_id\n FROM flow f\n LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version\n AND f.workspace_id = dm.workspace_id AND f.path = dm.path\n WHERE f.path = $1 AND f.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock_error_logs", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true + ] + }, + "hash": "1de29cdd474cbd61e15b63d111e1c42aefee683e14cc738a809ecca17370e6ee" +} diff --git a/backend/.sqlx/query-23759cb515e926e272bbc8e5d8a0a9d039b99bc2026e381e99ef41cdaf6ea19f.json b/backend/.sqlx/query-23759cb515e926e272bbc8e5d8a0a9d039b99bc2026e381e99ef41cdaf6ea19f.json new file mode 100644 index 0000000000..249c230319 --- /dev/null +++ b/backend/.sqlx/query-23759cb515e926e272bbc8e5d8a0a9d039b99bc2026e381e99ef41cdaf6ea19f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "23759cb515e926e272bbc8e5d8a0a9d039b99bc2026e381e99ef41cdaf6ea19f" +} diff --git a/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json b/backend/.sqlx/query-3d05d9d7e087eb6e1c14c2b8a20598581e6c7493ed99cb9ad1c2ee5d0b212d38.json similarity index 87% rename from backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json rename to backend/.sqlx/query-3d05d9d7e087eb6e1c14c2b8a20598581e6c7493ed99cb9ad1c2ee5d0b212d38.json index ecaf828737..521b7cb1c0 100644 --- a/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json +++ b/backend/.sqlx/query-3d05d9d7e087eb6e1c14c2b8a20598581e6c7493ed99cb9ad1c2ee5d0b212d38.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37)", "describe": { "columns": [], "parameters": { @@ -85,10 +85,11 @@ "Bool", "Jsonb", "Varchar", - "Int4" + "Int4", + "Bool" ] }, "nullable": [] }, - "hash": "0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d" + "hash": "3d05d9d7e087eb6e1c14c2b8a20598581e6c7493ed99cb9ad1c2ee5d0b212d38" } diff --git a/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json b/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json new file mode 100644 index 0000000000..31187af03a --- /dev/null +++ b/backend/.sqlx/query-513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT s.lock, s.lock_error_logs, dm.job_id\n FROM script s\n LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id\n WHERE s.hash = $1 AND s.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "lock_error_logs", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "513ed713afdbafb587026d1536c47a9bbaa6967e36777454746b8817d68219a5" +} diff --git a/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json b/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json deleted file mode 100644 index 902ed458c2..0000000000 --- a/backend/.sqlx/query-53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "53dee7c119d724624b9973ee981576154cec84a09069286d2d7144dbad54f4d6" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json b/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json similarity index 90% rename from backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json rename to backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json index fa013ba585..272201898c 100644 --- a/backend/.sqlx/query-3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5.json +++ b/backend/.sqlx/query-6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n cache_ignore_s3_path,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner,\n NULL as permissioned_as_end_user_email\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", "describe": { "columns": [ { @@ -193,26 +193,31 @@ }, { "ordinal": 26, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 27, "name": "priority", "type_info": "Int2" }, { - "ordinal": 27, + "ordinal": 28, "name": "preprocessed", "type_info": "Bool" }, { - "ordinal": 28, + "ordinal": 29, "name": "script_entrypoint_override", "type_info": "Varchar" }, { - "ordinal": 29, + "ordinal": 30, "name": "trigger", "type_info": "Varchar" }, { - "ordinal": 30, + "ordinal": 31, "name": "trigger_kind: JobTriggerKind", "type_info": { "Custom": { @@ -238,12 +243,12 @@ } }, { - "ordinal": 31, + "ordinal": 32, "name": "visible_to_owner", "type_info": "Bool" }, { - "ordinal": 32, + "ordinal": 33, "name": "permissioned_as_end_user_email", "type_info": "Text" } @@ -285,9 +290,10 @@ true, true, true, + true, false, null ] }, - "hash": "3162ec92bb32af47a71cc41172cc740b5dea1304ce4dfdb4d3d0efa4266f38c5" + "hash": "6c97ab28ab47b75fb3ff39ea70fa3627f08b61bbd33aecb9ea816f8f78a04ec5" } diff --git a/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json b/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json new file mode 100644 index 0000000000..571a374b96 --- /dev/null +++ b/backend/.sqlx/query-7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL\n DO UPDATE SET job_id = EXCLUDED.job_id", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "7abde47077c38ccf005ce7180a383f97076b2cbe2f617f7af39550a0db157b2b" +} diff --git a/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json b/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json similarity index 93% rename from backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json rename to backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json index 4b9bfe326b..5ce7b053fc 100644 --- a/backend/.sqlx/query-6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750.json +++ b/backend/.sqlx/query-7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, \n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", + "query": "SELECT \n j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, \n j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path\n FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id\n WHERE j.id = $1 AND j.workspace_id = $2", "describe": { "columns": [ { @@ -181,6 +181,11 @@ "ordinal": 19, "name": "cache_ttl", "type_info": "Int4" + }, + { + "ordinal": 20, + "name": "cache_ignore_s3_path", + "type_info": "Bool" } ], "parameters": { @@ -209,8 +214,9 @@ true, true, false, + true, true ] }, - "hash": "6cc922a5bbd348c938a9d1431aaa0f24f078ea814b429d44403aca1e5002e750" + "hash": "7b5ad10af2a9b34fa86429499ea24c0c09c6e7e9ebfa3af90035570133f7c579" } diff --git a/backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json b/backend/.sqlx/query-7f9b7ab9bec6a0f745273d0cd5602ceab46a7ec9fd225f7b9d16a2ddb9bad7b3.json similarity index 84% rename from backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json rename to backend/.sqlx/query-7f9b7ab9bec6a0f745273d0cd5602ceab46a7ec9fd225f7b9d16a2ddb9bad7b3.json index c540742fe2..9033ac194b 100644 --- a/backend/.sqlx/query-27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32.json +++ b/backend/.sqlx/query-7f9b7ab9bec6a0f745273d0cd5602ceab46a7ec9fd225f7b9d16a2ddb9bad7b3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script\n WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false)\n ORDER BY created_at DESC LIMIT 1", "describe": { "columns": [ { @@ -45,6 +45,11 @@ }, { "ordinal": 8, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 9, "name": "language: ScriptLang", "type_info": { "Custom": { @@ -80,27 +85,27 @@ } }, { - "ordinal": 9, + "ordinal": 10, "name": "dedicated_worker", "type_info": "Bool" }, { - "ordinal": 10, + "ordinal": 11, "name": "priority", "type_info": "Int2" }, { - "ordinal": 11, + "ordinal": 12, "name": "timeout", "type_info": "Int4" }, { - "ordinal": 12, + "ordinal": 13, "name": "on_behalf_of_email", "type_info": "Text" }, { - "ordinal": 13, + "ordinal": 14, "name": "created_by", "type_info": "Varchar" } @@ -121,6 +126,7 @@ true, true, true, + true, false, true, true, @@ -129,5 +135,5 @@ false ] }, - "hash": "27a54f8188c25c2c089c818a991ca1c092f67227be217161d6e6617ddbf77b32" + "hash": "7f9b7ab9bec6a0f745273d0cd5602ceab46a7ec9fd225f7b9d16a2ddb9bad7b3" } diff --git a/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json b/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json deleted file mode 100644 index 739184045f..0000000000 --- a/backend/.sqlx/query-97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "lock_error_logs", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "97bf27f210572499b42ce04f19f116cc87ed06c49dcca04360250ddfd89d7ab3" -} diff --git a/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json b/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json similarity index 91% rename from backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json rename to backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json index 73fb2e98d8..8b9b0246b2 100644 --- a/backend/.sqlx/query-d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd.json +++ b/backend/.sqlx/query-a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", \n permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", + "query": "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", \n permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, cache_ignore_s3_path, r.ping as last_ping, worker, memory_peak, running\n FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id)\n WHERE j.id = $1", "describe": { "columns": [ { @@ -184,21 +184,26 @@ }, { "ordinal": 20, + "name": "cache_ignore_s3_path", + "type_info": "Bool" + }, + { + "ordinal": 21, "name": "last_ping", "type_info": "Timestamptz" }, { - "ordinal": 21, + "ordinal": 22, "name": "worker", "type_info": "Varchar" }, { - "ordinal": 22, + "ordinal": 23, "name": "memory_peak", "type_info": "Int4" }, { - "ordinal": 23, + "ordinal": 24, "name": "running", "type_info": "Bool" } @@ -232,8 +237,9 @@ true, true, true, + true, false ] }, - "hash": "d48c9a748746080e9b6cf0366b2ca3516ac5c1d32a98946196b1a42e3f103efd" + "hash": "a84e67035584bbdb02482026b9cc0808086c50f78947d43bb88628a481f41a1d" } diff --git a/backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json b/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json similarity index 94% rename from backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json rename to backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json index 46db07e1e0..ee004c664c 100644 --- a/backend/.sqlx/query-bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971.json +++ b/backend/.sqlx/query-b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH inserted_job AS (\n INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) \n values ($1, $32, $33, $34, $35, $36, $37, $2, $41) \n ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31)", + "query": "WITH inserted_job AS (\n INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) \n values ($1, $32, $33, $34, $35, $36, $37, $2, $41) \n ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42)", "describe": { "columns": [], "parameters": { @@ -124,10 +124,11 @@ } }, "Bool", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "bf2163c542fb8c4e173167a8f333ef762fecf782424c5b61b89f32918b8d6971" + "hash": "b179a3f876ca659bed892d464bf51a733cc86a3204fcd9edccda63fddc97dced" } diff --git a/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json b/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json deleted file mode 100644 index e8cba862fe..0000000000 --- a/backend/.sqlx/query-e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "lock", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "lock_error_logs", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "e3ac59fcf6193007a21c808a275d3d84fd76e44f9240f4292c5c1208096b8563" -} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 38a2a68877..39f54b6fb4 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -3404,7 +3404,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", - "sourcemap 9.2.2", + "sourcemap 9.3.0", "swc_atoms", "swc_common", "swc_config", @@ -5378,14 +5378,14 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.5" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +checksum = "a2152dbcb980c05735e2a651d96011320a949eb31a0c8b38b72645ce97dec676" dependencies = [ "crc32fast", - "libz-rs-sys", "libz-sys", "miniz_oxide 0.8.9", + "zlib-rs", ] [[package]] @@ -7764,15 +7764,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "libz-rs-sys" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222" -dependencies = [ - "zlib-rs", -] - [[package]] name = "libz-sys" version = "1.1.23" @@ -8206,9 +8197,9 @@ dependencies = [ [[package]] name = "minicov" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" dependencies = [ "cc", "walkdir", @@ -8253,9 +8244,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ "libc", "wasi", @@ -12166,9 +12157,9 @@ dependencies = [ [[package]] name = "sourcemap" -version = "9.2.2" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22afbcb92ce02d23815b9795523c005cb9d3c214f8b7a66318541c240ea7935" +checksum = "c8131753a3c444a6177b92fc9e1bd0d2ecaf1c8953e6a41cc599e8e7ca317ef2" dependencies = [ "base64-simd 0.8.0", "bitvec", @@ -12693,7 +12684,7 @@ dependencies = [ "rustc-hash 1.1.0", "serde", "siphasher 0.3.11", - "sourcemap 9.2.2", + "sourcemap 9.3.0", "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", @@ -12757,7 +12748,7 @@ dependencies = [ "num-bigint", "once_cell", "serde", - "sourcemap 9.2.2", + "sourcemap 9.3.0", "swc_allocator", "swc_atoms", "swc_common", @@ -13647,7 +13638,7 @@ dependencies = [ "bytes", "io-uring", "libc", - "mio 1.1.0", + "mio 1.1.1", "parking_lot", "pin-project-lite", "signal-hook-registry", @@ -15165,7 +15156,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "aws-sdk-config", @@ -15227,7 +15218,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "argon2", @@ -15348,7 +15339,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.589.1" +version = "1.589.3" dependencies = [ "base64 0.22.1", "chrono", @@ -15363,7 +15354,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.589.1" +version = "1.589.3" dependencies = [ "chrono", "lazy_static", @@ -15377,7 +15368,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "axum", @@ -15396,7 +15387,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "async-recursion", @@ -15489,7 +15480,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.589.1" +version = "1.589.3" dependencies = [ "regex", "serde", @@ -15504,7 +15495,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "bytes", @@ -15528,7 +15519,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.589.1" +version = "1.589.3" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15544,7 +15535,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.589.1" +version = "1.589.3" dependencies = [ "convert_case 0.6.0", "serde", @@ -15553,7 +15544,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "lazy_static", @@ -15565,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "serde_json", @@ -15577,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "gosyn", @@ -15589,7 +15580,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "lazy_static", @@ -15601,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "serde_json", @@ -15613,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "nu-parser", @@ -15624,7 +15615,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15635,7 +15626,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15647,7 +15638,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "async-recursion", @@ -15671,7 +15662,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "lazy_static", @@ -15685,7 +15676,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15702,7 +15693,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "lazy_static", @@ -15716,7 +15707,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "lazy_static", @@ -15734,7 +15725,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "serde", @@ -15745,7 +15736,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "async-recursion", @@ -15782,7 +15773,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.589.1" +version = "1.589.3" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15792,7 +15783,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.589.1" +version = "1.589.3" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 67d8d351db..564f8f9b66 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.589.1" +version = "1.589.3" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.589.1" +version = "1.589.3" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f8adfef664..2861f96db9 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5e950542fed5ce5d6154932b3bc6b2172cb32c73 +9501f6dc030f4235984286cf00762ab4bb54c846 diff --git a/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql new file mode 100644 index 0000000000..0b04137182 --- /dev/null +++ b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.down.sql @@ -0,0 +1,2 @@ +-- Remove job_id column from deployment_metadata table +ALTER TABLE deployment_metadata DROP COLUMN IF EXISTS job_id; diff --git a/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql new file mode 100644 index 0000000000..8c54c702cd --- /dev/null +++ b/backend/migrations/20251204080000_add_job_id_to_deployment_metadata.up.sql @@ -0,0 +1,2 @@ +-- Add job_id column to deployment_metadata table to track the current deployment job +ALTER TABLE deployment_metadata ADD COLUMN IF NOT EXISTS job_id UUID; diff --git a/backend/migrations/20251204210743_cache_ignore_s3_path.down.sql b/backend/migrations/20251204210743_cache_ignore_s3_path.down.sql new file mode 100644 index 0000000000..cac9727e15 --- /dev/null +++ b/backend/migrations/20251204210743_cache_ignore_s3_path.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE v2_job_queue +DROP COLUMN cache_ignore_s3_path; + +ALTER TABLE script +DROP COLUMN cache_ignore_s3_path; diff --git a/backend/migrations/20251204210743_cache_ignore_s3_path.up.sql b/backend/migrations/20251204210743_cache_ignore_s3_path.up.sql new file mode 100644 index 0000000000..dfbf97a2c8 --- /dev/null +++ b/backend/migrations/20251204210743_cache_ignore_s3_path.up.sql @@ -0,0 +1,5 @@ +ALTER TABLE script +ADD COLUMN cache_ignore_s3_path BOOLEAN DEFAULT NULL; + +ALTER TABLE v2_job_queue +ADD COLUMN cache_ignore_s3_path BOOLEAN DEFAULT NULL; diff --git a/backend/tests/common/mod.rs b/backend/tests/common/mod.rs index 129fd86d63..7150eaaf84 100644 --- a/backend/tests/common/mod.rs +++ b/backend/tests/common/mod.rs @@ -684,16 +684,14 @@ pub async fn run_deployed_relative_imports( let job = RunJob::from(JobPayload::ScriptHash { path: "f/system/test_import".to_string(), hash: ScriptHash(script.hash), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language, priority: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), }) .push(&db2) .await; @@ -739,13 +737,11 @@ pub async fn run_preview_relative_imports( path: Some("f/system/test_import".to_string()), language, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .push(&db2) .await; diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 22e713329b..2ad2bf339d 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -52,12 +52,10 @@ mod job_payload { let result = RunJob::from(JobPayload::ScriptHash { hash: ScriptHash(123412), path: "f/system/hello".to_string(), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: ScriptLang::Deno, priority: None, @@ -86,16 +84,14 @@ mod job_payload { let job = RunJob::from(JobPayload::ScriptHash { hash: ScriptHash(123413), path: "f/system/hello_with_preprocessor".to_string(), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, language: ScriptLang::Deno, priority: None, apply_preprocessor: true, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), }) .run_until_complete_with(db, false, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -167,10 +163,9 @@ mod job_payload { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[0], language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, path: "f/system/hello/test-0".into(), }) @@ -187,10 +182,9 @@ mod job_payload { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[1], language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, path: "f/system/hello/test-0".into(), }) diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 549e052b65..cb5909f183 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -25,7 +25,10 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# workspace-dependencies-mode: manual\n# py: 3.11.11","tiny==0.1.3"], + vec![ + "# workspace-dependencies-mode: manual\n# py: 3.11.11", + "tiny==0.1.3", + ], ) .await?; Ok(()) @@ -186,12 +189,10 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, }); @@ -237,12 +238,10 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, }); @@ -273,12 +272,10 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, }); @@ -314,12 +311,10 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, }); @@ -353,12 +348,10 @@ def main(): path: None, language: ScriptLang::Python3, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, }); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 836e6d149c..b7a46b8c78 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -188,9 +188,8 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default() + .into(), is_trigger: None, assets: None, } @@ -202,6 +201,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -234,9 +234,8 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: + windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, } @@ -248,6 +247,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -267,6 +267,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -368,9 +369,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, @@ -382,6 +381,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -425,9 +425,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, }.into(), @@ -438,6 +436,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -466,9 +465,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, @@ -480,6 +477,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -499,6 +497,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -534,9 +533,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), is_trigger: None, assets: None, }.into(), @@ -547,6 +544,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -864,13 +862,11 @@ func main(derp string) (string, error) { path: None, lock: None, language: ScriptLang::Go, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("derp", json!("world")) .run_until_complete(&db, false, port) @@ -903,13 +899,11 @@ fn main(world: String) -> Result { path: None, lock: None, language: ScriptLang::Rust, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, + cache_ignore_s3_path: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), cache_ttl: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, })) .arg("world", json!("Hyrule")) .run_until_complete(&db, false, port) @@ -981,13 +975,11 @@ echo "hello $msg" path: None, lock: None, language: ScriptLang::Bash, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1016,13 +1008,11 @@ def main [ msg: string ] { path: None, lock: None, language: ScriptLang::Nu, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("msg", json!("world")) .run_until_complete(&db, false, port) @@ -1071,13 +1061,11 @@ def main [ path: None, lock: None, language: ScriptLang::Nu, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("3")) .arg("b", json!("null")) @@ -1135,13 +1123,11 @@ public class Main { path: None, lock: None, language: ScriptLang::Java, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!(3)) .arg("b", json!(3.0)) @@ -1172,13 +1158,11 @@ export async function main(a: Date) { path: None, lock: None, language: ScriptLang::Bun, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1209,13 +1193,11 @@ export async function main(a: Date) { path: None, lock: None, language: ScriptLang::Deno, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .run_until_complete(&db, false, port) @@ -1247,13 +1229,11 @@ def main(a: datetime, b: bytes): path: None, lock: None, language: ScriptLang::Python3, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: windmill_common::jobs::ConcurrencySettings::default().into(), + debouncing_settings: windmill_common::jobs::DebouncingSettings::default(), })) .arg("a", json!("2024-09-24T10:00:00.000Z")) .arg("b", json!("dGVzdA==")) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ff11d3d6ac..44bdee8d11 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.589.1 + version: 1.589.3 title: Windmill API contact: @@ -6005,6 +6005,9 @@ paths: type: string lock_error_logs: type: string + job_id: + type: string + format: uuid /w/{workspace}/jobs/list_selected_job_groups: # We use post because sending a huge array as a query param can produce @@ -6891,6 +6894,9 @@ paths: properties: lock_error_logs: type: string + job_id: + type: string + format: uuid /w/{workspace}/flows/get_triggers_count/{path}: get: @@ -16331,6 +16337,8 @@ components: type: integer cache_ttl: type: number + cache_ignore_s3_path: + type: boolean dedicated_worker: type: boolean ws_error_handler_muted: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 4c35907d05..08aca365f2 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1770,12 +1770,12 @@ async fn execute_component( // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), - force_viewer_one_of_fields: Some(one_of_inputs), - force_viewer_allow_user_resources: Some(allow_user_resources), + force_viewer_one_of_fields, + force_viewer_allow_user_resources, .. } => ( &Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() }, - &PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources }, + &PolicyTriggerableInputs { static_inputs, one_of_inputs: force_viewer_one_of_fields.unwrap_or_default(), allow_user_resources: force_viewer_allow_user_resources.unwrap_or_default() }, ), // 2. "run" mode. _ => { diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 88d4f08fe9..d64292765f 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -577,6 +577,20 @@ async fn create_flow( .execute(&mut *new_tx) .await?; + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await?; + new_tx.commit().await?; webhook.send_message( w_id.clone(), @@ -715,13 +729,12 @@ async fn get_flow_version_by_id( let mut tx = user_db.begin(&authed).await?; // First, fetch the path to perform authorization check early - let path: Option = sqlx::query_scalar( - "SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2", - ) - .bind(version) - .bind(&w_id) - .fetch_optional(&mut *tx) - .await?; + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2") + .bind(version) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; let path = not_found_if_none( path, @@ -788,13 +801,12 @@ async fn update_flow_history( let mut tx = user_db.begin(&authed).await?; // Fetch path and perform authorization check early - let path: Option = sqlx::query_scalar( - "SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2", - ) - .bind(&w_id) - .bind(version) - .fetch_optional(&mut *tx) - .await?; + let path: Option = + sqlx::query_scalar("SELECT path FROM flow_version WHERE workspace_id = $1 AND id = $2") + .bind(&w_id) + .bind(version) + .fetch_optional(&mut *tx) + .await?; let path = not_found_if_none( path, @@ -1115,6 +1127,25 @@ async fn update_flow( )) })?; + // Store the job_id in deployment_metadata for this flow deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, flow_version, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + nf.path, + version, + dependency_job_uuid + ) + .execute(&mut *new_tx) + .await + .map_err(|e| { + error::Error::internal_err(format!( + "Error updating deployment_metadata with job_id: {e:#}" + )) + })?; + if let Some(old_dep_job) = old_dep_job { sqlx::query!( "UPDATE v2_job_queue SET @@ -1154,9 +1185,10 @@ async fn list_tokens( list_tokens_internal(&db, &w_id, &path, true).await } -#[derive(FromRow, Serialize)] +#[derive(Serialize)] struct DeploymentStatus { lock_error_logs: Option, + job_id: Option, } async fn get_deployment_status( Extension(db): Extension, @@ -1164,9 +1196,12 @@ async fn get_deployment_status( ) -> JsonResult { let path = path.to_path(); let mut tx = db.begin().await?; - let status_o: Option = sqlx::query_as!( - DeploymentStatus, - "SELECT lock_error_logs FROM flow WHERE path = $1 AND workspace_id = $2", + let status_o = sqlx::query!( + "SELECT f.lock_error_logs, dm.job_id + FROM flow f + LEFT JOIN deployment_metadata dm ON f.versions[array_upper(f.versions, 1)] = dm.flow_version + AND f.workspace_id = dm.workspace_id AND f.path = dm.path + WHERE f.path = $1 AND f.workspace_id = $2", path, w_id, ) @@ -1175,8 +1210,11 @@ async fn get_deployment_status( let status = not_found_if_none(status_o, "DeploymentStatus", path)?; + let deployment_status = + DeploymentStatus { lock_error_logs: status.lock_error_logs, job_id: status.job_id }; + tx.commit().await?; - Ok(Json(status)) + Ok(Json(deployment_status)) } async fn get_flow_by_path( @@ -1440,10 +1478,9 @@ async fn archive_flow_by_path( /// Validates that flow debouncing configuration is supported by all workers /// Returns an error if debouncing is configured but workers are behind required version async fn guard_flow_from_debounce_data(nf: &NewFlow) -> Result<()> { - if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await && { - let flow_value = nf.parse_flow_value()?; - flow_value.debounce_key.is_some() || flow_value.debounce_delay_s.is_some() - } { + if !*MIN_VERSION_SUPPORTS_DEBOUNCING.read().await + && !nf.parse_flow_value()?.debouncing_settings.is_default() + { tracing::warn!( "Flow debouncing configuration rejected: workers are behind minimum required version for debouncing feature" ); @@ -1562,6 +1599,7 @@ mod tests { ConstantDelay, ExponentialDelay, FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, StopAfterIf, }, + jobs::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, scripts, }; @@ -1594,6 +1632,7 @@ mod tests { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -1612,11 +1651,9 @@ mod tests { path: None, lock: None, tag: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, is_trigger: None, assets: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), }), stop_after_if: Some(StopAfterIf { expr: "foo = 'bar'".to_string(), @@ -1628,6 +1665,7 @@ mod tests { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -1660,6 +1698,7 @@ mod tests { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -1691,6 +1730,7 @@ mod tests { retry: None, sleep: None, cache_ttl: None, + cache_ignore_s3_path: None, mock: None, timeout: None, priority: None, @@ -1702,17 +1742,15 @@ mod tests { })), preprocessor_module: None, same_worker: false, - concurrent_limit: None, - concurrency_time_window_s: None, skip_expr: None, cache_ttl: None, + cache_ignore_s3_path: None, priority: None, early_return: None, - concurrency_key: None, chat_input_enabled: None, flow_env: None, - debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }; let expect = serde_json::json!({ "modules": [ diff --git a/backend/windmill-api/src/folders.rs b/backend/windmill-api/src/folders.rs index 86bc5ce7c6..64a792a982 100644 --- a/backend/windmill-api/src/folders.rs +++ b/backend/windmill-api/src/folders.rs @@ -166,7 +166,7 @@ async fn create_folder( Path(w_id): Path, Json(ng): Json, ) -> Result { - let mut tx = user_db.begin(&authed).await?; + let mut tx = user_db.clone().begin(&authed).await?; if !VALID_FOLDER_NAME.is_match(&ng.name) { return Err(windmill_common::error::Error::BadRequest(format!( @@ -216,6 +216,9 @@ async fn create_folder( ) .execute(&mut *tx) .await { + drop(tx); + let mut tx = user_db.begin(&authed).await?; + let exists_for_user = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM folder WHERE name = $1 AND workspace_id = $2 AND $3 = ANY(owners))", ng.name, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 7e4fa4ada2..f3a0cc7e11 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -36,7 +36,8 @@ use windmill_common::flow_conversations::add_message_to_conversation_tx; use windmill_common::flow_status::{JobResult, RestartedFrom}; use windmill_common::jobs::{ check_tag_available_for_workspace_internal, format_completed_job_result, format_result, - DynamicInput, JobTriggerKind, RunInlinePreviewScriptFnParams, ENTRYPOINT_OVERRIDE, + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, DynamicInput, + JobTriggerKind, RunInlinePreviewScriptFnParams, ENTRYPOINT_OVERRIDE, }; use windmill_common::s3_helpers::{upload_artifact_to_store, BundleFormat}; use windmill_common::utils::{RunnableKind, WarnAfterExt}; @@ -287,10 +288,7 @@ pub fn workspaced_service() -> Router { "/completed/import", post(crate::jobs_export::import_completed_jobs).layer(cors.clone()), ) - .route( - "/delete", - post(crate::jobs_export::delete_jobs), - ) + .route("/delete", post(crate::jobs_export::delete_jobs)) .route( "/completed/get/:id", get(get_completed_job).layer(cors.clone()), @@ -892,7 +890,7 @@ macro_rules! get_job_query { get_job_query!( @impl "v2_job_queue", ($($opts)*), "scheduled_for, running, ping as last_ping, suspend, suspend_until, same_worker, pre_run_error, visible_to_owner, \ - flow_innermost_root_job AS root_job, flow_leaf_jobs AS leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl,\ + flow_innermost_root_job AS root_job, flow_leaf_jobs AS leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl, cache_ignore_s3_path, \ script_entrypoint_override", "LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id", ) @@ -1777,6 +1775,7 @@ pub struct RunJobQuery { pub tag: Option, pub timeout: Option, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub skip_preprocessor: Option, pub poll_delay_ms: Option, pub memory_id: Option, @@ -3542,6 +3541,7 @@ impl<'a> From for Job { timeout: None, flow_step_id: None, cache_ttl: None, + cache_ignore_s3_path: None, priority: uj.priority, preprocessed: uj.preprocessed, }, @@ -4598,16 +4598,18 @@ pub async fn run_workflow_as_code( path: job.script_path, language: job.language.unwrap_or_else(|| ScriptLang::Deno), lock: raw_lock, - custom_concurrency_key: windmill_queue::custom_concurrency_key(&db, &job.id) - .await - .map_err(to_anyhow)?, - concurrent_limit: job.concurrent_limit, - concurrency_time_window_s: job.concurrency_time_window_s, + concurrency_settings: windmill_common::jobs::ConcurrencySettingsWithCustom { + custom_concurrency_key: windmill_queue::custom_concurrency_key(&db, &job.id) + .await + .map_err(to_anyhow)?, + concurrent_limit: job.concurrent_limit, + concurrency_time_window_s: job.concurrency_time_window_s, + }, cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, dedicated_worker: None, // TODO(debouncing): enable for this mode - custom_debounce_key: None, - debounce_delay_s: None, + debouncing_settings: DebouncingSettings::default(), }), Some(job.tag.clone()), None, @@ -5426,6 +5428,7 @@ pub async fn run_wait_result_script_by_hash( debounce_key, debounce_delay_s, mut cache_ttl, + mut cache_ignore_s3_path, language, dedicated_worker, priority, @@ -5438,6 +5441,7 @@ pub async fn run_wait_result_script_by_hash( } = get_script_info_for_hash(Some(userdb_authed), &db, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); + cache_ignore_s3_path = run_query.cache_ignore_s3_path; } check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; @@ -5468,12 +5472,19 @@ pub async fn run_wait_result_script_by_hash( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key: concurrency_key, - concurrent_limit: concurrent_limit, - concurrency_time_window_s: concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + concurrency_settings: windmill_common::jobs::ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrent_limit: concurrent_limit, + concurrency_time_window_s: concurrency_time_window_s, + } + .into(), + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() // TODO + }, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, @@ -5944,12 +5955,10 @@ async fn run_preview_script( path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), lock: preview.lock, - custom_concurrency_key: None, - concurrent_limit: None, // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here - concurrency_time_window_s: None, // TODO(gbouv): same as above - custom_debounce_key: None, // TODO(pyra): same as for concurrency limits. - debounce_delay_s: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), // TODO(gbouv): once I find out how to store limits in the content of a script, should be easy to plug limits here + debouncing_settings: DebouncingSettings::default(), // TODO(pyra): same as for concurrency limits. cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: preview.dedicated_worker, }), }, @@ -6099,13 +6108,11 @@ async fn run_bundle_preview_script( path: preview.path, language: preview.language.unwrap_or(ScriptLang::Deno), lock: preview.lock, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: preview.dedicated_worker, - custom_concurrency_key: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&args), authed.display_username(), @@ -6509,19 +6516,19 @@ async fn add_batch_jobs( add_virtual_items_if_necessary(&mut value.modules); let flow_status = FlowStatus::new(&value); ( - None, // script_hash - path, // script_path - job_kind, // job_kind - None, // language - None, // dedicated_worker - value.concurrency_key.clone(), // custom_concurrency_key - value.concurrent_limit.clone(), // concurrent_limit - value.concurrency_time_window_s, // concurrency_time_window_s - None, // timeout - None, // raw_code - None, // raw_lock - Some(value), // raw_flow - Some(flow_status), // flow_status + None, // script_hash + path, // script_path + job_kind, // job_kind + None, // language + None, // dedicated_worker + value.concurrency_settings.concurrency_key.clone(), // custom_concurrency_key + value.concurrency_settings.concurrent_limit.clone(), // concurrent_limit + value.concurrency_settings.concurrency_time_window_s, // concurrency_time_window_s + None, // timeout + None, // raw_code + None, // raw_lock + Some(value), // raw_flow + Some(flow_status), // flow_status ) } "noop" => ( @@ -6890,13 +6897,11 @@ async fn run_dynamic_select( path: None, language: dynamic_input.x_windmill_dyn_select_lang, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default().into(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&request.args.unwrap_or_default()), authed.display_username(), @@ -6976,6 +6981,7 @@ pub async fn run_job_by_hash_inner( debounce_delay_s, debounce_key, mut cache_ttl, + mut cache_ignore_s3_path, language, dedicated_worker, priority, @@ -6990,6 +6996,7 @@ pub async fn run_job_by_hash_inner( check_scopes(&authed, || format!("jobs:run:scripts:{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); + cache_ignore_s3_path = run_query.cache_ignore_s3_path; } let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); @@ -7020,12 +7027,18 @@ pub async fn run_job_by_hash_inner( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key: concurrency_key, - concurrent_limit: concurrent_limit, - concurrency_time_window_s: concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + concurrency_settings: ConcurrencySettings { + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 7cbcf8cc03..fb777b6ed6 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -94,6 +94,8 @@ pub struct ScriptWDraft { #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub dedicated_worker: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ws_error_handler_muted: Option, @@ -801,8 +803,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37)", &w_id, &hash.0, ns.path, @@ -843,6 +845,7 @@ async fn create_script_internal<'c>( ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), ns.debounce_key, ns.debounce_delay_s, + ns.cache_ignore_s3_path, ) .execute(&mut *tx) .await?; @@ -998,14 +1001,14 @@ async fn create_script_internal<'c>( } let tx = PushIsolationLevel::Transaction(tx); - let (_, new_tx) = windmill_queue::push( + let (job_id, mut new_tx) = windmill_queue::push( &db, tx, &w_id, JobPayload::Dependencies { hash, language: ns.language, - path: ns.path, + path: ns.path.clone(), dedicated_worker: ns.dedicated_worker, }, windmill_queue::PushArgs::from(&args), @@ -1034,6 +1037,21 @@ async fn create_script_internal<'c>( None, ) .await?; + + // Store the job_id in deployment_metadata for this script deployment + sqlx::query!( + "INSERT INTO deployment_metadata (workspace_id, path, script_hash, job_id) + VALUES ($1, $2, $3, $4) + ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL + DO UPDATE SET job_id = EXCLUDED.job_id", + w_id, + ns.path, + hash.0, + job_id + ) + .execute(&mut *new_tx) + .await?; + Ok((hash, new_tx, None)) } else { if codebase.is_none() { @@ -1223,7 +1241,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor, on_behalf_of_email, assets, debounce_key, debounce_delay_s FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", @@ -1750,19 +1768,22 @@ async fn raw_script_by_hash( Ok(r.script.content) } -#[derive(FromRow, Serialize)] +#[derive(Serialize)] struct DeploymentStatus { lock: Option, lock_error_logs: Option, + job_id: Option, } async fn get_deployment_status( Extension(db): Extension, Path((w_id, hash)): Path<(String, ScriptHash)>, ) -> JsonResult { let mut tx = db.begin().await?; - let status_o: Option = sqlx::query_as!( - DeploymentStatus, - "SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND workspace_id = $2", + let status_o = sqlx::query!( + "SELECT s.lock, s.lock_error_logs, dm.job_id + FROM script s + LEFT JOIN deployment_metadata dm ON s.hash = dm.script_hash AND s.workspace_id = dm.workspace_id + WHERE s.hash = $1 AND s.workspace_id = $2", hash.0, w_id, ) @@ -1771,8 +1792,14 @@ async fn get_deployment_status( let status = not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?; + let deployment_status = DeploymentStatus { + lock: status.lock, + lock_error_logs: status.lock_error_logs, + job_id: status.job_id, + }; + tx.commit().await?; - Ok(Json(status)) + Ok(Json(deployment_status)) } pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index e592c43bb3..b580138191 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -844,12 +844,10 @@ async fn trigger_script_with_retry_and_error_handler( JobPayload::ScriptHash { hash, path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, cache_ttl, + cache_ignore_s3_path, priority, apply_preprocessor, .. @@ -862,16 +860,14 @@ async fn trigger_script_with_retry_and_error_handler( error_handler_path, error_handler_args, skip_handler: None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, priority, tag_override: tag.clone(), apply_preprocessor, trigger_path: Some(trigger_path), - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, }, _ => { return Err(windmill_common::error::Error::internal_err(format!( diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 18d4d7c423..8c1a83ce13 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -1216,7 +1216,7 @@ async fn edit_ducklake_config( ) .await?; - // Check that non-superadmins are not abusing Instance catalogs + // Check that non-superadmins are not abusing Instance databases if !is_superadmin { let old_ducklakes = sqlx::query_scalar!( r#" @@ -1240,7 +1240,7 @@ async fn edit_ducklake_config( || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path { return Err(Error::BadRequest( - "Only superadmins can create or modify ducklakes with Instance catalogs" + "Only superadmins can create or modify ducklakes with Instance databases" .to_string(), )); } @@ -1288,7 +1288,7 @@ async fn edit_datatable_config( ) .await?; - // Check that non-superadmins are not abusing Instance catalogs + // Check that non-superadmins are not abusing Instance databases if !is_superadmin { let old_datatables = sqlx::query_scalar!( r#" @@ -1312,7 +1312,7 @@ async fn edit_datatable_config( || old_dt.unwrap().database.resource_path != dt.database.resource_path { return Err(Error::BadRequest( - "Only superadmins can create or modify data tables with Instance catalogs" + "Only superadmins can create or modify data tables with Instance databases" .to_string(), )); } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 9134da3c41..678ec06c68 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -24,6 +24,7 @@ use crate::{ cache, db::DB, error::{Error, Result as WindmillResult}, + jobs::{ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings}, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, worker::{to_raw_value, Connection}, @@ -171,23 +172,17 @@ pub struct FlowValue { #[serde(default)] #[serde(skip_serializing_if = "is_default")] pub same_worker: bool, - - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub concurrency_time_window_s: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub debounce_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub debounce_delay_s: Option, - + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettings, + #[serde(flatten)] + pub debouncing_settings: DebouncingSettings, #[serde(skip_serializing_if = "Option::is_none")] pub skip_expr: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub early_return: Option, #[serde(skip_serializing_if = "Option::is_none")] // Priority at the flow level @@ -449,6 +444,8 @@ pub struct FlowModule { pub sleep: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, #[serde( default, deserialize_with = "raw_value_to_input_transform::<_, i32>", @@ -919,12 +916,8 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "is_none_or_empty")] tag: Option, language: ScriptLang, - #[serde(skip_serializing_if = "Option::is_none")] - custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrency_time_window_s: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "is_none_or_empty_vec")] @@ -945,12 +938,8 @@ pub enum FlowModuleValue { #[serde(skip_serializing_if = "is_none_or_empty")] tag: Option, language: ScriptLang, - #[serde(skip_serializing_if = "Option::is_none")] - custom_concurrency_key: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrent_limit: Option, - #[serde(skip_serializing_if = "Option::is_none")] - concurrency_time_window_s: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, #[serde(skip_serializing_if = "Option::is_none")] is_trigger: Option, #[serde(skip_serializing_if = "is_none_or_empty_vec")] @@ -989,9 +978,6 @@ struct UntaggedFlowModuleValue { lock: Option, tag: Option, language: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, is_trigger: Option, id: Option, default_node: Option, @@ -1000,6 +986,8 @@ struct UntaggedFlowModuleValue { tools: Option>, pass_flow_input_directly: Option, squash: Option, + #[serde(flatten)] + concurrency_settings: ConcurrencySettingsWithCustom, } impl<'de> Deserialize<'de> for FlowModuleValue { @@ -1074,9 +1062,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { language: untagged .language .ok_or_else(|| serde::de::Error::missing_field("language"))?, - custom_concurrency_key: untagged.custom_concurrency_key, - concurrent_limit: untagged.concurrent_limit, - concurrency_time_window_s: untagged.concurrency_time_window_s, + concurrency_settings: untagged.concurrency_settings, is_trigger: untagged.is_trigger, assets: untagged.assets, }), @@ -1089,9 +1075,7 @@ impl<'de> Deserialize<'de> for FlowModuleValue { language: untagged .language .ok_or_else(|| serde::de::Error::missing_field("language"))?, - custom_concurrency_key: untagged.custom_concurrency_key, - concurrent_limit: untagged.concurrent_limit, - concurrency_time_window_s: untagged.concurrency_time_window_s, + concurrency_settings: untagged.concurrency_settings, is_trigger: untagged.is_trigger, assets: untagged.assets, }), @@ -1164,6 +1148,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec) { sleep: None, suspend: None, cache_ttl: None, + cache_ignore_s3_path: None, timeout: None, priority: None, delete_after_use: None, @@ -1234,11 +1219,9 @@ pub async fn resolve_module( id, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, } = std::mem::replace(&mut val, Identity) else { unreachable!() @@ -1258,11 +1241,9 @@ pub async fn resolve_module( path: None, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }; } ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5388ba7427..e88fb9f25c 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -75,7 +75,7 @@ impl std::fmt::Display for JobTriggerKind { } } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Default)] #[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] pub enum JobKind { @@ -90,6 +90,7 @@ pub enum JobKind { Identity, FlowDependencies, AppDependencies, + #[default] Noop, DeploymentCallback, FlowScript, @@ -187,6 +188,8 @@ pub struct QueuedJob { #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, #[serde(skip_serializing_if = "Option::is_none")] pub preprocessed: Option, @@ -260,6 +263,7 @@ impl Default for QueuedJob { timeout: None, flow_step_id: None, cache_ttl: None, + cache_ignore_s3_path: None, priority: None, preprocessed: None, } @@ -346,23 +350,14 @@ pub enum JobPayload { ScriptHash { hash: ScriptHash, path: String, - /// Override default concurrency key - custom_concurrency_key: Option, - /// How many jobs can run at the same time - concurrent_limit: Option, - /// In seconds - concurrency_time_window_s: Option, - /// If not set, will be inferred from the hash(path + step_id + inputs) - custom_debounce_key: Option, - /// Debouncing delay will be determined by the first job with the key. - /// All subsequent jobs with Some will get debounced. - /// If the job has no delay, it will execute immediately, fully ignoring pending delays. - debounce_delay_s: Option, cache_ttl: Option, + cache_ignore_s3_path: Option, dedicated_worker: Option, language: ScriptLang, priority: Option, apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, }, /// Execute flow step (can be subflow only). @@ -374,13 +369,12 @@ pub enum JobPayload { /// Execute flow step FlowScript { id: FlowNodeId, // flow_node(id). - language: ScriptLang, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, - cache_ttl: Option, - dedicated_worker: Option, path: String, + language: ScriptLang, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + concurrency_settings: ConcurrencySettings, }, /// Inline App Script @@ -460,19 +454,18 @@ pub enum JobPayload { error_handler_path: Option, error_handler_args: Option>>, skip_handler: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, - custom_debounce_key: Option, - debounce_delay_s: Option, cache_ttl: Option, + cache_ignore_s3_path: Option, priority: Option, tag_override: Option, trigger_path: Option, apply_preprocessor: bool, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, }, DeploymentCallback { path: String, + // debouncing_settings: Option, }, Identity, Noop, @@ -481,6 +474,108 @@ pub enum JobPayload { }, } +// TODO: Add validation logic. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct DebouncingSettings { + #[serde( + skip_serializing_if = "Option::is_none", + rename = "debounce_key", + alias = "custom_debounce_key" + )] + /// debounce key is usually stored in the db + /// including when: + /// + /// 1. User have created custom debounce key from ui or cli + /// 2. User used default one + /// + /// in either cases this argument serves as reactive way of overwriting debounce key from the backend. + /// Default: hash(path + step_id + inputs) + pub custom_key: Option, + + #[serde(skip_serializing_if = "Option::is_none", rename = "debounce_delay_s")] + /// Debouncing delay will be determined by the first job with the key. + /// All subsequent jobs with Some will get debounced. + /// If the job has no delay, it will execute immediately, fully ignoring pending delays. + pub delay_s: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "max_total_debouncing_time" + )] + pub max_total_time: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "max_total_debounces_amount" + )] + pub max_total_amount: Option, + + #[serde( + skip_serializing_if = "Option::is_none", + rename = "debounce_args_to_accumulate" + )] + /// top level arguments to preserve + /// For every debounce selected arguments will be saved + /// in the end (when job finally starts) arguments will be appended and passed to runnable + /// + /// NOTE: selected args should be the lists. + pub args_to_accumulate: Option>, +} + +impl DebouncingSettings { + pub fn is_default(&self) -> bool { + self == &Self::default() + } +} + +#[derive(Debug, Default, Clone, Serialize, Deserialize, sqlx::FromRow)] +pub struct ConcurrencySettings { + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow, Default)] +pub struct ConcurrencySettingsWithCustom { + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrent_limit: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub concurrency_time_window_s: Option, +} + +impl From for ConcurrencySettingsWithCustom { + fn from( + ConcurrencySettings { concurrency_key, concurrent_limit, concurrency_time_window_s }: ConcurrencySettings, + ) -> Self { + ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + +impl From for ConcurrencySettings { + fn from( + ConcurrencySettingsWithCustom { + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }: ConcurrencySettingsWithCustom, + ) -> Self { + ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrency_time_window_s, + concurrent_limit, + } + } +} + #[derive(Clone, Serialize, Deserialize, Debug)] pub struct SkipHandler { pub path: String, @@ -489,20 +584,50 @@ pub struct SkipHandler { pub stop_message: String, } -#[derive(Clone, Serialize, Deserialize, Debug, Default)] +#[derive(Clone, Deserialize, Debug, Default)] pub struct RawCode { pub content: String, pub path: Option, pub hash: Option, pub language: ScriptLang, pub lock: Option, - pub custom_concurrency_key: Option, - pub concurrent_limit: Option, - pub concurrency_time_window_s: Option, - pub custom_debounce_key: Option, - pub debounce_delay_s: Option, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub dedicated_worker: Option, + #[serde(flatten)] + pub concurrency_settings: ConcurrencySettingsWithCustom, + #[serde(flatten)] + // NOTE: Since we can only deserialize the struct, + // even though the older versions pass `custom_debounce_key` to RawCode, + // we can still have `debounce_key` in DebouncingSettings + // we just add alias `custom_debounce_key` + // however, serializing this settings will produce `debounce_key` + pub debouncing_settings: DebouncingSettings, +} + +impl JobPayload { + pub fn job_kind(&self) -> JobKind { + match self { + JobPayload::Noop => JobKind::Noop, + JobPayload::Identity => JobKind::Identity, + JobPayload::Code { .. } => JobKind::Preview, + JobPayload::AIAgent { .. } => JobKind::AIAgent, + JobPayload::FlowNode { .. } => JobKind::FlowNode, + JobPayload::ScriptHash { .. } => JobKind::Script, + JobPayload::AppScript { .. } => JobKind::AppScript, + JobPayload::RawFlow { .. } => JobKind::FlowPreview, + JobPayload::ScriptHub { .. } => JobKind::Script_Hub, + JobPayload::FlowScript { .. } => JobKind::FlowScript, + JobPayload::Dependencies { .. } => JobKind::Dependencies, + JobPayload::SingleStepFlow { .. } => JobKind::SingleStepFlow, + JobPayload::AppDependencies { .. } => JobKind::AppDependencies, + JobPayload::FlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::RawScriptDependencies { .. } => JobKind::Dependencies, + JobPayload::RawFlowDependencies { .. } => JobKind::FlowDependencies, + JobPayload::DeploymentCallback { .. } => JobKind::DeploymentCallback, + JobPayload::Flow { .. } | JobPayload::RestartedFlow { .. } => JobKind::Flow, + } + } } type Tag = String; @@ -575,6 +700,7 @@ pub async fn script_path_to_payload<'e>( debounce_key, debounce_delay_s, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, @@ -599,17 +725,24 @@ pub async fn script_path_to_payload<'e>( JobPayload::ScriptHash { hash: ScriptHash(hash), path: script_path.to_owned(), - custom_concurrency_key: concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, apply_preprocessor: !skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), + concurrency_settings: ConcurrencySettingsWithCustom { + custom_concurrency_key: concurrency_key, + concurrent_limit, + concurrency_time_window_s, + } + .into(), + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, }, tag, delete_after_use, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 1f3dbd414b..73390ef6f2 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -659,6 +659,7 @@ pub struct ScriptHashInfo { pub debounce_key: Option, pub debounce_delay_s: Option, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub language: ScriptLang, pub dedicated_worker: Option, pub priority: Option, @@ -804,7 +805,7 @@ async fn get_script_info_for_hash_inner<'e, E: sqlx::PgExecutor<'e>>( ) -> error::Result> { let r = sqlx::query_as!( ScriptHashInfo, - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", hash, w_id ) @@ -1022,6 +1023,7 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( Option, Option, Option, + Option, ScriptLang, Option, Option, @@ -1030,7 +1032,7 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( String, )> { let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, debounce_key, debounce_delay_s, cache_ttl, cache_ignore_s3_path, language as \"language: ScriptLang\", dedicated_worker, priority, timeout, on_behalf_of_email, created_by FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false AND (lock IS NOT NULL OR $3 = false) ORDER BY created_at DESC LIMIT 1", script_path, @@ -1051,6 +1053,7 @@ pub async fn get_latest_hash_for_path<'c, E: sqlx::PgExecutor<'c>>( script.debounce_key, script.debounce_delay_s, script.cache_ttl, + script.cache_ignore_s3_path, script.language, script.dedicated_worker, script.priority, diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 191fe7343a..347d69c1a5 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -393,6 +393,8 @@ pub struct Script { #[serde(skip_serializing_if = "Option::is_none")] pub cache_ttl: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub cache_ignore_s3_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] pub delete_after_use: Option, @@ -495,6 +497,10 @@ pub struct NewScript { pub tag: Option, pub draft_only: Option, pub envs: Option>, + // NOTE: concurrency and debounce data is inline, + // bc it was this before refactor + // and rust seems to hash it differently + // for backwards compat we keep them inline pub concurrency_key: Option, pub concurrent_limit: Option, pub concurrency_time_window_s: Option, @@ -503,6 +509,7 @@ pub struct NewScript { #[serde(skip_serializing_if = "Option::is_none")] pub debounce_delay_s: Option, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub dedicated_worker: Option, pub ws_error_handler_muted: Option, pub priority: Option, @@ -853,6 +860,7 @@ pub async fn fetch_script_for_update<'a>( ws_error_handler_muted, priority, cache_ttl, + cache_ignore_s3_path, timeout, delete_after_use, restart_unless_cancelled, @@ -910,6 +918,7 @@ pub async fn clone_script<'c>( concurrent_limit: s.concurrent_limit, concurrency_time_window_s: s.concurrency_time_window_s, cache_ttl: s.cache_ttl, + cache_ignore_s3_path: s.cache_ignore_s3_path, dedicated_worker: s.dedicated_worker, ws_error_handler_muted: s.ws_error_handler_muted, priority: s.priority, @@ -941,14 +950,14 @@ pub async fn clone_script<'c>( INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \ created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ - draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ - draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 4801b93a78..1c98976bcf 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -411,7 +411,7 @@ fn format_pull_query(peek: String) -> String { WHERE id = (SELECT id FROM peek) RETURNING started_at, scheduled_for, - canceled_by, canceled_reason, worker + canceled_by, canceled_reason, worker, cache_ignore_s3_path ), r AS NOT MATERIALIZED ( UPDATE v2_job_runtime SET ping = now() @@ -437,7 +437,7 @@ fn format_pull_query(peek: String) -> String { flow_status, j.script_lang, j.same_worker, j.pre_run_error, j.visible_to_owner, j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, - j.timeout, j.flow_step_id, j.cache_ttl, j.priority, j.raw_code, j.raw_lock, j.raw_flow, + j.timeout, j.flow_step_id, j.cache_ttl, q.cache_ignore_s3_path, j.priority, j.raw_code, j.raw_lock, j.raw_flow, j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path, COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 25d10784bb..6dce1bb229 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -14,9 +14,9 @@ use async_recursion::async_recursion; use chrono::{DateTime, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; -use quick_cache::sync::Cache; #[cfg(feature = "prometheus")] use prometheus::IntCounter; +use quick_cache::sync::Cache; use regex::Regex; use reqwest::Client; use serde::Deserialize; @@ -38,7 +38,10 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; +use windmill_common::jobs::{ + ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, JobTriggerKind, + EMAIL_ERROR_HANDLER_USER_EMAIL, +}; use windmill_common::utils::{configure_client, now_from_db}; use windmill_common::worker::{Connection, MIN_VERSION_SUPPORTS_DEBOUNCING, SCRIPT_TOKEN_EXPIRY}; @@ -168,7 +171,6 @@ pub async fn cancel_single_job<'c>( db: &Pool, force_cancel: bool, ) -> error::Result<(Transaction<'c, Postgres>, Option)> { - let id = job_running.id; if force_cancel || (job_running.parent_job.is_none() && !job_running.running) { let username = username.to_string(); @@ -236,7 +238,6 @@ pub async fn cancel_job<'c>( //TODO fetch mini completed job instead of QueuedJob let job = get_queued_job_v2(&mut *tx, &id).await?; - if job.is_none() { return Ok((tx, None)); } @@ -249,7 +250,6 @@ pub async fn cancel_job<'c>( )); } - if job.workspace_id != w_id { return Err(Error::BadRequest( "You are not authorized to cancel this job belonging to another workspace".to_string(), @@ -324,16 +324,8 @@ ORDER BY depth, id tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len()); } - let (ntx, _) = cancel_single_job( - username, - reason.clone(), - job, - w_id, - tx, - db, - force_cancel, - ) - .await?; + let (ntx, _) = + cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel).await?; tx = ntx; if !force_cancel { @@ -357,16 +349,9 @@ ORDER BY depth, id let job = get_queued_job_v2(&mut *tx, &job_id).await?; if let Some(job) = job { - let (ntx, _) = cancel_single_job( - username, - reason.clone(), - job, - w_id, - tx, - db, - force_cancel, - ) - .await?; + let (ntx, _) = + cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel) + .await?; tx = ntx; } } @@ -445,13 +430,11 @@ pub async fn push_init_job<'c>( path: Some(format!("{INIT_SCRIPT_PATH_PREFIX}{worker_name}")), language: ScriptLang::Bash, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&ehm), worker_name, @@ -504,13 +487,11 @@ pub async fn push_periodic_bash_job<'c>( )), language: ScriptLang::Bash, lock: None, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, - custom_debounce_key: None, - debounce_delay_s: None, cache_ttl: None, + cache_ignore_s3_path: None, dedicated_worker: None, + concurrency_settings: ConcurrencySettingsWithCustom::default(), + debouncing_settings: DebouncingSettings::default(), }), PushArgs::from(&ehm), worker_name, @@ -867,7 +848,6 @@ pub async fn add_completed_job( restart_job_if_perpetual(db, completed_job, &canceled_by).await?; - // tracing::error!("4 {:?}", start.elapsed()); Ok((completed_job.id, duration)) @@ -1343,10 +1323,8 @@ async fn restart_job_if_perpetual_inner( JobPayload::ScriptHash { hash, path: queued_job.runnable_path.clone().unwrap_or_default(), - custom_concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: queued_job.cache_ttl, + cache_ignore_s3_path: queued_job.cache_ignore_s3_path, dedicated_worker: None, language: queued_job .script_lang @@ -1354,9 +1332,13 @@ async fn restart_job_if_perpetual_inner( .unwrap_or_else(|| ScriptLang::Deno), priority: queued_job.priority, apply_preprocessor: false, + concurrency_settings: ConcurrencySettings { + concurrency_key: custom_concurrency_key(db, &queued_job.id).await?, + concurrent_limit: None, + concurrency_time_window_s: None, + }, // TODO(debouncing): handle properly - custom_debounce_key: None, - debounce_delay_s: None, + debouncing_settings: DebouncingSettings::default(), }, PushArgs::from(&args.0), &queued_job.created_by, @@ -1442,7 +1424,6 @@ fn apply_completed_job_cloud_usage( tracing::error!("Failed to get team plan status to update usage for workspace {w_id}: {err:#}"); } }; - }).await; if let Err(_) = result { @@ -1571,8 +1552,9 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> (cached.0.clone(), cached.1.clone(), cached.2) } else { // Cache expired, fetch from database - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" + let row_result = + sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" SELECT error_handler, error_handler_extra_args, @@ -1582,25 +1564,33 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> WHERE workspace_id = $1 "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| { + Error::internal_err(format!("no workspace settings for id {w_id}")) + })?; // Update cache with 60s TTL let expiry = now + 60; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ( + row_result.0.clone(), + row_result.1.clone(), + row_result.2, + expiry, + ), ); row_result } } else { // Cache miss, fetch from database - let row_result = sqlx::query_as::<_, (Option, Option>>, bool)>( - r#" + let row_result = + sqlx::query_as::<_, (Option, Option>>, bool)>( + r#" SELECT error_handler, error_handler_extra_args, @@ -1610,18 +1600,25 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> WHERE workspace_id = $1 "#, - ) - .bind(&w_id) - .fetch_optional(db) - .await - .context("fetching error handler info from workspace_settings")? - .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))?; + ) + .bind(&w_id) + .fetch_optional(db) + .await + .context("fetching error handler info from workspace_settings")? + .ok_or_else(|| { + Error::internal_err(format!("no workspace settings for id {w_id}")) + })?; // Store in cache with 60s TTL let expiry = now + 60; WORKSPACE_ERROR_HANDLER_CACHE.insert( w_id.clone(), - (row_result.0.clone(), row_result.1.clone(), row_result.2, expiry) + ( + row_result.0.clone(), + row_result.1.clone(), + row_result.2, + expiry, + ), ); row_result }; @@ -1982,6 +1979,7 @@ pub struct MiniPulledJob { pub timeout: Option, pub flow_step_id: Option, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub priority: Option, pub preprocessed: Option, pub script_entrypoint_override: Option, @@ -1991,7 +1989,6 @@ pub struct MiniPulledJob { pub permissioned_as_end_user_email: Option, } - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MiniCompletedJob { @@ -2016,6 +2013,7 @@ pub struct MiniCompletedJob { pub concurrent_limit: Option, pub tag: String, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, } impl From for MiniCompletedJob { @@ -2041,7 +2039,7 @@ impl From for MiniCompletedJob { concurrent_limit: job.concurrent_limit, tag: job.tag, cache_ttl: job.cache_ttl, - + cache_ignore_s3_path: job.cache_ignore_s3_path, } } } @@ -2070,6 +2068,7 @@ impl From for MiniCompletedJob { concurrent_limit: job.concurrent_limit, tag: job.tag, cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, } } } @@ -2097,6 +2096,7 @@ impl From> for MiniCompletedJob { concurrent_limit: job.concurrent_limit, tag: job.tag.clone(), cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, } } } @@ -2116,11 +2116,16 @@ impl MiniCompletedJob { pub fn is_dependency(&self) -> bool { self.kind.is_dependency() } - } -fn schedule_path(trigger_kind: &Option, trigger: &Option) -> Option { - if trigger_kind.as_ref().is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) { +fn schedule_path( + trigger_kind: &Option, + trigger: &Option, +) -> Option { + if trigger_kind + .as_ref() + .is_some_and(|t| matches!(t, JobTriggerKind::Schedule)) + { trigger.clone() } else { None @@ -2191,6 +2196,7 @@ impl MiniPulledJob { timeout: job.timeout.clone(), flow_step_id: job.flow_step_id.clone(), cache_ttl: job.cache_ttl.clone(), + cache_ignore_s3_path: job.cache_ignore_s3_path.clone(), priority: job.priority.clone(), preprocessed: job.preprocessed.clone(), script_entrypoint_override: job.script_entrypoint_override.clone(), @@ -2398,6 +2404,7 @@ pub async fn get_mini_pulled_job<'c>( timeout, flow_step_id, cache_ttl, + cache_ignore_s3_path, v2_job_queue.priority, preprocessed, script_entrypoint_override, @@ -2413,7 +2420,6 @@ pub async fn get_mini_pulled_job<'c>( Ok(job) } - pub struct QueuedJobV2 { pub id: Uuid, pub workspace_id: String, @@ -2436,6 +2442,7 @@ pub struct QueuedJobV2 { pub concurrent_limit: Option, pub tag: String, pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, pub last_ping: Option>, pub worker: Option, pub memory_peak: Option, @@ -2449,11 +2456,13 @@ impl QueuedJobV2 { } pub async fn get_queued_job_v2<'c>( - e: impl PgExecutor<'c>, job_id: &Uuid) -> error::Result> { + e: impl PgExecutor<'c>, + job_id: &Uuid, +) -> error::Result> { let job = sqlx::query_as!( QueuedJobV2, "SELECT id, q.workspace_id, j.runnable_id as \"runnable_id: ScriptHash\", scheduled_for, parent_job, flow_innermost_root_job, runnable_path, kind as \"kind: JobKind\", started_at, permissioned_as, created_by, script_lang as \"script_lang: ScriptLang\", - permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, r.ping as last_ping, worker, memory_peak, running + permissioned_as_email, flow_step_id, trigger_kind as \"trigger_kind: JobTriggerKind\", trigger, q.priority, concurrent_limit, q.tag, cache_ttl, cache_ignore_s3_path, r.ping as last_ping, worker, memory_peak, running FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status s USING (id) WHERE j.id = $1", job_id, @@ -2469,7 +2478,7 @@ pub struct PulledJobResult { pub job: Option, pub suspended: bool, pub missing_concurrency_key: bool, - pub error_while_preprocessing: Option + pub error_while_preprocessing: Option, } #[derive(thiserror::Error, Debug)] @@ -2480,7 +2489,6 @@ pub enum PulledJobResultToJobErr { ErrorWhilePreprocessing(JobCompleted), } - impl PulledJobResult { pub fn to_pulled_job(self) -> Result, PulledJobResultToJobErr> { match self { @@ -2761,7 +2769,6 @@ impl PulledJobResult { } } else { tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); - } // This will unblock pusher. tx.commit().await?; @@ -2853,21 +2860,21 @@ pub async fn pull( // Concurrency limit is available for either enterprise job or dependency job && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING)) => { - crate::jobs_ee::apply_concurrency_limit( - db, - pull_loop_count, - suspended, - job, - ) - .await? - .unwrap_or(PulledJobResult { - job: None, - suspended, - missing_concurrency_key: false, - error_while_preprocessing: None, - }) + crate::jobs_ee::apply_concurrency_limit(db, pull_loop_count, suspended, job) + .await? + .unwrap_or(PulledJobResult { + job: None, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }) } - _ => PulledJobResult { job, suspended, missing_concurrency_key: false, error_while_preprocessing: None }, + _ => PulledJobResult { + job, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }, }; Ok::<_, Error>(pulled_job_result) @@ -2885,7 +2892,12 @@ pub async fn pull( ) .await?; let Some(job) = job else { - return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false, error_while_preprocessing: None }); + return Ok(PulledJobResult { + job: None, + suspended, + missing_concurrency_key: false, + error_while_preprocessing: None, + }); }; let has_concurent_limit = job.concurrent_limit.is_some(); @@ -2896,7 +2908,10 @@ pub async fn pull( } #[cfg(not(feature = "enterprise"))] - let has_concurent_limit = job.is_dependency() && job.concurrent_limit.is_some() && cfg!(feature = "private") && !*WMDEBUG_NO_DJOB_DEBOUNCING; + let has_concurent_limit = job.is_dependency() + && job.concurrent_limit.is_some() + && cfg!(feature = "private") + && !*WMDEBUG_NO_DJOB_DEBOUNCING; // if we don't have private flag, we don't have concurrency limit // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue @@ -3040,7 +3055,7 @@ pub async fn concurrency_key( .await .map(|x| { if x.is_none() { - tracing::info!("No concurrency key found for job {id}, defaulting to empty string"); + tracing::info!("No concurrency key found for job {id}"); } return x; }) @@ -3593,18 +3608,18 @@ pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result + Send + 'a, ->(id: &'a Uuid, w_id: &'a str, db: A) -> impl Future>> + Send + 'a { - async move { - let mut conn = db.acquire().await?; +pub fn get_mini_completed_job<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a>( + id: &'a Uuid, + w_id: &'a str, + db: A, +) -> impl Future>> + Send + 'a { + async move { + let mut conn = db.acquire().await?; sqlx::query_as!( MiniCompletedJob, "SELECT j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as, - j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl + j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: JobTriggerKind\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id WHERE j.id = $1 AND j.workspace_id = $2", id, @@ -3616,7 +3631,6 @@ A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, } } - pub enum PushIsolationLevel<'c> { IsolatedRoot(DB), Isolated(UserDB, Authed), @@ -3828,8 +3842,7 @@ pub async fn push<'c, 'd>( mut email: &str, mut permissioned_as: String, token_prefix: Option<&str>, - #[allow(unused_mut)] - mut scheduled_for_o: Option>, + #[allow(unused_mut)] mut scheduled_for_o: Option>, schedule_path: Option, parent_job: Option, root_job: Option, @@ -4031,87 +4044,93 @@ pub async fn push<'c, 'd>( } } + #[derive(Default)] + struct JobPayloadUntagged { + runnable_id: Option, + runnable_path: Option, + raw_code_tuple: Option<(String, Option)>, // (content, lock) + job_kind: JobKind, + raw_flow: Option, + flow_status: Option, + language: Option, + cache_ttl: Option, + cache_ignore_s3_path: Option, + dedicated_worker: Option, + _low_level_priority: Option, + concurrency_settings: ConcurrencySettings, + debouncing_settings: DebouncingSettings, + } let mut preprocessed = None; - #[allow(unused)] - let ( - script_hash, - script_path, + #[allow(unused)] + let JobPayloadUntagged { + runnable_id, + runnable_path, raw_code_tuple, job_kind, raw_flow, flow_status, language, - mut custom_concurrency_key, - mut concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, dedicated_worker, _low_level_priority, - custom_debounce_key, - debounce_delay_s, - ) = match job_payload { + concurrency_settings: + ConcurrencySettings { + mut concurrency_key, + mut concurrent_limit, + concurrency_time_window_s, // + }, + debouncing_settings, + } = match job_payload { JobPayload::ScriptHash { hash, path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, apply_preprocessor, - custom_debounce_key, - debounce_delay_s, + concurrency_settings, + debouncing_settings, } => { if apply_preprocessor { preprocessed = Some(false); } - ( - Some(hash.0), - Some(path), - None, - JobKind::Script, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: Some(hash.0), + runnable_path: Some(path), + job_kind: JobKind::Script, + language: Some(language), + concurrency_settings, + debouncing_settings, cache_ttl, + cache_ignore_s3_path, dedicated_worker, - priority, - custom_debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + ..Default::default() + } } JobPayload::FlowScript { id, // flow_node(id). language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, dedicated_worker, path, - } => ( - Some(id.0), - Some(path), - None, - JobKind::FlowScript, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, + } => JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: Some(path), + job_kind: JobKind::FlowScript, + language: Some(language), + concurrency_settings, cache_ttl, + cache_ignore_s3_path, dedicated_worker, - None, - None, // custom_debounce_key removed for flow steps - None, // debounce_delay_s removed for flow steps - ), + ..Default::default() + }, JobPayload::FlowNode { id, path } => { let data = cache::flow::fetch_flow(_db, id).await?; let value = data.value(); @@ -4124,46 +4143,28 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - Some(id.0), - Some(path), - None, - JobKind::FlowNode, - value_o, - status, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ) + JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: Some(path), + job_kind: JobKind::FlowNode, + raw_flow: value_o, + flow_status: status, + ..Default::default() + } } JobPayload::AppScript { id, // app_script(id). path, language, cache_ttl, - } => ( - Some(id.0), - path, - None, - JobKind::AppScript, - None, - None, - Some(language), - None, - None, - None, + } => JobPayloadUntagged { + runnable_id: Some(id.0), + runnable_path: path, + job_kind: JobKind::AppScript, + language: Some(language), cache_ttl, - None, - None, - None, - None, - ), + ..Default::default() + }, JobPayload::ScriptHub { path, apply_preprocessor } => { if path == "hub/7771/slack" || path == "hub/7836/slack" || path == "hub/9084/slack" { // these scripts send app reports to slack @@ -4180,24 +4181,12 @@ pub async fn push<'c, 'd>( get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(_db)) .await?; - ( - None, - Some(path), - None, - // Some((script.content, script.lockfile)), - JobKind::Script_Hub, - None, - None, - Some(hub_script.language), - None, - None, - None, - None, - None, - None, - None, - None, - ) + JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::Script_Hub, + language: Some(hub_script.language), + ..Default::default() + } } JobPayload::Code(RawCode { content, @@ -4205,85 +4194,51 @@ pub async fn push<'c, 'd>( hash, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, dedicated_worker, - custom_debounce_key, - debounce_delay_s, - }) => ( - hash, - path, - Some((content, lock)), - JobKind::Preview, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, + debouncing_settings, + }) => JobPayloadUntagged { + runnable_id: hash, + runnable_path: path, + raw_code_tuple: Some((content, lock)), + job_kind: JobKind::Preview, + language: Some(language), + concurrency_settings: concurrency_settings.into(), + debouncing_settings, cache_ttl, + cache_ignore_s3_path, dedicated_worker, - None, - custom_debounce_key, - debounce_delay_s, - ), - JobPayload::Dependencies { hash, language, path, dedicated_worker } => ( - Some(hash.0), - Some(path.clone()), - None, - JobKind::Dependencies, - None, - None, - Some(language), - None, - None, - None, - None, + ..Default::default() + }, + JobPayload::Dependencies { hash, language, path, dedicated_worker } => JobPayloadUntagged { + runnable_id: Some(hash.0), + runnable_path: Some(path.clone()), + job_kind: JobKind::Dependencies, + language: Some(language), dedicated_worker, - None, - None, - None, - ), + ..Default::default() + }, // CLI usage, is not modifying db, no need for debouncing. - JobPayload::RawScriptDependencies { script_path, content, language } => ( - None, - Some(script_path), - Some((content, None)), - JobKind::Dependencies, - None, - None, - Some(language), - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::RawScriptDependencies { script_path, content, language } => { + JobPayloadUntagged { + runnable_path: Some(script_path), + raw_code_tuple: Some((content, None)), + job_kind: JobKind::Dependencies, + language: Some(language), + ..Default::default() + } + } // CLI usage, is not modifying db, no need for debouncing. - JobPayload::RawFlowDependencies { path, flow_value } => ( - None, - Some(path), - None, - JobKind::FlowDependencies, - Some(flow_value), - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::RawFlowDependencies { path, flow_value } => JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::FlowDependencies, + raw_flow: Some(flow_value), + ..Default::default() + }, JobPayload::FlowDependencies { path, dedicated_worker, version } => { #[cfg(test)] let skip_compat = args @@ -4305,41 +4260,21 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - Some(version), - Some(path.clone()), - None, - JobKind::FlowDependencies, - value_o, - None, - None, - None, - None, - None, - None, + JobPayloadUntagged { + runnable_id: Some(version), + runnable_path: Some(path.clone()), + job_kind: JobKind::FlowDependencies, + raw_flow: value_o, dedicated_worker, - None, - None, - None, - ) + ..Default::default() + } } - JobPayload::AppDependencies { path, version } => ( - Some(version), - Some(path.clone()), - None, - JobKind::AppDependencies, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::AppDependencies { path, version } => JobPayloadUntagged { + runnable_id: Some(version), + runnable_path: Some(path.clone()), + job_kind: JobKind::AppDependencies, + ..Default::default() + }, JobPayload::RawFlow { mut value, path, restarted_from } => { add_virtual_items_if_necessary(&mut value.modules); @@ -4386,30 +4321,20 @@ pub async fn push<'c, 'd>( FlowStatus::new(&value) } // this is a new flow being pushed, flow_status is set to flow_value }; - let concurrency_key = value.concurrency_key.clone(); - let concurrent_limit = value.concurrent_limit; - let concurrency_time_window_s = value.concurrency_time_window_s; - let debounce_key = value.debounce_key.clone(); - let debounce_delay_s = value.debounce_delay_s; let cache_ttl = value.cache_ttl.map(|x| x as i32); let priority = value.priority; - ( - None, - path, - None, - JobKind::FlowPreview, - Some(value), - Some(flow_status), - None, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_path: path, + job_kind: JobKind::FlowPreview, + flow_status: Some(flow_status), cache_ttl, - None, - priority, - debounce_key, - debounce_delay_s, - ) + cache_ignore_s3_path: value.cache_ignore_s3_path, + _low_level_priority: priority, + concurrency_settings: value.concurrency_settings.clone(), + debouncing_settings: value.debouncing_settings.clone(), + raw_flow: Some(value), + ..Default::default() + } } JobPayload::SingleStepFlow { path, @@ -4420,16 +4345,14 @@ pub async fn push<'c, 'd>( error_handler_args, skip_handler, args, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, priority, tag_override, trigger_path, apply_preprocessor, - custom_debounce_key, - debounce_delay_s, + debouncing_settings, + concurrency_settings, } => { // Determine if this is a flow or a script let is_flow = flow_version.is_some(); @@ -4563,13 +4486,11 @@ pub async fn push<'c, 'd>( let flow_value = FlowValue { modules, failure_module, - concurrency_time_window_s, - concurrent_limit, - debounce_key: custom_debounce_key.clone(), - debounce_delay_s, + concurrency_settings: concurrency_settings.clone(), + debouncing_settings: debouncing_settings.clone(), priority, cache_ttl: cache_ttl.map(|val| val as u32), - concurrency_key: custom_concurrency_key.clone(), + cache_ignore_s3_path: cache_ignore_s3_path, same_worker: false, early_return: None, skip_expr: None, @@ -4579,23 +4500,18 @@ pub async fn push<'c, 'd>( }; // this is a new flow being pushed, flow_status is set to flow_value: let flow_status: FlowStatus = FlowStatus::new(&flow_value); - ( - None, // No version needed - flow is stored in raw_flow like FlowPreview - Some(path), - None, - JobKind::SingleStepFlow, - Some(flow_value), - Some(flow_status), - None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::SingleStepFlow, + raw_flow: Some(flow_value), + flow_status: Some(flow_status), cache_ttl, - None, - priority, - custom_debounce_key, - debounce_delay_s, - ) + cache_ignore_s3_path, + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => { let mut ntx = tx.into_tx().await?; @@ -4618,20 +4534,19 @@ pub async fn push<'c, 'd>( let mut value = data.value().clone(); let priority = value.priority; let cache_ttl = value.cache_ttl.map(|x| x as i32); - let custom_concurrency_key = value.concurrency_key.clone(); - let concurrency_time_window_s = value.concurrency_time_window_s; - let mut concurrent_limit = value.concurrent_limit; - - let custom_debounce_key = value.debounce_key.clone(); - let mut debounce_delay_s = value.debounce_delay_s; + let cache_ignore_s3_path = value.cache_ignore_s3_path; + let mut concurrency_settings = value.concurrency_settings.clone(); + let mut debouncing_settings = value.debouncing_settings.clone(); if !apply_preprocessor { value.preprocessor_module = None; } else { tag = None; - concurrent_limit = None; + + concurrency_settings.concurrent_limit = None; // TODO: May be re-enable? - debounce_delay_s = None; + debouncing_settings.delay_s = None; + preprocessed = Some(false); } @@ -4652,23 +4567,20 @@ pub async fn push<'c, 'd>( // by additional checks when handling the flow. None }; - ( - Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull. - Some(path), - None, - JobKind::Flow, - value_o, - Some(status), - None, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull. + runnable_path: Some(path), + job_kind: JobKind::Flow, + raw_flow: value_o, + flow_status: Some(status), cache_ttl, + cache_ignore_s3_path, dedicated_worker, - priority, - custom_debounce_key, - debounce_delay_s, - ) + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } JobPayload::RestartedFlow { completed_job_id, step_id, branch_or_iteration_n } => { let ( @@ -4687,6 +4599,7 @@ pub async fn push<'c, 'd>( branch_or_iteration_n, ) .await?; + let restarted_flow_status = FlowStatus { step: step_n, modules: truncated_modules, @@ -4715,11 +4628,8 @@ pub async fn push<'c, 'd>( }; let value = flow_data.value(); let priority = value.priority; - let concurrency_key = value.concurrency_key.clone(); - let concurrent_limit = value.concurrent_limit; - let concurrency_time_window_s = value.concurrency_time_window_s; - let debounce_key = value.debounce_key.clone(); - let debounce_delay_s = value.debounce_delay_s; + let concurrency_settings = value.concurrency_settings.clone(); + let debouncing_settings = value.debouncing_settings.clone(); let cache_ttl = value.cache_ttl.map(|x| x as i32); // Keep inserting `value` if not all workers are updated. // Starting at `v1.440`, the value is fetched on pull from the version id. @@ -4729,92 +4639,39 @@ pub async fn push<'c, 'd>( // `raw_flow` is fetched on pull. None }; - ( - version, - flow_path, - None, - JobKind::Flow, - value_o, - Some(restarted_flow_status), - None, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, + JobPayloadUntagged { + runnable_id: version, + runnable_path: flow_path, + job_kind: JobKind::Flow, + raw_flow: value_o, + flow_status: Some(restarted_flow_status), cache_ttl, - None, - priority, - debounce_key, - debounce_delay_s, - ) + cache_ignore_s3_path: value.cache_ignore_s3_path, + _low_level_priority: priority, + concurrency_settings, + debouncing_settings, + ..Default::default() + } } - JobPayload::DeploymentCallback { path } => ( - None, - Some(path.clone()), - None, - JobKind::DeploymentCallback, - None, - None, - None, - Some(format!("{workspace_id}:git_sync")), - Some(1), - Some(0), - None, - None, - None, - None, - None, - ), - JobPayload::Identity => ( - None, - None, - None, - JobKind::Identity, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), - JobPayload::Noop => ( - None, - None, - None, - JobKind::Noop, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), - JobPayload::AIAgent { path } => ( - None, - Some(path), - None, - JobKind::AIAgent, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - ), + JobPayload::DeploymentCallback { path } => JobPayloadUntagged { + runnable_path: Some(path.clone()), + job_kind: JobKind::DeploymentCallback, + concurrency_settings: ConcurrencySettings { + concurrency_key: Some(format!("{workspace_id}:git_sync")), + concurrent_limit: Some(1), + concurrency_time_window_s: Some(0), + }, + ..Default::default() + }, + JobPayload::Identity => { + JobPayloadUntagged { job_kind: JobKind::Identity, ..Default::default() } + } + JobPayload::Noop => JobPayloadUntagged { job_kind: JobKind::Noop, ..Default::default() }, + JobPayload::AIAgent { path } => JobPayloadUntagged { + runnable_path: Some(path), + job_kind: JobKind::AIAgent, + ..Default::default() + }, }; // Enforce concurrency limit on all dependency jobs. @@ -4823,13 +4680,13 @@ pub async fn push<'c, 'd>( // // This is not the case for scripts, so we can potentially have multiple djobs for scripts at the same time. if let (Some(path), true) = ( - &script_path, + &runnable_path, cfg!(feature = "private") && job_kind.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING && *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, ) { - custom_concurrency_key = Some(format!("dependency:{workspace_id}/{path}")); + concurrency_key = Some(format!("dependency:{workspace_id}/{path}")); concurrent_limit = Some(1); } @@ -4895,7 +4752,7 @@ pub async fn push<'c, 'd>( } else { "" }, - script_path.clone().expect("dedicated script has a path") + runnable_path.clone().expect("dedicated script has a path") ) } else { if tag == Some("".to_string()) { @@ -4973,7 +4830,7 @@ pub async fn push<'c, 'd>( match ( scheduled_for_o.is_some(), job_kind.is_dependency(), - script_path.clone(), + runnable_path.clone(), *WMDEBUG_NO_DJOB_DEBOUNCING, *MIN_VERSION_SUPPORTS_DEBOUNCING.read().await, // We only do debouncing for jobs triggered by relative imports @@ -5155,10 +5012,10 @@ pub async fn push<'c, 'd>( if schedule_path.is_none() { if let Some(debounced_job_id) = crate::jobs_ee::maybe_apply_debouncing( &job_id, - debounce_delay_s, - custom_debounce_key, + debouncing_settings.delay_s, + debouncing_settings.custom_key, workspace_id, - script_path.clone(), + runnable_path.clone(), &job_kind, &args, &mut scheduled_for_o, @@ -5174,9 +5031,9 @@ pub async fn push<'c, 'd>( insert_concurrency_key( workspace_id, &args, - &script_path, + &runnable_path, job_kind, - custom_concurrency_key, + concurrency_key, &mut tx, job_id, ) @@ -5283,8 +5140,8 @@ pub async fn push<'c, 'd>( ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email ) INSERT INTO v2_job_queue - (workspace_id, id, running, scheduled_for, started_at, tag, priority) - VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31)", + (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path) + VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42)", job_id, workspace_id, raw_code, @@ -5294,8 +5151,8 @@ pub async fn push<'c, 'd>( parent_job, user, permissioned_as, - script_hash, - script_path.clone(), + runnable_id, + runnable_path.clone(), Json(args) as Json, job_kind.clone() as JobKind, schedule_path, @@ -5330,6 +5187,7 @@ pub async fn push<'c, 'd>( trigger_kind as Option, running, end_user_email, + cache_ignore_s3_path, ) .execute(&mut *tx) .warn_after_seconds(1) @@ -5384,7 +5242,7 @@ pub async fn push<'c, 'd>( let operation_name = match job_kind { JobKind::Preview => "jobs.run.preview", JobKind::Script => { - s = ScriptHash(script_hash.unwrap()).to_string(); + s = ScriptHash(runnable_id.unwrap()).to_string(); hm.insert("hash", s.as_str()); "jobs.run.script" } @@ -5430,7 +5288,7 @@ pub async fn push<'c, 'd>( operation_name, ActionKind::Execute, workspace_id, - script_path.as_ref().map(|x| x.as_str()), + runnable_path.as_ref().map(|x| x.as_str()), Some(hm), ) .warn_after_seconds(1) @@ -5721,8 +5579,6 @@ async fn restarted_flows_resolution( )) } - - // Wrapper struct to send both job and optional flow_runners to dedicated workers pub struct DedicatedWorkerJob { pub job: Arc, @@ -5740,7 +5596,11 @@ pub struct FlowRunners { impl Drop for FlowRunners { fn drop(&mut self) { let total_runners = self.handles.len(); - tracing::info!("dropping {} flow runners for job {}", total_runners, self.job_id); + tracing::info!( + "dropping {} flow runners for job {}", + total_runners, + self.job_id + ); // First, drop all senders to signal workers to stop gracefully self.runners.clear(); @@ -5756,12 +5616,17 @@ impl Drop for FlowRunners { // Wait up to 5 seconds for natural termination let timeout_result = tokio::time::timeout( tokio::time::Duration::from_secs(5), - futures::future::join_all(handles) - ).await; + futures::future::join_all(handles), + ) + .await; match timeout_result { Ok(_) => { - tracing::info!("all {} flow runners for job {} terminated gracefully", total_runners, job_id); + tracing::info!( + "all {} flow runners for job {} terminated gracefully", + total_runners, + job_id + ); } Err(_) => { // Timeout reached, abort only the handles that haven't finished @@ -5829,6 +5694,7 @@ pub async fn get_same_worker_job( v2_job.timeout, v2_job.flow_step_id, v2_job.cache_ttl, + v2_job_queue.cache_ignore_s3_path, v2_job_queue.priority, v2_job.preprocessed, v2_job.script_entrypoint_override, @@ -5859,4 +5725,3 @@ pub async fn get_same_worker_job( )) }) } - diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 52a397ce33..16358b453e 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -20,6 +20,8 @@ use windmill_common::flows::Retry; use windmill_common::get_flow_version_info_from_version; use windmill_common::get_latest_flow_version_id_for_path; use windmill_common::jobs::check_tag_available_for_workspace_internal; +use windmill_common::jobs::ConcurrencySettings; +use windmill_common::jobs::DebouncingSettings; use windmill_common::jobs::JobPayload; use windmill_common::schedule::schedule_to_user; use windmill_common::scripts::ScriptHash; @@ -90,6 +92,7 @@ async fn get_schedule_metadata<'c>( _debounce_key, _debounce_delay_s, _cache_ttl, + _cache_ignore_s3_path, _language, _dedicated_worker, _priority, @@ -258,16 +261,14 @@ pub async fn push_scheduled_job<'c>( stop_condition, stop_message, }), - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl: None, + cache_ignore_s3_path: None, priority: None, tag_override: schedule.tag.clone(), trigger_path: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -322,6 +323,7 @@ pub async fn push_scheduled_job<'c>( custom_debounce_key, debounce_delay_s, cache_ttl, + cache_ignore_s3_path, language, dedicated_worker, priority, @@ -360,16 +362,14 @@ pub async fn push_scheduled_job<'c>( error_handler_args: None, skip_handler: None, args: static_args, - custom_concurrency_key: None, - concurrent_limit: None, - concurrency_time_window_s: None, cache_ttl, + cache_ignore_s3_path, priority, tag_override: schedule.tag.clone(), trigger_path: None, apply_preprocessor: false, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings: ConcurrencySettings::default(), + debouncing_settings: DebouncingSettings::default(), }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() @@ -385,16 +385,22 @@ pub async fn push_scheduled_job<'c>( JobPayload::ScriptHash { hash, path: schedule.script_path.clone(), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl, + cache_ignore_s3_path, dedicated_worker, language, priority, apply_preprocessor: false, - custom_debounce_key, - debounce_delay_s, + debouncing_settings: DebouncingSettings { + custom_key: custom_debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, + concurrency_settings: ConcurrencySettings { + concurrency_key: custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 84304f232c..638129e34d 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -366,9 +366,7 @@ async fn execute_windmill_tool( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = path @@ -379,33 +377,22 @@ async fn execute_windmill_tool( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, tool_module, tag, tool_module.delete_after_use.unwrap_or(false), ) } - FlowModuleValue::FlowScript { - id, - language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - tag, - .. - } => { + FlowModuleValue::FlowScript { id, language, concurrency_settings, tag, .. } => { let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id); let payload = JobPayloadWithTag { payload: JobPayload::FlowScript { id, language, - custom_concurrency_key: custom_concurrency_key.clone(), - concurrent_limit, - concurrency_time_window_s, + concurrency_settings: concurrency_settings.into(), cache_ttl: tool_module.cache_ttl.map(|x| x as i32), + cache_ignore_s3_path: tool_module.cache_ignore_s3_path.clone(), dedicated_worker: None, path, }, diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index eb22066436..c68bf16a0b 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -118,6 +118,7 @@ pub struct AIAgentArgs { pub user_images: Option>, pub streaming: Option, pub messages_context_length: Option, + pub max_iterations: Option, } #[derive(Deserialize, Debug)] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index d18e32e845..ca62324bdb 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -75,7 +75,8 @@ lazy_static::lazy_static! { }; } -const MAX_AGENT_ITERATIONS: usize = 10; +const DEFAULT_MAX_AGENT_ITERATIONS: usize = 10; +const HARD_MAX_AGENT_ITERATIONS: usize = 1000; pub async fn handle_ai_agent_job( // connection @@ -547,8 +548,13 @@ pub async fn run_agent( None }; + let max_iterations = args + .max_iterations + .map(|m| m.clamp(1, HARD_MAX_AGENT_ITERATIONS)) + .unwrap_or(DEFAULT_MAX_AGENT_ITERATIONS); + // Main agent loop - for i in 0..MAX_AGENT_ITERATIONS { + for i in 0..max_iterations { if used_structured_output_tool { break; } @@ -717,7 +723,7 @@ pub async fn run_agent( if tool_calls.is_empty() { break; - } else if i == MAX_AGENT_ITERATIONS - 1 { + } else if i == max_iterations - 1 { return Err(Error::internal_err( "AI agent reached max iterations, but there are still tool calls" .to_string(), diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 69a348fc33..ea1f6a763d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -746,24 +746,48 @@ pub async fn resolve_job_timeout( } async fn hash_args( - _db: &DB, - _client: &AuthedClient, - _workspace_id: &str, + #[allow(unused)] db: &DB, + #[allow(unused)] client: &AuthedClient, + #[allow(unused)] workspace_id: &str, v: &Option>>>, hasher: &mut sha2::Sha256, + #[allow(unused)] job_id: &Uuid, + #[allow(unused)] ignore_s3_path: bool, ) { if let Some(Json(hm)) = v { for k in hm.keys().sorted() { hasher.update(k.as_bytes()); let arg_value = hm.get(k).unwrap(); + #[cfg(feature = "parquet")] - let (_, arg_additions) = - arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await; - hasher.update(arg_value.get().as_bytes()); + let etag = match serde_json::from_str::(arg_value.get()).ok() { + Some(s3_object) => { + let s3_resource = get_workspace_s3_resource_path( + db, + client, + workspace_id, + s3_object.storage.as_ref(), + job_id, + ) + .await + .ok() + .flatten(); + match s3_resource { + Some(s3_resource) => get_etag_or_empty(&s3_resource, s3_object).await, + None => None, + } + } + None => None, + }; + #[cfg(feature = "parquet")] - for (_, arg_addition) in arg_additions { - hasher.update(arg_addition.as_bytes()); + if let Some(etag) = etag { + hasher.update(etag.as_bytes()); + if ignore_s3_path { + continue; + } } + hasher.update(arg_value.get().as_bytes()); } } } @@ -788,7 +812,16 @@ pub async fn cached_result_path( _ => {} } } - hash_args(db, client, &job.workspace_id, &job.args, &mut hasher).await; + hash_args( + db, + client, + &job.workspace_id, + &job.args, + &mut hasher, + &job.id, + job.cache_ignore_s3_path.unwrap_or(false), + ) + .await; format!("g/results/{:064x}", hasher.finalize()) } @@ -798,6 +831,7 @@ async fn get_workspace_s3_resource_path( client: &AuthedClient, workspace_id: &str, storage: Option<&String>, + job_id: &Uuid, ) -> windmill_common::error::Result> { use windmill_common::{ job_s3_helpers_oss::get_s3_resource_internal, s3_helpers::StorageResourceType, @@ -860,7 +894,10 @@ async fn get_workspace_s3_resource_path( }; let s3_resource_value_raw = client - .get_resource_value::(path.as_str()) + .get_resource_value_interpolated::( + path.as_str(), + Some(job_id.to_string()), + ) .await?; get_s3_resource_internal( rt, @@ -872,41 +909,10 @@ async fn get_workspace_s3_resource_path( .map(Some) } -#[cfg(feature = "parquet")] -async fn arg_value_hash_additions( - db: &DB, - client: &AuthedClient, - workspace_id: &str, - raw_value: &Box, -) -> (Option, HashMap) { - let mut result: HashMap = HashMap::new(); - - let parsed_value = serde_json::from_str::(raw_value.get()); - - let mut storage = None; - if let Ok(s3_object) = parsed_value { - let s3_resource_opt = - get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.as_ref()) - .await; - storage = s3_object.storage.clone(); - - if let Some(mut s3_resource) = s3_resource_opt.ok().flatten() { - let etag = get_etag_or_empty(&mut s3_resource, s3_object.clone()).await; - tracing::warn!("Enriching s3 arg value with etag: {:?}", etag); - result.insert(s3_object.s3.clone(), etag.unwrap_or_default()); // TODO: maybe inject a random value to invalidate the cache? - } - } - - return (storage, result); -} - #[derive(Deserialize, Serialize)] struct CachedResource { expire: i64, - #[serde(skip_serializing_if = "Option::is_none")] - s3_etags: Option>, value: Arc>, - storage: Option, } impl CachedResource { @@ -923,7 +929,6 @@ lazy_static! { pub async fn get_cached_resource_value_if_valid( _db: &DB, client: &AuthedClient, - _job_id: &Uuid, _workspace_id: &str, cached_res_path: &str, ) -> Option>> { @@ -955,42 +960,6 @@ pub async fn get_cached_resource_value_if_valid( }, }; - #[cfg(feature = "parquet")] - { - let empty_etags = HashMap::new(); - let s3_etags = resource.s3_etags.as_ref().unwrap_or(&empty_etags); - let object_store_resource_opt: Option = if s3_etags.is_empty() { - None - } else { - get_workspace_s3_resource_path(_db, &client, _workspace_id, resource.storage.as_ref()) - .await - .ok() - .flatten() - }; - - if !s3_etags.is_empty() && object_store_resource_opt.is_none() { - tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated"); - return None; - } - for (s3_file_key, s3_file_etag) in s3_etags { - if let Some(object_store_resource) = object_store_resource_opt.as_ref() { - let etag = get_etag_or_empty( - object_store_resource, - S3Object { - s3: s3_file_key.clone(), - storage: resource.storage.clone(), - ..Default::default() - }, - ) - .await; - if etag.as_ref() != Some(s3_file_etag) { - tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag); - return None; - } - } - } - } - Some(resource.value.clone()) } @@ -1003,21 +972,7 @@ pub async fn save_in_cache( ) { let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64; - #[cfg(feature = "parquet")] - let (storage, s3_etags) = - arg_value_hash_additions(db, _client, job.workspace_id.as_str(), &r).await; - - #[cfg(feature = "parquet")] - let s3_etags = if s3_etags.is_empty() { - None - } else { - Some(s3_etags) - }; - - #[cfg(not(feature = "parquet"))] - let (storage, s3_etags) = (None, None); - - let store_cache_resource = CachedResource { expire, s3_etags, value: r, storage }; + let store_cache_resource = CachedResource { expire, value: r }; let raw_json = Json(&store_cache_resource); if let Err(e) = sqlx::query!( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index c1d962c306..1ffb0f11d9 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2729,7 +2729,6 @@ pub async fn handle_queued_job( let cached_result_maybe = get_cached_resource_value_if_valid( db, &client, - &job.id, &job.workspace_id, &cached_res_path, ) @@ -2828,7 +2827,7 @@ pub async fn handle_queued_job( } #[cfg(not(feature = "enterprise"))] - if job.concurrent_limit.is_some() { + if job.concurrent_limit.is_some() && !job.kind.is_dependency() { logs.push_str("---\n"); logs.push_str("WARNING: This job has concurrency limits enabled. Concurrency limits are an EE feature and the setting is ignored.\n"); logs.push_str("---\n"); @@ -4282,6 +4281,7 @@ pub fn init_worker_internal_server_inline_utils( timeout: None, flow_step_id: None, cache_ttl: None, + cache_ignore_s3_path: None, priority: None, preprocessed: None, script_entrypoint_override: None, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index dd45d3b786..fdbc9ba636 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -43,7 +43,8 @@ use windmill_common::flow_status::{ }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, + JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; @@ -3956,9 +3957,7 @@ async fn compute_next_flow_transform( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = path.unwrap_or_else(|| get_path(flow_job, status, module)); @@ -3968,9 +3967,7 @@ async fn compute_next_flow_transform( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, module, tag, delete_after_use, @@ -3984,9 +3981,7 @@ async fn compute_next_flow_transform( id, // flow_node(id). tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => { let path = get_path(flow_job, status, module); @@ -3995,10 +3990,9 @@ async fn compute_next_flow_transform( payload: JobPayload::FlowScript { id, language, - custom_concurrency_key: custom_concurrency_key.clone(), - concurrent_limit, - concurrency_time_window_s, + concurrency_settings: concurrency_settings.into(), cache_ttl: module.cache_ttl.map(|x| x as i32), + cache_ignore_s3_path: module.cache_ignore_s3_path.clone(), dedicated_worker: None, path, }, @@ -4625,18 +4619,14 @@ async fn payload_from_simple_module( language, lock, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => raw_script_to_payload( path.unwrap_or_else(|| inner_path), content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, module, tag, delete_after_use, @@ -4645,20 +4635,17 @@ async fn payload_from_simple_module( id, // flow_node(id). tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, + concurrency_settings, .. } => JobPayloadWithTag { payload: JobPayload::FlowScript { id, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), + cache_ignore_s3_path: module.cache_ignore_s3_path, dedicated_worker: None, path: inner_path, + concurrency_settings: concurrency_settings.into(), }, tag, delete_after_use, @@ -4674,9 +4661,7 @@ pub fn raw_script_to_payload( content: String, language: windmill_common::scripts::ScriptLang, lock: Option, - custom_concurrency_key: Option, - concurrent_limit: Option, - concurrency_time_window_s: Option, + concurrency_settings: ConcurrencySettingsWithCustom, module: &FlowModule, tag: Option, delete_after_use: bool, @@ -4688,13 +4673,12 @@ pub fn raw_script_to_payload( content, language, lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32), + cache_ignore_s3_path: module.cache_ignore_s3_path, dedicated_worker: None, - custom_debounce_key: None, - debounce_delay_s: None, + concurrency_settings, + // TODO: Should this have debouncing? + debouncing_settings: DebouncingSettings::default(), }), tag, delete_after_use, @@ -4756,11 +4740,6 @@ pub async fn script_to_payload( let ScriptHashInfo { tag, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, - debounce_key, - debounce_delay_s, cache_ttl, language, dedicated_worker, @@ -4769,6 +4748,11 @@ pub async fn script_to_payload( timeout, on_behalf_of_email, created_by, + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + debounce_key, + debounce_delay_s, .. } = get_script_info_for_hash(None, db, &flow_job.workspace_id, hash.0).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { @@ -4784,12 +4768,18 @@ pub async fn script_to_payload( JobPayload::ScriptHash { hash, path: script_path, - custom_concurrency_key: concurrency_key, - concurrent_limit, - concurrency_time_window_s, - custom_debounce_key: debounce_key, - debounce_delay_s, + debouncing_settings: DebouncingSettings { + custom_key: debounce_key, + delay_s: debounce_delay_s, + ..Default::default() + }, + concurrency_settings: ConcurrencySettings { + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + }, cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), + cache_ignore_s3_path: module.cache_ignore_s3_path, language, dedicated_worker, priority, diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index b00d15f4eb..56a3c82fe2 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1171,11 +1171,9 @@ async fn lock_modules<'c>( mut language, input_transforms, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, } = e.get_value()? else { let mut nmodified_ids = Vec::new(); @@ -1541,11 +1539,9 @@ async fn lock_modules<'c>( content, language, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }); new_flow_modules.push(e); @@ -1737,11 +1733,9 @@ async fn reduce_flow<'c>( language, input_transforms, tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, .. } = std::mem::replace(&mut val, Identity) else { @@ -1764,11 +1758,9 @@ async fn reduce_flow<'c>( id, tag, language, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, is_trigger, assets, + concurrency_settings, }; } ForloopFlow { modules, modules_node, .. } diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index f105d8c415..e92092480b 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -395,15 +395,19 @@ pub async fn get_tag_and_concurrency(job_id: &Uuid, db: &DB) -> Option cache::flow::fetch_version(db, version).await, }; let flow_value = flow.map(|f| f.value().clone()).ok(); + let concurrency_key = flow_value .as_ref() - .map(|fv| fv.concurrency_key.clone()) - .flatten(); - let concurrent_limit = flow_value.as_ref().map(|fv| fv.concurrent_limit).flatten(); + .and_then(|fv| fv.concurrency_settings.concurrency_key.to_owned()); + + let concurrent_limit = flow_value + .as_ref() + .and_then(|fv| fv.concurrency_settings.concurrent_limit); + let concurrent_time_window_s = flow_value .as_ref() - .map(|fv| fv.concurrency_time_window_s) - .flatten(); + .and_then(|fv| fv.concurrency_settings.concurrency_time_window_s); + Some(TagAndConcurrencyKey { tag: tag_and_concurrency_key.tag, concurrency_key, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 4434ab9b47..ccffd57d42 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.589.1"; +export const VERSION = "v1.589.3"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/cursor b/cli/cursor new file mode 120000 index 0000000000..80be7b6b55 --- /dev/null +++ b/cli/cursor @@ -0,0 +1 @@ +/usr/bin/cursor \ No newline at end of file diff --git a/cli/deps.ts b/cli/deps.ts index 28d8909d3c..f8ae3f0b65 100644 --- a/cli/deps.ts +++ b/cli/deps.ts @@ -57,7 +57,7 @@ export { WebSocketServer, WebSocket } from "npm:ws"; export * as getPort from "npm:get-port@7.1.0"; export * as open from "npm:open"; export * as esMain from "npm:es-main"; -export * as windmillUtils from "jsr:@windmill-labs/shared-utils@1.0.10"; +export * as windmillUtils from "jsr:@windmill-labs/shared-utils@1.0.11"; // needed for dnt transform import * as wsTypes from "npm:@types/ws"; diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 25c3fccbd2..b1c5c97c1b 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -20,15 +20,19 @@ import { ScriptLanguage, workspaceDependenciesLanguages, } from "../../utils/script_common.ts"; -import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; -import { AppFile as RawAppFile } from "./raw_apps.ts"; +import { + AppFile as RawAppFile, + loadRunnablesFromBackend, + writeRunnableToBackend, +} from "./raw_apps.ts"; import { replaceInlineScripts, AppFile as NormalAppFile } from "./apps.ts"; import { newPathAssigner, + newRawAppPathAssigner, SupportedLanguage, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; @@ -164,21 +168,29 @@ export async function generateAppLocksInternal( ); if (rawApp) { - const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER) + SEP; + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) const rawAppFile = appFile as RawAppFile; + let runnables = await loadRunnablesFromBackend(runnablesPath); + if (Object.keys(runnables).length === 0 && rawAppFile.runnables) { + // Fall back to old format + runnables = rawAppFile.runnables; + } // Replace inline scripts for changed runnables - replaceInlineScripts(rawAppFile.runnables, runnablesPath, false); + replaceInlineScripts(runnables, runnablesPath + SEP, false); - // Update the app runnables with new locks - rawAppFile.runnables = await updateRawAppRunnables( + // Update the app runnables with new locks (writes to separate files) + await updateRawAppRunnables( workspace, - rawAppFile.runnables, + runnables, remote_path, appFolder, rawWorkspaceDependencies, opts.defaultTs ); + // Note: updateRawAppRunnables now writes each runnable to its own file } else { const normalAppFile = appFile as NormalAppFile; @@ -194,13 +206,13 @@ export async function generateAppLocksInternal( rawWorkspaceDependencies, opts.defaultTs ); - } - // Write the updated app file - writeIfChanged( - appFilePath, - yamlStringify(appFile as Record, yamlOptions) - ); + // Write the updated app file (only for normal apps, raw apps use separate files) + writeIfChanged( + appFilePath, + yamlStringify(appFile as Record, yamlOptions) + ); + } } else { log.info(colors.gray(`No scripts changed in ${appFolder}`)); } @@ -279,8 +291,9 @@ async function traverseAndProcessInlineScripts( } /** - * Updates locks for all runnables in a raw app, generating locks inline script by inline script - * Also writes content and locks back to the runnables folder + * Updates locks for all runnables in a raw app, generating locks inline script by inline script. + * Writes each runnable to its own YAML file in the backend folder (new format). + * Also writes content and lock files to the runnables folder. */ async function updateRawAppRunnables( workspace: Workspace, @@ -289,7 +302,7 @@ async function updateRawAppRunnables( appFolder: string, rawDeps?: Record, defaultTs: "bun" | "deno" = "bun" -): Promise> { +): Promise { const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); // Ensure runnables folder exists @@ -299,15 +312,16 @@ async function updateRawAppRunnables( // Folder may already exist } - const pathAssigner = newPathAssigner(defaultTs); - - // Process each runnable - const updatedRunnables: Record = {}; + const pathAssigner = newRawAppPathAssigner(defaultTs); for (const [runnableId, runnable] of Object.entries(runnables)) { - // Only process inline scripts (runnableByName with inlineScript) - if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) { - updatedRunnables[runnableId] = runnable; + // Only process inline scripts (runnableByName/inline with inlineScript) + if ( + (runnable?.type !== "runnableByName" && runnable?.type !== "inline") || + !runnable?.inlineScript + ) { + // Write non-inline runnables to their own file as-is + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } @@ -316,7 +330,7 @@ async function updateRawAppRunnables( const content = inlineScript.content; if (!content || !language) { - updatedRunnables[runnableId] = runnable; + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } @@ -327,13 +341,30 @@ async function updateRawAppRunnables( `Runnable ${runnableId} content is still an !inline reference, skipping` ) ); - updatedRunnables[runnableId] = runnable; + writeRunnableToBackend(runnablesFolder, runnableId, runnable); continue; } // Skip frontend scripts - they don't need locks if (language === "frontend") { - updatedRunnables[runnableId] = runnable; + // Still need to write the runnable YAML file + const [basePathO, ext] = pathAssigner.assignPath( + runnable.name ?? runnableId, + language + ); + const basePath = basePathO.replaceAll(SEP, "/"); + const contentPath = path.join(runnablesFolder, `${basePath}${ext}`); + writeIfChanged(contentPath, content); + + // Write simplified runnable YAML - just type: 'inline' plus metadata + // inlineScript is not needed since content/language can be derived from files + const simplifiedRunnable: Record = { type: "inline" }; + for (const [key, value] of Object.entries(runnable)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } + writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable); continue; } @@ -354,7 +385,10 @@ async function updateRawAppRunnables( ); // Determine file extension for this language - const [basePathO, ext] = pathAssigner.assignPath(runnable.name, language); + const [basePathO, ext] = pathAssigner.assignPath( + runnable.name ?? runnableId, + language + ); const basePath = basePathO.replaceAll(SEP, "/"); const contentPath = path.join(runnablesFolder, `${basePath}${ext}`); const lockPath = path.join(runnablesFolder, `${basePath}lock`); @@ -367,23 +401,21 @@ async function updateRawAppRunnables( writeIfChanged(lockPath, lock); } - // Update the runnable with !inline references (preserve existing schema) - const inlineContentRef = `!inline ${basePath}${ext}`; - const inlineLockRef = - lock && lock !== "" ? `!inline ${basePath}lock` : ""; + // Write simplified runnable YAML - just type: 'inline' plus metadata + // inlineScript is not needed since content/lock/language can be derived from files + const simplifiedRunnable: Record = { type: "inline" }; + for (const [key, value] of Object.entries(runnable)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } - updatedRunnables[runnableId] = { - ...runnable, - inlineScript: { - ...inlineScript, - content: inlineContentRef, - lock: inlineLockRef, - }, - }; + // Write the runnable to its own YAML file + writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable); log.info( colors.gray( - ` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` + ` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` ) ); } catch (error: any) { @@ -392,12 +424,10 @@ async function updateRawAppRunnables( `Failed to generate lock for runnable ${runnableId}: ${error.message}` ) ); - // Continue with other runnables even if one fails - updatedRunnables[runnableId] = runnable; + // Write the original runnable even if lock generation fails + writeRunnableToBackend(runnablesFolder, runnableId, runnable); } } - - return updatedRunnables; } /** @@ -434,33 +464,30 @@ async function updateAppInlineScripts( return inlineScript; } - // Skip frontend scripts - they don't need locks - if (language === "frontend") { - return inlineScript; - } - // Get the name from the parent object (following extractInlineScriptsForApps pattern) // For normal apps, the name is stored in the component's "name" property const scriptName = context.parentObject?.["name"] || "unnamed"; const scriptPath = `${remotePath}/${context.path.join("/")}`; - log.info( - colors.gray( - `Generating lock for inline script "${scriptName}" at ${context.path.join( - "." - )} (${language})` - ) - ); - try { - const lock = await generateInlineScriptLock( - workspace, - content, - language, - scriptPath, - rawDeps - ); + let lock: string | undefined; + if (language !== "frontend") { + log.info( + colors.gray( + `Generating lock for inline script "${scriptName}" at ${context.path.join( + "." + )} (${language})` + ) + ); + lock = await generateInlineScriptLock( + workspace, + content, + language, + scriptPath, + rawDeps + ); + } // Determine file extension for this language (following extractInlineScriptsForApps pattern) const [basePathO, ext] = pathAssigner.assignPath(scriptName, language); const basePath = basePathO.replaceAll(SEP, "/"); @@ -489,7 +516,7 @@ async function updateAppInlineScripts( return { ...inlineScript, content: inlineContentRef, - lock: inlineLockRef, + ...(lock ? { lock: inlineLockRef } : {}), }; } catch (error: any) { log.error( @@ -589,7 +616,7 @@ export interface InferredSchemaResult { /** * Infers schema for a single runnable from its file content. * Used by dev server to update schema in memory (for wmill.d.ts generation). - * Does NOT write to raw_app.yaml - schema is kept in memory only. + * Does NOT write to the runnable YAML file - schema is kept in memory only. * * @param appFolder - The folder containing the raw app * @param runnableFilePath - The path to the changed runnable file (relative to runnables folder) @@ -599,33 +626,52 @@ export async function inferRunnableSchemaFromFile( appFolder: string, runnableFilePath: string ): Promise { - // Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable") + // Extract runnable ID from file path (e.g., "myRunnable.ts" -> "myRunnable") const fileName = path.basename(runnableFilePath); - // Skip lock files - if (fileName.endsWith(".lock")) { + // Skip lock files and yaml files (runnable metadata) + if (fileName.endsWith(".lock") || fileName.endsWith(".yaml")) { return undefined; } - // Match pattern: {runnableId}.inline_script.{ext} - const match = fileName.match(/^(.+)\.inline_script\.[^.]+$/); + // Match pattern: {runnableId}.{ext} - extract the runnable ID (everything before the last dot) + const match = fileName.match(/^(.+)\.[^.]+$/); if (!match) { return undefined; } const runnableId = match[1]; - // Read the app file to get the language - const appFilePath = path.join(appFolder, "raw_app.yaml"); - const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; + // Read the runnable from its separate YAML file (new format) + const runnableFilePath2 = path.join( + appFolder, + APP_BACKEND_FOLDER, + `${runnableId}.yaml` + ); - if (!appFile.runnables?.[runnableId]) { - log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`)); - return undefined; + let runnable: any; + try { + runnable = await yamlParseFile(runnableFilePath2); + } catch { + // Fall back to reading from raw_app.yaml (old format) + try { + const appFilePath = path.join(appFolder, "raw_app.yaml"); + const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; + if (!appFile.runnables?.[runnableId]) { + log.warn( + colors.yellow(`Runnable ${runnableId} not found in backend folder or raw_app.yaml`) + ); + return undefined; + } + runnable = appFile.runnables[runnableId]; + } catch { + log.warn( + colors.yellow(`Could not read runnable ${runnableId} from any source`) + ); + return undefined; + } } - const runnable = appFile.runnables[runnableId]; - // Only process inline scripts if (!runnable?.inlineScript) { return undefined; diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index e2b715b199..c55c396209 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -16,6 +16,7 @@ import { ListableApp, Policy } from "../../../gen/types.gen.ts"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; import devCommand from "./dev.ts"; +import lintCommand from "./lint.ts"; import { isVersionsGeq1585 } from "../sync/global.ts"; export interface AppFile { @@ -230,6 +231,7 @@ const command = new Command() .arguments(" ") .action(push as any) .command("dev", devCommand) + .command("lint", lintCommand) .command( "generate-locks", "re-generate the lockfiles for app runnables inline scripts that have changed" diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 8f45cb5819..3b8ebc03c5 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -206,16 +206,21 @@ export async function createBundle( const wmillPlugin = { name: "wmill-virtual", setup(build: any) { - - - // Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill - build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => { - log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); - return { - path: args.path, - namespace: "wmill-virtual", - }; - }); + // Intercept imports of wmill with various path formats: + // - wmill, wmill.ts (bare import) + // - /wmill, /wmill.ts (absolute) + // - ./wmill, ./wmill.ts (same directory) + // - ../wmill, ../../wmill, etc. (parent directories) + build.onResolve( + { filter: /^(\.\.\/)+wmill(\.ts)?$|^(\.\/|\/)?wmill(\.ts)?$/ }, + (args: any) => { + log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); + return { + path: args.path, + namespace: "wmill-virtual", + }; + } + ); // Provide the virtual module content build.onLoad({ filter: /.*/, namespace: "wmill-virtual" }, (args: any) => { diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 327f06864e..b5b6f22c31 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -34,6 +34,7 @@ import { APP_BACKEND_FOLDER, inferRunnableSchemaFromFile, } from "./app_metadata.ts"; +import { loadRunnablesFromBackend } from "./raw_apps.ts"; const DEFAULT_PORT = 4000; const DEFAULT_HOST = "localhost"; @@ -189,14 +190,21 @@ async function dev(opts: DevOptions) { const wmillPlugin = { name: "wmill-virtual", setup(build: any) { - // Intercept imports of /wmill.ts, /wmill, ./wmill.ts, or ./wmill - build.onResolve({ filter: /^(\.\/|\/)?wmill(\.ts)?$/ }, (args: any) => { - log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); - return { - path: args.path, - namespace: "wmill-virtual", - }; - }); + // Intercept imports of wmill with various path formats: + // - wmill, wmill.ts (bare import) + // - /wmill, /wmill.ts (absolute) + // - ./wmill, ./wmill.ts (same directory) + // - ../wmill, ../../wmill, etc. (parent directories) + build.onResolve( + { filter: /^(\.\.\/)+wmill(\.ts)?$|^(\.\/|\/)?wmill(\.ts)?$/ }, + (args: any) => { + log.info(colors.yellow(`[wmill-virtual] Intercepted: ${args.path}`)); + return { + path: args.path, + namespace: "wmill-virtual", + }; + } + ); // Provide the virtual module content build.onLoad( @@ -665,19 +673,35 @@ export default command; /** * Generates wmill.d.ts with type definitions for runnables. - * Merges in-memory inferred schemas with runnables from raw_app.yaml. + * Loads runnables from separate YAML files in the backend folder (new format) + * or falls back to raw_app.yaml (old format). + * Merges in-memory inferred schemas with runnables. * * @param schemaOverrides - In-memory schema overrides (runnableId -> schema) */ async function genRunnablesTs(schemaOverrides: Record = {}) { log.info(colors.blue("🔄 Generating wmill.d.ts...")); - const rawApp = (await yamlParseFile( - path.join(process.cwd(), "raw_app.yaml") - )) as any; - const runnables = rawApp?.["runnables"] as any; + + const localPath = process.cwd(); + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) + let runnables = await loadRunnablesFromBackend(backendPath); + + if (Object.keys(runnables).length === 0) { + // Fall back to old format + try { + const rawApp = (await yamlParseFile( + path.join(localPath, "raw_app.yaml") + )) as any; + runnables = rawApp?.["runnables"] ?? {}; + } catch { + runnables = {}; + } + } // Apply schema overrides from in-memory cache - if (runnables && Object.keys(schemaOverrides).length > 0) { + if (Object.keys(schemaOverrides).length > 0) { for (const [runnableId, schema] of Object.entries(schemaOverrides)) { if (runnables[runnableId]?.inlineScript) { runnables[runnableId].inlineScript.schema = schema; @@ -697,16 +721,22 @@ async function genRunnablesTs(schemaOverrides: Record = {}) { async function loadRunnables(): Promise> { try { const localPath = process.cwd(); - const rawApp = (await yamlParseFile( - path.join(localPath, "raw_app.yaml") - )) as any; - replaceInlineScripts( - rawApp.runnables, - path.join(localPath, APP_BACKEND_FOLDER) + SEP, - true - ); + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); - return rawApp?.runnables ?? {}; + // Load runnables from separate files (new format) or fall back to raw_app.yaml (old format) + let runnables = await loadRunnablesFromBackend(backendPath); + + if (Object.keys(runnables).length === 0) { + // Fall back to old format + const rawApp = (await yamlParseFile( + path.join(localPath, "raw_app.yaml") + )) as any; + runnables = rawApp?.runnables ?? {}; + } + + replaceInlineScripts(runnables, backendPath + SEP, true); + + return runnables; } catch (error: any) { log.error(colors.red(`Failed to load runnables: ${error.message}`)); return {}; @@ -755,14 +785,12 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; - } else if ( - (runnable.type === "path" || runnable.type === "runnableByPath") && - runnable.path - ) { - const runType = runnable.runType ?? "script"; + } else if (runnable.type === "path" && runnable.runType && runnable.path) { + // Path-based runnables have type: "path" and runType: "script"|"hubscript"|"flow" + const prefix = runnable.runType; requestBody.path = - runType !== "hubscript" - ? `${runType}/${runnable.path}` + prefix !== "hubscript" + ? `${prefix}/${runnable.path}` : `script/${runnable.path}`; } diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts new file mode 100644 index 0000000000..a4d73c6983 --- /dev/null +++ b/cli/src/commands/app/lint.ts @@ -0,0 +1,226 @@ +// deno-lint-ignore-file no-explicit-any +import * as fs from "node:fs"; +import * as path from "node:path"; +import process from "node:process"; +import { Command, colors, log, yamlParseFile } from "../../../deps.ts"; +import { GlobalOptions } from "../../types.ts"; +import { createBundle } from "./bundle.ts"; +import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; +import { loadRunnablesFromBackend } from "./raw_apps.ts"; + +interface LintOptions extends GlobalOptions { + fix?: boolean; +} + +interface LintResult { + valid: boolean; + errors: string[]; + warnings: string[]; +} + +/** + * Validates the structure of raw_app.yaml + */ +function validateRawAppYaml(appData: any): { + errors: string[]; + warnings: string[]; +} { + const errors: string[] = []; + const warnings: string[] = []; + + // Check required fields + if (!appData.summary) { + errors.push("Missing required field: 'summary'"); + } else if (typeof appData.summary !== "string") { + errors.push("Field 'summary' must be a string"); + } + + // Note: 'runnables' is no longer required in raw_app.yaml + // Runnables can be stored in separate files in the backend folder + + return { errors, warnings }; +} + +/** + * Validates that runnables exist either in backend/*.yaml files or in raw_app.yaml + */ +async function validateRunnables( + appDir: string, + appData: any +): Promise<{ errors: string[]; warnings: string[] }> { + const errors: string[] = []; + const warnings: string[] = []; + + const backendPath = path.join(appDir, APP_BACKEND_FOLDER); + + // Load runnables from separate files (new format) + const runnablesFromBackend = await loadRunnablesFromBackend(backendPath); + const hasBackendRunnables = Object.keys(runnablesFromBackend).length > 0; + + // Check for runnables in raw_app.yaml (old format) + const hasYamlRunnables = + appData.runnables && + typeof appData.runnables === "object" && + !Array.isArray(appData.runnables) && + Object.keys(appData.runnables).length > 0; + + if (!hasBackendRunnables && !hasYamlRunnables) { + errors.push( + "No runnables found. Expected either:\n" + + " - Runnable YAML files in the 'backend/' folder (e.g., backend/myRunnable.yaml)\n" + + " - Or a 'runnables' field in raw_app.yaml (legacy format)" + ); + } else if (hasBackendRunnables) { + log.info( + colors.gray( + ` Found ${Object.keys(runnablesFromBackend).length} runnable(s) in backend folder` + ) + ); + } else if (hasYamlRunnables) { + log.info( + colors.gray( + ` Found ${Object.keys(appData.runnables).length} runnable(s) in raw_app.yaml (legacy format)` + ) + ); + warnings.push( + "Using legacy format with runnables in raw_app.yaml. Consider migrating to separate files in backend/" + ); + } + + return { errors, warnings }; +} + +/** + * Checks if the app can be built successfully + */ +async function validateBuild( + appDir: string +): Promise<{ errors: string[]; warnings: string[] }> { + const errors: string[] = []; + const warnings: string[] = []; + + try { + log.info(colors.blue("🔨 Testing build...")); + + // Try to create a bundle - this will validate that all dependencies are in place + await createBundle({ + production: true, + minify: false, + }); + + log.info(colors.green("✅ Build successful")); + } catch (error: any) { + errors.push(`Build failed: ${error.message}`); + } + + return { errors, warnings }; +} + +/** + * Validates a raw app folder + */ +async function lintRawApp( + appDir: string, + opts: LintOptions +): Promise { + const errors: string[] = []; + const warnings: string[] = []; + + // Check if we're in a .raw_app folder + const currentDirName = path.basename(appDir); + if (!currentDirName.endsWith(".raw_app")) { + errors.push( + `Not a raw app folder: '${currentDirName}' does not end with '.raw_app'` + ); + return { valid: false, errors, warnings }; + } + + // Check if raw_app.yaml exists + const rawAppPath = path.join(appDir, "raw_app.yaml"); + if (!fs.existsSync(rawAppPath)) { + errors.push("Missing raw_app.yaml file"); + return { valid: false, errors, warnings }; + } + + log.info(colors.blue("📋 Validating raw_app.yaml structure...")); + + // Parse and validate raw_app.yaml + let appData: any; + try { + appData = await yamlParseFile(rawAppPath); + } catch (error: any) { + errors.push(`Failed to parse raw_app.yaml: ${error.message}`); + return { valid: false, errors, warnings }; + } + + const yamlValidation = validateRawAppYaml(appData); + errors.push(...yamlValidation.errors); + warnings.push(...yamlValidation.warnings); + + if (errors.length > 0) { + return { valid: false, errors, warnings }; + } + + log.info(colors.green("✅ raw_app.yaml structure is valid")); + + // Validate runnables (either in backend folder or in raw_app.yaml) + log.info(colors.blue("📋 Validating runnables...")); + const runnablesValidation = await validateRunnables(appDir, appData); + errors.push(...runnablesValidation.errors); + warnings.push(...runnablesValidation.warnings); + + if (errors.length > 0) { + return { valid: false, errors, warnings }; + } + + log.info(colors.green("✅ Runnables are valid")); + + // Validate build + const buildValidation = await validateBuild(appDir); + errors.push(...buildValidation.errors); + warnings.push(...buildValidation.warnings); + + return { + valid: errors.length === 0, + errors, + warnings, + }; +} + +/** + * Main lint command + */ +async function lint(opts: LintOptions, appFolder?: string) { + const targetDir = appFolder ?? process.cwd(); + + log.info(colors.bold.blue(`\n🔍 Linting raw app: ${targetDir}\n`)); + + const result = await lintRawApp(targetDir, opts); + + // Display results + if (result.warnings.length > 0) { + log.info(colors.yellow("\n⚠️ Warnings:")); + result.warnings.forEach((warning) => { + log.info(colors.yellow(` - ${warning}`)); + }); + } + + if (result.errors.length > 0) { + log.info(colors.red("\n❌ Errors:")); + result.errors.forEach((error) => { + log.info(colors.red(` - ${error}`)); + }); + log.info(colors.red("\n❌ Lint failed\n")); + Deno.exit(1); + } + + log.info(colors.green("\n✅ All checks passed\n")); +} + +const command = new Command() + .description("Lint a raw app folder to validate structure and buildability") + .arguments("[app_folder:string]") + .option("--fix", "Attempt to fix common issues (not implemented yet)") + .action(lint as any); + +export default command; diff --git a/cli/src/commands/app/metadata.ts b/cli/src/commands/app/metadata.ts index a9e9dee812..4f56ab4fb4 100644 --- a/cli/src/commands/app/metadata.ts +++ b/cli/src/commands/app/metadata.ts @@ -13,9 +13,8 @@ export type Runnable = fields?: Record; } | { - type: "runnableByPath" | "path"; + type: "path"; + runType: "script" | "hubscript" | "flow"; path: string; - runType?: "script" | "flow" | "hubscript"; fields?: Record; - schema?: any; }; diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 200b697024..bd2e89e2b8 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -7,24 +7,197 @@ import { SEP, windmillUtils, yamlParseFile, + yamlStringify, } from "../../../deps.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { Policy } from "../../../gen/types.gen.ts"; +import path from "node:path"; import { GlobalOptions, isSuperset } from "../../types.ts"; import { replaceInlineScripts, repopulateFields } from "./apps.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; +import { writeIfChanged } from "../../utils/utils.ts"; +import { yamlOptions } from "../sync/sync.ts"; +import { + EXTENSION_TO_LANGUAGE, + getLanguageFromExtension, +} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; export interface AppFile { - runnables: any; + runnables?: any; custom_path: string; public?: boolean; summary: string; policy: Policy; } +/** + * Finds the content file for a runnable by looking for files matching the runnableId. + * Returns the file extension and content, or undefined if not found. + */ +async function findRunnableContentFile( + backendPath: string, + runnableId: string, + allFiles: string[] +): Promise<{ ext: string; content: string } | undefined> { + // Look for files matching pattern: {runnableId}.{ext} + // where ext is a known language extension + for (const fileName of allFiles) { + // Skip yaml and lock files + if (fileName.endsWith(".yaml") || fileName.endsWith(".lock")) { + continue; + } + + // Check if file starts with runnableId followed by a dot + if (!fileName.startsWith(runnableId + ".")) { + continue; + } + + // Extract extension (everything after the first dot following runnableId) + const ext = fileName.substring(runnableId.length + 1); + + // Check if this is a recognized extension + if (EXTENSION_TO_LANGUAGE[ext]) { + try { + const content = await Deno.readTextFile( + path.join(backendPath, fileName) + ); + return { ext, content }; + } catch { + continue; + } + } + } + return undefined; +} + +/** + * Loads all runnables from separate YAML files in the backend folder. + * Each runnable is stored in a file named `.yaml`. + * + * Converts from file format to API format: + * - For inline scripts (type: 'inline'): derives inlineScript from sibling files + * - For path-based runnables (type: 'script'|'hubscript'|'flow'): converts to API format + * e.g., { type: "script" } -> { type: "path", runType: "script" } + * + * Returns an empty object if the backend folder doesn't exist. + * + * @param backendPath - Path to the backend folder + * @param defaultTs - Default TypeScript runtime ("bun" or "deno") + */ +export async function loadRunnablesFromBackend( + backendPath: string, + defaultTs: "bun" | "deno" = "bun" +): Promise> { + const runnables: Record = {}; + + try { + // First, collect all files in the backend folder + const allFiles: string[] = []; + for await (const entry of Deno.readDir(backendPath)) { + if (entry.isFile) { + allFiles.push(entry.name); + } + } + + // Process YAML files (runnable metadata files) + for (const fileName of allFiles) { + if (!fileName.endsWith(".yaml")) { + continue; + } + + const runnableId = fileName.replace(".yaml", ""); + const filePath = path.join(backendPath, fileName); + const runnable = (await yamlParseFile(filePath)) as Record; + + // If this is an inline script (type: 'inline'), derive inlineScript from files + if (runnable?.type === "inline") { + const contentFile = await findRunnableContentFile( + backendPath, + runnableId, + allFiles + ); + + if (contentFile) { + const language = getLanguageFromExtension(contentFile.ext, defaultTs); + + // Try to load lock file + let lock: string | undefined; + try { + lock = await Deno.readTextFile( + path.join(backendPath, `${runnableId}.lock`) + ); + } catch { + // No lock file, that's fine + } + + // Reconstruct inlineScript object + runnable.inlineScript = { + content: contentFile.content, + language, + ...(lock ? { lock } : {}), + }; + } + } else if ( + runnable?.type === "script" || + runnable?.type === "hubscript" || + runnable?.type === "flow" + ) { + // For path-based runnables, convert from file format to API format + // { type: "script" } -> { type: "path", runType: "script" } + // { type: "hubscript" } -> { type: "path", runType: "hubscript" } + // { type: "flow" } -> { type: "path", runType: "flow" } + const { type, schema: _schema, ...rest } = runnable; + runnable.type = "path"; + runnable.runType = type; + // Remove schema if present + delete runnable.schema; + Object.assign(runnable, rest); + } + + runnables[runnableId] = runnable; + } + } catch (error: any) { + if (error.name !== "NotFound") { + throw error; + } + } + + return runnables; +} + +/** + * Writes a single runnable to its YAML file in the backend folder. + * The file will be named `.yaml`. + * + * Converts from API format to file format: + * - For inline scripts: keeps type: "inline" + * - For path-based runnables: converts { type: "path", runType: "script" } to { type: "script" } + * and removes schema field + */ +export function writeRunnableToBackend( + backendPath: string, + runnableId: string, + runnable: any +): void { + let runnableToWrite = { ...runnable }; + + // Convert path-based runnables from API format to file format + if (runnable.type === "path" && runnable.runType) { + // { type: "path", runType: "script" } -> { type: "script" } + const { type: _type, runType, schema: _schema, ...rest } = runnable; + runnableToWrite = { + type: runType, + ...rest, + }; + } + + const filePath = path.join(backendPath, `${runnableId}.yaml`); + writeIfChanged(filePath, yamlStringify(runnableToWrite, yamlOptions)); +} + const alreadySynced: string[] = []; async function collectAppFiles( @@ -50,10 +223,10 @@ async function collectAppFiles( await readDirRecursive(fullPath + SEP, relativePath + SEP); } else if (entry.isFile) { // Skip raw_app.yaml as it's metadata, not an app file - // Skip node_modules and package-lock.json as they are generated + // Skip package-lock.json as it's generated if ( - relativePath === "raw_app.yaml" || - relativePath === "package-lock.json" + entry.name === "raw_app.yaml" || + entry.name === "package-lock.json" ) { continue; } @@ -99,15 +272,48 @@ export async function pushRawApp( if (!localPath.endsWith(SEP)) { localPath += SEP; } - const path = localPath + "raw_app.yaml"; - const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts( - localApp.runnables, - localPath + SEP + APP_BACKEND_FOLDER + SEP, - true + const appFilePath = localPath + "raw_app.yaml"; + const localApp = (await yamlParseFile(appFilePath)) as AppFile; + + // Load runnables from separate YAML files in the backend folder + // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) + const backendPath = path.join(localPath, APP_BACKEND_FOLDER); + const runnablesFromBackend = await loadRunnablesFromBackend(backendPath); + + let runnables: Record; + if (Object.keys(runnablesFromBackend).length > 0) { + // Use runnables from separate files (new format) + runnables = runnablesFromBackend; + log.info( + colors.gray( + `Loaded ${Object.keys(runnables).length} runnables from backend folder` + ) + ); + } else if (localApp.runnables) { + // Fall back to runnables from raw_app.yaml (old format) + runnables = localApp.runnables; + log.info( + colors.gray( + `Loaded ${ + Object.keys(runnables).length + } runnables from raw_app.yaml (legacy format)` + ) + ); + } else { + runnables = {}; + } + + replaceInlineScripts(runnables, backendPath + SEP, true); + repopulateFields(runnables); + + // Create a temporary app object for policy generation + const appForPolicy = { ...localApp, runnables }; + await generatingPolicy( + appForPolicy, + remotePath, + localApp?.["public"] ?? false ); - repopulateFields(localApp.runnables); - await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false); + const files = await collectAppFiles(localPath); async function createBundleRaw() { log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); @@ -123,7 +329,7 @@ export async function pushRawApp( }); } if (app) { - if (isSuperset(localApp, app)) { + if (isSuperset({ ...localApp, runnables }, app)) { log.info(colors.green(`App ${remotePath} is up to date`)); return; } @@ -134,10 +340,10 @@ export async function pushRawApp( path: remotePath, formData: { app: { - value: { runnables: localApp.runnables, files }, + value: { runnables, files }, path: remotePath, summary: localApp.summary, - policy: localApp.policy, + policy: appForPolicy.policy, deployment_message: message, custom_path: localApp.custom_path, }, @@ -151,10 +357,10 @@ export async function pushRawApp( workspace, formData: { app: { - value: { runnables: localApp.runnables, files }, + value: { runnables, files }, path: remotePath, summary: localApp.summary, - policy: localApp.policy, + policy: appForPolicy.policy, deployment_message: message, custom_path: localApp.custom_path, }, @@ -162,16 +368,6 @@ export async function pushRawApp( css, }, }); - // await wmill.createApp({ - // workspace, - // requestBody: { - // path: remotePath, - // deployment_message: message, - // value: { runnables: localApp.runnables, files }, - // summary: localApp.summary, - // policy: localApp.policy, - // }, - // }); } } diff --git a/cli/src/commands/init/init.ts b/cli/src/commands/init/init.ts index 24b2fbfeb6..d4e13c65e6 100644 --- a/cli/src/commands/init/init.ts +++ b/cli/src/commands/init/init.ts @@ -262,15 +262,13 @@ async function initAction(opts: InitOptions) { await Deno.writeTextFile( "CLAUDE.md", ` - # Claude +You are a helpful assistant that can help with Windmill scripts and flows creation. - You are a helpful assistant that can help with Windmill scripts and flows creation. +## Script Guidance +${scriptGuidanceContent} - ## Script Guidance - ${scriptGuidanceContent} - - ## Flow Guidance - ${flowGuidanceContent} +## Flow Guidance +${flowGuidanceContent} ` ); log.info(colors.green("Created CLAUDE.md")); diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 97a4f085cc..d359d3884c 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -66,6 +66,17 @@ export interface ScriptFile { kind?: "script" | "failure" | "trigger" | "command" | "approval"; } +/** + * Checks if a path is inside a raw app backend folder. + * Matches patterns like: .../myApp.raw_app/backend/... + */ +export function isRawAppBackendPath(filePath: string): boolean { + // Normalize path separators for consistent matching + const normalizedPath = filePath.replaceAll(SEP, "/"); + // Check if path contains pattern: *.raw_app/backend/ + return /\.raw_app\/backend\//.test(normalizedPath); +} + type PushOptions = GlobalOptions; async function push(opts: PushOptions, filePath: string) { opts = await mergeConfigWithConfigFile(opts); @@ -199,6 +210,7 @@ export async function handleFile( ): Promise { if ( !path.includes(".inline_script.") && + !isRawAppBackendPath(path) && exts.some((exts) => path.endsWith(exts)) ) { if (alreadySynced.includes(path)) { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 57222f482b..40a0928f59 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -70,6 +70,7 @@ import { OpenFlow } from "../../../gen/types.gen.ts"; import { pushResource } from "../resource/resource.ts"; import { newPathAssigner, + newRawAppPathAssigner, PathAssigner, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; @@ -284,10 +285,6 @@ function extractFields(fields: Record) { fields[k] = undefined; } } - // if (k == 'runType') { - // fields["type"] = undefined - // fields["schema"] = undefined - // } }); } @@ -311,11 +308,7 @@ export function extractInlineScriptsForApps( } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { - if (k == "runType") { - rec["type"] = undefined; - rec["schema"] = undefined; - return []; - } else if (k == "inlineScript" && typeof v == "object") { + if (k == "inlineScript" && typeof v == "object") { rec["type"] = undefined; const o: Record = v as any; const name = toId(key ?? "", rec); @@ -509,13 +502,14 @@ function ZipFSElement( rawApp.policy = undefined; let inlineScripts; const value = rawApp?.["value"]; + const runnables = value?.["runnables"] ?? {}; // console.log("FOOB", value?.["runnables"]) - extractFieldsForRawApps(value?.["runnables"]); + extractFieldsForRawApps(runnables); try { inlineScripts = extractInlineScriptsForApps( undefined, value, - newPathAssigner(defaultTs), + newRawAppPathAssigner(defaultTs), (key, val_) => key ); } catch (error) { @@ -549,6 +543,7 @@ function ZipFSElement( throw error; } + // Yield inline script content and lock files for (const s of inlineScripts) { yield { isDirectory: false, @@ -561,12 +556,58 @@ function ZipFSElement( }; } - const runnables = value?.["runnables"]; - if (runnables) { - rawApp.runnables = runnables; - delete rawApp?.["value"]; + // Yield each runnable as a separate YAML file in the backend folder + // For inline scripts, simplify the YAML - inlineScript is not needed since + // content/lock/language can be derived from sibling files + for (const [runnableId, runnable] of Object.entries(runnables)) { + const runnableObj = runnable as Record; + let simplifiedRunnable: Record; + + if (runnableObj.inlineScript) { + // For inline scripts, remove inlineScript and just keep type: 'inline' + // plus any other metadata (name, fields, etc.) + simplifiedRunnable = { type: "inline" }; + + // Copy over any other fields that aren't inlineScript or type + for (const [key, value] of Object.entries(runnableObj)) { + if (key !== "inlineScript" && key !== "type") { + simplifiedRunnable[key] = value; + } + } + } else if (runnableObj.type === "path" && runnableObj.runType) { + // For path-based runnables, convert from API format to file format + // { type: "path", runType: "script" } -> { type: "script" } + // Also remove schema field + const { type: _type, runType, schema: _schema, ...rest } = + runnableObj; + simplifiedRunnable = { + type: runType, + ...rest, + }; + } else { + // For other runnables, keep as-is + simplifiedRunnable = runnableObj; + } + + yield { + isDirectory: false, + path: path.join( + finalPath, + APP_BACKEND_FOLDER, + `${runnableId}.yaml` + ), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return yamlStringify(simplifiedRunnable, yamlOptions); + }, + }; } + // Remove runnables and value from raw_app.yaml - they are now in separate files + delete rawApp?.["value"]; + // Don't include runnables in raw_app.yaml anymore + yield { isDirectory: false, path: path.join(finalPath, "raw_app.yaml"), diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index cd52265b52..ecad4ed35e 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -18,7 +18,10 @@ import { removeType, TRIGGER_TYPES, } from "../../types.ts"; -import { fromBranchSpecificPath, isBranchSpecificFile } from "../../core/specific_items.ts"; +import { + fromBranchSpecificPath, + isBranchSpecificFile, +} from "../../core/specific_items.ts"; import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; @@ -32,7 +35,7 @@ type Trigger = { mqtt: MqttTrigger; sqs: SqsTrigger; gcp: GcpTrigger; - email: EmailTrigger + email: EmailTrigger; }; type TriggerFile = Omit< @@ -95,9 +98,7 @@ async function updateTrigger( postgres: wmill.updatePostgresTrigger, mqtt: wmill.updateMqttTrigger, sqs: wmill.updateSqsTrigger, - gcp: async (args) => { - throw new Error("GCP triggers are not supported yet"); - }, + gcp: wmill.updateGcpTrigger, email: wmill.updateEmailTrigger, }; const triggerFunction = triggerFunctions[triggerType]; @@ -124,9 +125,7 @@ async function createTrigger( postgres: wmill.createPostgresTrigger, mqtt: wmill.createMqttTrigger, sqs: wmill.createSqsTrigger, - gcp: async (args) => { - throw new Error("GCP triggers are not supported yet"); - }, + gcp: wmill.createGcpTrigger, email: wmill.createEmailTrigger, }; const triggerFunction = triggerFunctions[triggerType]; diff --git a/cli/src/main.ts b/cli/src/main.ts index 227ba4e003..b9366e6130 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -70,7 +70,7 @@ export { // } // }); -export const VERSION = "1.589.1"; +export const VERSION = "1.589.3"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index d1a79a3a32..d34dcb35db 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -40,13 +40,13 @@ export const LANGUAGE_EXTENSIONS: Record = { /** * Gets the appropriate file extension for a given programming language. * Handles special cases for TypeScript variants based on the default runtime. - * + * * @param language - The programming language to get extension for * @param defaultTs - Default TypeScript runtime ("bun" or "deno") * @returns File extension string (without the dot) */ export function getLanguageExtension( - language: SupportedLanguage, + language: SupportedLanguage, defaultTs: "bun" | "deno" = "bun" ): string { if (language === defaultTs || language === "bunnative") { @@ -55,13 +55,67 @@ export function getLanguageExtension( return LANGUAGE_EXTENSIONS[language] || "no_ext"; } +/** + * Reverse mapping from file extensions to languages. + * Used when deriving language from file extension. + */ +export const EXTENSION_TO_LANGUAGE: Record = { + "py": "python3", + "bun.ts": "bun", + "deno.ts": "deno", + "go": "go", + "sh": "bash", + "ps1": "powershell", + "pg.sql": "postgresql", + "my.sql": "mysql", + "bq.sql": "bigquery", + "odb.sql": "oracledb", + "sf.sql": "snowflake", + "ms.sql": "mssql", + "gql": "graphql", + "native.ts": "nativets", + "frontend.js": "frontend", + "php": "php", + "rs": "rust", + "cs": "csharp", + "nu": "nu", + "playbook.yml": "ansible", + "java": "java", + "duckdb.sql": "duckdb", + // Plain .ts defaults to bun (will be overridden by defaultTs setting) + "ts": "bun", +}; + +/** + * Gets the language from a file extension. + * + * @param ext - File extension (e.g., "py", "ts", "bun.ts") + * @param defaultTs - Default TypeScript runtime for plain .ts files + * @returns The language, or undefined if not recognized + */ +export function getLanguageFromExtension( + ext: string, + defaultTs: "bun" | "deno" = "bun" +): SupportedLanguage | undefined { + // Check for compound extensions first (e.g., "bun.ts", "pg.sql") + const lang = EXTENSION_TO_LANGUAGE[ext]; + if (lang) { + // For plain .ts, return the default TypeScript runtime + if (ext === "ts") { + return defaultTs; + } + return lang; + } + return undefined; +} + export interface PathAssigner { assignPath(summary: string | undefined, language: SupportedLanguage): [string, string]; } /** * Creates a new path assigner for inline scripts. - * + * * @param defaultTs - Default TypeScript runtime ("bun" or "deno") * @returns Path assigner function */ @@ -94,4 +148,43 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { return [`${name}.inline_script.`, ext]; } return { assignPath }; +} + +/** + * Creates a new path assigner for raw app runnables. + * Unlike newPathAssigner, this does NOT add ".inline_script." prefix since + * everything in raw_app/backend/ is already known to be for inline scripts. + * + * @param defaultTs - Default TypeScript runtime ("bun" or "deno") + * @returns Path assigner function + */ +export function newRawAppPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { + let counter = 0; + const seen_names = new Set(); + function assignPath( + summary: string | undefined, + language: SupportedLanguage + ): [string, string] { + let name; + + name = summary?.toLowerCase()?.replaceAll(" ", "_") ?? ""; + + let original_name = name; + + if (name == "") { + original_name = "runnable"; + name = `runnable_0`; + } + + while (seen_names.has(name)) { + counter++; + name = `${original_name}_${counter}`; + } + seen_names.add(name); + + const ext = getLanguageExtension(language, defaultTs); + + return [`${name}.`, ext]; + } + return { assignPath }; } \ No newline at end of file diff --git a/context7.json b/context7.json new file mode 100644 index 0000000000..04ca59792f --- /dev/null +++ b/context7.json @@ -0,0 +1,4 @@ +{ + "url": "https://context7.com/windmill-labs/windmill", + "public_key": "pk_vV2JZHHPk7T5vJ33aWRkH" +} diff --git a/frontend/.gitignore b/frontend/.gitignore index c1addf7ed8..bc35cf2ec3 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -12,4 +12,5 @@ storageState.json dist/ static/tsdocs/ static/ui_builder/ -ui_builder.tar.gz \ No newline at end of file +ui_builder.tar.gz +ui_builder_serve/ \ No newline at end of file diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 29c8aa120a..018107154f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.589.1", + "version": "1.589.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.589.1", + "version": "1.589.3", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 4380aa4359..54672c3367 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.589.1", + "version": "1.589.3", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/serve_ui_builder.sh b/frontend/serve_ui_builder.sh new file mode 100755 index 0000000000..59eb6c3bd3 --- /dev/null +++ b/frontend/serve_ui_builder.sh @@ -0,0 +1,17 @@ +node ./scripts/untar_ui_builder.js + +mkdir ui_builder_serve || true +cp -r static/ui_builder ui_builder_serve/ui_builder || true +rm -rf static/ui_builder || true +python3 -c " +import os +os.chdir('ui_builder_serve') +from http.server import HTTPServer, SimpleHTTPRequestHandler +class H(SimpleHTTPRequestHandler): + def end_headers(self): + self.send_header('Cross-Origin-Opener-Policy', 'same-origin') + self.send_header('Cross-Origin-Embedder-Policy', 'require-corp') + self.send_header('Cross-Origin-Resource-Policy', 'cross-origin') + super().end_headers() +HTTPServer(('', 4000), H).serve_forever() +" \ No newline at end of file diff --git a/frontend/sharedUtils/vite.sharedUtils.config.js b/frontend/sharedUtils/vite.sharedUtils.config.js index 5bf4bcd688..257e103c9f 100644 --- a/frontend/sharedUtils/vite.sharedUtils.config.js +++ b/frontend/sharedUtils/vite.sharedUtils.config.js @@ -5,7 +5,7 @@ import { exec } from 'child_process' import { promisify } from 'util' const execAsync = promisify(exec) -const VERSION = '1.0.10' +const VERSION = '1.0.11' export default defineConfig({ build: { diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 6e4854e550..942e9387cc 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -312,7 +312,8 @@ path: resourceType }) const props: Record = resourceTypeInfo?.schema?.['properties'] ?? {} - const newArgsKeys = Object.keys(props) ?? [] + const newArgsKeys = Object.keys(props).filter((x) => props?.[x]?.type == 'string') ?? [] + const passwords = newArgsKeys.filter((x) => { return props?.[x]?.password }) diff --git a/frontend/src/lib/components/AssignableTagsInner.svelte b/frontend/src/lib/components/AssignableTagsInner.svelte index 4b59d56539..cfca54bc99 100644 --- a/frontend/src/lib/components/AssignableTagsInner.svelte +++ b/frontend/src/lib/components/AssignableTagsInner.svelte @@ -8,8 +8,16 @@ import { superadmin, devopsRole } from '$lib/stores' import NoWorkerWithTagWarning from './runs/NoWorkerWithTagWarning.svelte' import { CUSTOM_TAGS_SETTING } from '$lib/consts' - import { base } from '$lib/base' import { createEventDispatcher } from 'svelte' + import TextInput from './text_input/TextInput.svelte' + import { twMerge } from 'tailwind-merge' + import Badge from './common/badge/Badge.svelte' + + interface Props { + variant?: 'popover' | 'drawer' + } + + let { variant = 'popover' }: Props = $props() let newTag: string = $state('') let customTags: string[] | undefined = $state(undefined) @@ -58,16 +66,48 @@ }) loadCustomTags() + + function onKeyDown(e: KeyboardEvent) { + if (e.key === 'Enter' && newTag.trim() !== '' && tagEditor) { + e.stopPropagation() + e.preventDefault() + saveCustomTag(newTag) + } + } + + async function saveCustomTag(tag: string, restoreCustomTags: boolean = false) { + try { + await SettingService.setGlobal({ + key: CUSTOM_TAGS_SETTING, + requestBody: { value: [...(customTags ?? []), tag.trim().replaceAll(' ', '_')] } + }) + dispatch('refresh') + loadCustomTags() + sendUserToast(restoreCustomTags ? 'Tag restored' : 'Tag added') + if (!restoreCustomTags) { + newTag = '' + } + } catch (err) { + sendUserToast(`Could not ${restoreCustomTags ? 'restore' : 'save'} custom tag: ${err}`, true) + } + } -
+ + +
{#if customTags == undefined} {:else} -
+
{#each customTags as customTag} -
{customTag}
+ + {customTag} + {#if tagEditor} {/if} -
+ {/each}
- + +
+ + +
{#if extractedCustomTag} -
+
Workspace specific tag
Tag: @@ -140,55 +201,40 @@ {/if} {/if} - - Configure worker groups to listen to tags - For tags specific to some workspaces, use 
tag(workspace1+workspace2)
 - To exclude 'workspace1' and 'workspace2' from a tag, use 
tag(^workspace1^workspace2)
 - For tag(workspace1+workspace2) +
{#if variant !== 'drawer'}
{/if} + To exclude 'workspace1' and 'workspace2' from a tag, use + 
tag(^workspace1^workspace2)
+
{#if variant !== 'drawer'}
{/if} + For + dynamic tagsdynamic tags - based on the workspace, use 
$workspace
, e.g: - 
tag-$workspace
 - For $workspace, e.g: + 
tag-$workspace

+ {#if variant !== 'drawer'}
{/if} + + For + dynamic tagsdynamic tags - based on args input, use 
$args[a.b.c]
where - 
a.b.c
is the path to the value in the args object + based on args input, use 
$args[a.b.c]
where + 
a.b.c
is the path to the value in the args object. + {/if}
diff --git a/frontend/src/lib/components/AutoscalingConfigEditor.svelte b/frontend/src/lib/components/AutoscalingConfigEditor.svelte index 37e8dcf4c4..82f260f502 100644 --- a/frontend/src/lib/components/AutoscalingConfigEditor.svelte +++ b/frontend/src/lib/components/AutoscalingConfigEditor.svelte @@ -5,26 +5,35 @@ import Tooltip from './Tooltip.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import { Button } from './common' + import { Alert, Button } from './common' import { ExternalLink } from 'lucide-svelte' - import { createEventDispatcher } from 'svelte' + import TextInput from './text_input/TextInput.svelte' import Label from './Label.svelte' import MultiSelect from './select/MultiSelect.svelte' import { safeSelectItems } from './select/utils.svelte' import { ConfigService } from '$lib/gen' + import Select from './select/Select.svelte' + import ScriptPicker from './ScriptPicker.svelte' + import Badge from './common/badge/Badge.svelte' interface Props { config: AutoscalingConfig | undefined worker_tags: string[] | undefined + disabled: boolean } - let { config = $bindable(), worker_tags }: Props = $props() - - const dispatch = createEventDispatcher() + let { config = $bindable(), worker_tags, disabled }: Props = $props() let test_input: number = $state(3) let healthCheckLoading: boolean = $state(false) let healthCheckResult: { success: boolean; error?: string } | null = $state(null) + function validateMinMax(): string | undefined { + if (config?.min_workers && config?.max_workers && config.min_workers > config.max_workers) { + return 'Minimum cannot be greater than maximum' + } + return undefined + } + async function checkKubernetesHealth() { if (!config?.integration || config.integration.type !== 'kubernetes') return @@ -43,366 +52,258 @@ healthCheckLoading = false } } + + let collapsed: boolean = $state(true) -
-
-
Rules
- { - dispatch('dirty') - if (e.detail) { - if (!config) { - config = { - enabled: true, - min_workers: 3, - max_workers: 10, - integration: { type: 'dryrun' } +
+ {#snippet labelExtra()} + Beta + {/snippet} + {#snippet header()} +
+ { + if (e.detail) { + collapsed = false + if (!config) { + config = { + enabled: true, + min_workers: 3, + max_workers: 10, + integration: { type: 'dryrun' } + } + } else { + config.enabled = true } } else { - config.enabled = true + config = { + ...(config ?? { + min_workers: 3, + max_workers: 10, + integration: { type: 'dryrun' } + }), + enabled: false + } } - } else { - config = { - ...(config ?? { - min_workers: 3, - max_workers: 10, - integration: { type: 'dryrun' } - }), - enabled: false - } - } - }} - /> - -
+ {/snippet} + +
+ - - + -
-
-
- - - - - - + +
- - - - - - - - +
-
-
-
-
Integration
- {#if config?.integration} - dispatch('dirty')} - bind:selected={config.integration.type} - class="mb-4 mt-2" - > - {#snippet children({ item })} - - - - - - {/snippet} - - {#if config.integration.type === 'script'} - - - - -
- - -
-
- -
- - workers -
-
- {/if} - - {#if config.integration.type === 'kubernetes'} -
- Kubernetes configuration is automatically inferred from the cluster environment. The - worker group name and namespace will be detected automatically. -
- -
-
- - -
- - {#if healthCheckResult !== null} -
- {#if healthCheckResult.success} - Kubernetes autoscaling is healthy - {:else} - {healthCheckResult.error} - {#if healthCheckResult.error?.includes('permissions') || healthCheckResult.error?.includes('role')} -
Please follow the setup guide above to configure proper RBAC permissions.Create from template + {disabled} + + {:else} + {/if} +
+ Script must be in the 'admins' workspace + + +
+
+ +
+ {:else if config.integration.type === 'kubernetes'} +
+
+ Kubernetes configuration is automatically inferred from the cluster environment. The + worker group name and namespace will be detected automatically. +
+ +
+
+
+ + +
+ +
+ +
+ + workers +
+
+
+ + {#if healthCheckResult !== null} + + {#if healthCheckResult.success} + Kubernetes autoscaling is healthy + {:else} + {healthCheckResult.error} + {#if healthCheckResult.error?.includes('permissions') || healthCheckResult.error?.includes('role')} +
Please follow the setup guide above to configure proper RBAC permissions. + {/if} + {/if} + {/if}
- {/if} - -
- -
- - workers -
-
- {/if} + {:else if config.integration.type === 'dryrun'} +
+ + In dry run mode, autoscaling will be simulated and events will be logged but no actual + scaling will be performed. + +
+ {/if} +
{:else} {#snippet children({ item })} @@ -414,10 +315,186 @@ {/snippet} - + {/if} -
-
+ + +
+ + + + + + + + +
+ + + +
+ + + + + +
+ diff --git a/frontend/src/lib/components/AutoscalingEvents.svelte b/frontend/src/lib/components/AutoscalingEvents.svelte index 7a9a6e313f..47626dbb44 100644 --- a/frontend/src/lib/components/AutoscalingEvents.svelte +++ b/frontend/src/lib/components/AutoscalingEvents.svelte @@ -1,11 +1,14 @@ -
-
Autoscaling events {#if $enterpriseLicense}(5 last) - -
+
+
+
+

Autoscaling events

+ {#if $enterpriseLicense && events && events.length > 0} + Showing last {Math.min(limit, events.length)} + {/if} +
+ {#if $enterpriseLicense} +
+ {#if !$enterpriseLicense} -
Autoscaling is an EE feature
+
Autoscaling is an EE feature
{:else if loading} {:else if events} {#if events.length == 0} -
No events, is autoscaling set in the worker group config?
- {:else} -
- {#each events as event} -
-
{event.event_type} to {event.desired_workers}
-
{event.reason}
-
-
- {/each} +
+ No events. Is autoscaling configured in the worker group config?
+ {:else} + + + + Event type + Desired workers + Reason + Time + + + + {#each events as event} + + {event.event_type ?? 'N/A'} + {event.desired_workers} + {event.reason ?? 'N/A'} + + + + + {/each} + + + + {#if events.length >= limit && limit < 100} +
+ +
+ {/if} {/if} -
- -
+ {#if limit > 50} -
- Note that autoscaling events are only stored for the last 30 days. +
+ Note: Autoscaling events are only stored for the last 30 days.
{/if} {/if} diff --git a/frontend/src/lib/components/CenteredPage.svelte b/frontend/src/lib/components/CenteredPage.svelte index b6b85933d6..79143e5266 100644 --- a/frontend/src/lib/components/CenteredPage.svelte +++ b/frontend/src/lib/components/CenteredPage.svelte @@ -1,9 +1,18 @@ -
-
- -
+
{@render children?.({ width })}
diff --git a/frontend/src/lib/components/CollapseLink.svelte b/frontend/src/lib/components/CollapseLink.svelte index a723eea703..09cf94f301 100644 --- a/frontend/src/lib/components/CollapseLink.svelte +++ b/frontend/src/lib/components/CollapseLink.svelte @@ -11,7 +11,7 @@
- {/snippet} - {#snippet content()} - - {/snippet} - diff --git a/frontend/src/lib/components/DefaultTagsInner.svelte b/frontend/src/lib/components/DefaultTagsInner.svelte index 610da15112..ab2d403ec8 100644 --- a/frontend/src/lib/components/DefaultTagsInner.svelte +++ b/frontend/src/lib/components/DefaultTagsInner.svelte @@ -1,21 +1,40 @@ -
- {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
- {/if} +
+
+ Jobs that have not been specifically assigned custom tags will use a default tags based on the language they are in or their kind. +
+ + {#snippet action()} + {#if !$enterpriseLicense} + Read only + {:else} + + {/if} + {/snippet} + {#if defaultTagPerWorkspace == undefined || defaultTags == undefined} - {:else} -
- {#each defaultTags.sort() as tag (tag)} -
{tag}
→ -
-
+ {:else if !$enterpriseLicense} + +
+ {#each $state.snapshot(defaultTags).sort() as tag (tag)} + {defaultTagPerWorkspace ? `${tag}-$workspace` : tag} {/each}
+ {:else} +
- +
+ +
{#if defaultTagPerWorkspace} - + {#if limitToWorkspaces} {/if} {/if}
- - When tags use 
$workspace
, the final tag has - 
$workspace
replaced with the workspace id, allowing multi-vpc setup with - more ease, without having to assign a specific tag each time. +
+
Job language or kind
+
+
Default tag
+
+ + +
+ {#each $state.snapshot(defaultTags).sort() as tag (tag)} +
+
+ {tag} +
+ +
→
+
+ {defaultTagPerWorkspace ? `${tag}-$workspace` : tag} +
+
+ {/each} +
{/if} -
+
diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index bbf97a35ec..ebb12077d1 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -4,6 +4,7 @@ import { twMerge } from 'tailwind-merge' import type { MenubarMenuElements } from '@melt-ui/svelte' import type { Item } from '$lib/utils' + import { Tooltip } from './meltComponents' interface Props { aiId?: string @@ -53,6 +54,13 @@ {item.displayName}

{@render item.extra?.()} + {#if item.tooltip} + + {#snippet text()} + {item.tooltip} + {/snippet} + + {/if} {/each}
diff --git a/frontend/src/lib/components/EEOnly.svelte b/frontend/src/lib/components/EEOnly.svelte new file mode 100644 index 0000000000..0bf54b7b68 --- /dev/null +++ b/frontend/src/lib/components/EEOnly.svelte @@ -0,0 +1,33 @@ + + + + + {#snippet text()} + {#if children} + {@render children()} + {:else} + Enterprise Edition only feature + {/if} + {/snippet} + diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 92718fe175..67549a812b 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -101,6 +101,8 @@ import { setMonacoTypescriptOptions } from './monacoLanguagesOptions' import { copilotInfo } from '$lib/aiStore' import { getDbSchemas } from './apps/components/display/dbtable/metadata' + import { rawAppLintStore, type MonacoLintError } from './raw_apps/lintStore' + import { MarkerSeverity } from 'monaco-editor' import { resource, watch } from 'runed' // import EditorTheme from './EditorTheme.svelte' @@ -133,6 +135,8 @@ class?: string | undefined moduleId?: string enablePreprocessorSnippet?: boolean + /** When set, enables raw app lint collection mode and reports Monaco markers to the lint store under this key */ + rawAppRunnableKey?: string | undefined } let { @@ -160,7 +164,8 @@ key = undefined, class: clazz = undefined, moduleId = undefined, - enablePreprocessorSnippet = false + enablePreprocessorSnippet = false, + rawAppRunnableKey = undefined }: Props = $props() $effect.pre(() => { @@ -353,6 +358,25 @@ editor.pushUndoStop() } } + // Update lint diagnostics after code change + updateRawAppLintDiagnostics() + } + + /** Collect Monaco markers and update the raw app lint store */ + function updateRawAppLintDiagnostics(): void { + if (!rawAppRunnableKey || !model) return + const markers = meditor.getModelMarkers({ resource: model.uri }) + const lintErrors: MonacoLintError[] = markers + .filter((m) => m.severity === MarkerSeverity.Error || m.severity === MarkerSeverity.Warning) + .map((m) => ({ + message: m.message, + severity: m.severity === MarkerSeverity.Error ? 'error' : 'warning', + startLineNumber: m.startLineNumber, + startColumn: m.startColumn, + endLineNumber: m.endLineNumber, + endColumn: m.endColumn + })) + rawAppLintStore.setDiagnostics(rawAppRunnableKey, lintErrors) } function updateCode() { @@ -1331,6 +1355,20 @@ // updateEditorKeybindingsMode(editor, 'vim', undefined) + // Raw app lint collection: listen for marker changes and report to store + let markerChangeDisposable: IDisposable | undefined = undefined + if (rawAppRunnableKey && model) { + markerChangeDisposable = meditor.onDidChangeMarkers((uris) => { + if (!model || !rawAppRunnableKey) return + const modelUri = model.uri.toString() + if (uris.some((u) => u.toString() === modelUri)) { + updateRawAppLintDiagnostics() + } + }) + // Initial lint diagnostics collection + updateRawAppLintDiagnostics() + } + let ataModel: number | undefined = undefined editor?.onDidChangeModelContent((event) => { @@ -1450,6 +1488,9 @@ closeWebsockets() vimDisposable?.dispose() closeAIInlineWidget() + markerChangeDisposable?.dispose() + // Note: We don't clear lint diagnostics on dispose - they persist across runnable switches + // Diagnostics are only updated when Monaco reports new markers for this runnable console.log('disposing editor') model?.dispose() editor && editor.dispose() diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index abd2ca5a67..00741605a4 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -37,6 +37,7 @@ Plus, RotateCw, Save, + Settings, Users } from 'lucide-svelte' import { capitalize, formatS3Object, toCamel } from '$lib/utils' @@ -746,7 +747,62 @@ JsonNode ${windmillPathToCamelCaseName(path)} = JsonNode.Parse(await client.GetS (await WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? 'NO_W' })).map( (path) => ({ path }) )} - /> + > + {#snippet submission()} +
+ +
+ {/snippet} + +{/if} + +{#if showDataTablePicker} + { + if (lang === 'duckdb') { + const connStr = name == 'main' ? 'datatable' : `datatable://${name}` + editor?.insertAtCursor(`ATTACH '${connStr}' AS dt;\n`) + } else if (lang === 'python3') { + if (!editor?.getCode().includes('import wmill')) { + editor?.insertAtBeginning('import wmill\n') + } + editor?.insertAtCursor(`db = wmill.datatable(${name == 'main' ? '' : `'${name}'`})\n`) + } else if (['javascript', 'typescript'].includes(scriptLangToEditorLang(lang))) { + if (!editor?.getCode().includes('import * as wmill from')) { + editor?.insertAtBeginning(`import * as wmill from "npm:windmill-client@1"\n`) + } + editor?.insertAtCursor(`let sql = wmill.datatable(${name == 'main' ? '' : `'${name}'`})\n`) + } + }} + tooltip="Attach a datatable to your script." + documentationLink="https://www.windmill.dev/docs/core_concepts/data_tables" + itemName="data table" + loadItems={async () => + (await WorkspaceService.listDataTables({ workspace: $workspaceStore ?? 'NO_W' })).map( + (path) => ({ path }) + )} + > + {#snippet submission()} +
+ +
+ {/snippet} + {/if} {#if showDataTablePicker} diff --git a/frontend/src/lib/components/FlowGraphViewerStep.svelte b/frontend/src/lib/components/FlowGraphViewerStep.svelte index 07ad7756c0..118a1f6f50 100644 --- a/frontend/src/lib/components/FlowGraphViewerStep.svelte +++ b/frontend/src/lib/components/FlowGraphViewerStep.svelte @@ -21,6 +21,7 @@ export let schema: any | undefined = undefined export let stepDetail: FlowModule | string | undefined = undefined + export let jobScriptHash: string | undefined = undefined let codeViewer: Drawer @@ -219,7 +220,7 @@ >
{:else} - + {/if} {:else if stepDetail.value.type == 'aiagent'}
diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 120c7cb9a1..96d5f803ca 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -509,7 +509,12 @@ if (localModuleStates) { innerModules?.forEach((mod, i) => { if (mod.type === 'WaitingForEvents' && innerModules?.[i - 1]?.type === 'Success') { - setModuleState(mod.id ?? '', { type: mod.type, args: job?.args, tag: job?.tag }) + setModuleState(mod.id ?? '', { + type: mod.type, + args: job?.args, + tag: job?.tag, + script_hash: job?.script_hash + }) } else if ( mod.type === 'WaitingForExecutor' && localModuleStates[mod.id ?? '']?.scheduled_for == undefined @@ -527,7 +532,8 @@ job_id: job?.id, parent_module: mod['parent_module'], args: job?.args, - tag: job?.tag + tag: job?.tag, + script_hash: job?.script_hash } setModuleState(mod.id ?? '', newState) @@ -851,7 +857,8 @@ args: job.args, tag: job.tag, started_at, - parent_module: mod['parent_module'] + parent_module: mod['parent_module'], + script_hash: job.script_hash }, force ) @@ -888,7 +895,8 @@ iteration_total: mod.iterator?.itered?.length, retries: mod?.failed_retries?.length, skipped: mod.skipped, - agent_actions: mod.agent_actions + agent_actions: mod.agent_actions, + script_hash: job.script_hash // retries: flowStateStore?.raw_flow }, force @@ -1979,7 +1987,8 @@ >{/if}
{:else if rightColumnSelect == 'node_definition'} - + {@const node = selectedNode ? localModuleStates[selectedNode] : undefined} + {:else if rightColumnSelect == 'user_states'}
diff --git a/frontend/src/lib/components/FlowViewerInner.svelte b/frontend/src/lib/components/FlowViewerInner.svelte index 7fab797408..07052a42d4 100644 --- a/frontend/src/lib/components/FlowViewerInner.svelte +++ b/frontend/src/lib/components/FlowViewerInner.svelte @@ -5,7 +5,7 @@ import { Tab, Tabs, Button } from './common' import { copyToClipboard } from '../utils' - import { ArrowDown, Clipboard } from 'lucide-svelte' + import { ArrowDown, Copy } from 'lucide-svelte' import YAML from 'yaml' import { yaml } from 'svelte-highlight/languages' import HighlightTheme from './HighlightTheme.svelte' @@ -80,7 +80,7 @@ color="light" variant="border" size="xs" - startIcon={{ icon: Clipboard }} + startIcon={{ icon: Copy }} btnClasses="absolute top-2 right-2 w-min z-20" iconOnly /> diff --git a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte index fd0a14dab7..ed6abfda70 100644 --- a/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte +++ b/frontend/src/lib/components/HttpAgentWorkerDrawer.svelte @@ -1,12 +1,18 @@ {#if (!setting.cloudonly || isCloudHosted()) && showSetting(setting.key, $values) && !(setting.hiddenIfNull && $values[setting.key] == null) && !(setting.hiddenIfEmpty && !$values[setting.key])} {#if setting.ee_only != undefined && !$enterpriseLicense} -
- - EE only {#if setting.ee_only != ''}{setting.ee_only}{/if} -
+ + {#if setting.ee_only != ''}{setting.ee_only}{/if} + {/if} {#if setting.fieldType == 'select'}
@@ -571,13 +561,17 @@ {@const currentTeam = $values['critical_error_channels'][i]?.teams_channel ? { team_id: $values['critical_error_channels'][i]?.teams_channel?.team_id, - team_name: $values['critical_error_channels'][i]?.teams_channel?.team_name + team_name: + $values['critical_error_channels'][i]?.teams_channel?.team_name } : undefined} - {@const currentChannel = $values['critical_error_channels'][i]?.teams_channel?.channel_id + {@const currentChannel = $values['critical_error_channels'][i]?.teams_channel + ?.channel_id ? { - channel_id: $values['critical_error_channels'][i]?.teams_channel?.channel_id, - channel_name: $values['critical_error_channels'][i]?.teams_channel?.channel_name + channel_id: + $values['critical_error_channels'][i]?.teams_channel?.channel_id, + channel_name: + $values['critical_error_channels'][i]?.teams_channel?.channel_name } : undefined}
diff --git a/frontend/src/lib/components/Label.svelte b/frontend/src/lib/components/Label.svelte index 42474e5b77..e00ee53cdc 100644 --- a/frontend/src/lib/components/Label.svelte +++ b/frontend/src/lib/components/Label.svelte @@ -2,6 +2,8 @@ import { twMerge } from 'tailwind-merge' import Required from './Required.svelte' import Tooltip from './Tooltip.svelte' + import { enterpriseLicense } from '$lib/stores' + import EEOnly from './EEOnly.svelte' interface Props { label?: string | undefined @@ -12,6 +14,7 @@ class?: string | undefined for?: string | undefined tooltip?: string | undefined + eeOnly?: boolean header?: import('svelte').Snippet error?: import('svelte').Snippet action?: import('svelte').Snippet @@ -27,6 +30,7 @@ class: clazz = undefined, for: forAttr = undefined, tooltip = undefined, + eeOnly = false, header, error, action, @@ -49,6 +53,11 @@ {tooltip} {/if} + {#if eeOnly} + {#if !$enterpriseLicense} + + {/if} + {/if} {@render header?.()}
{/if} diff --git a/frontend/src/lib/components/ManageTagsDrawer.svelte b/frontend/src/lib/components/ManageTagsDrawer.svelte new file mode 100644 index 0000000000..f6fee2fe38 --- /dev/null +++ b/frontend/src/lib/components/ManageTagsDrawer.svelte @@ -0,0 +1,71 @@ + + + + drawer?.closeDrawer?.()}> +
+ +
+ Tags determine which worker group will execute a given job. Workers process only those jobs + whose tags match those defined in their worker group + configuration. +
+ + +
+ +
+ { + if (onRefresh) { + onRefresh() + } + }} + /> +
+ + + + + +
+
+
+ + diff --git a/frontend/src/lib/components/OccupancyBars.svelte b/frontend/src/lib/components/OccupancyBars.svelte new file mode 100644 index 0000000000..7efbf8433e --- /dev/null +++ b/frontend/src/lib/components/OccupancyBars.svelte @@ -0,0 +1,46 @@ + + +
+ {#each rates as rate} + +
+ {#if rate.value !== undefined && rate.value > 0} + {@const heightPercent = Math.min(rate.value * 100, 100)} + {@const minHeight = heightPercent > 0 && heightPercent < 3 ? 1 : heightPercent} +
+ {/if} +
+ {#snippet text()} + {rate.label}: {rate.value ? displayOccupancyRate(rate.value) : '--'} + {/snippet} + + {/each} +
diff --git a/frontend/src/lib/components/QueueAlerts.svelte b/frontend/src/lib/components/QueueAlerts.svelte new file mode 100644 index 0000000000..1f147f52dd --- /dev/null +++ b/frontend/src/lib/components/QueueAlerts.svelte @@ -0,0 +1,572 @@ + + +
+ {#snippet action()} + {#if $enterpriseLicense} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+

Add queue alert

+ +
+ + + Tags that identify which workers to monitor for this alert + +
+ + {#if newAlertForm.tags_to_monitor.length === 0} + + {/if} +
+ {#if formErrors.tags_to_monitor} + {formErrors.tags_to_monitor} + {/if} +
+ +
+ + + Trigger alert when queue exceeds this many jobs + + + {#if formErrors.jobs_num_threshold} + {formErrors.jobs_num_threshold} + {/if} +
+ +
+ + + Wait time between alerts for the same condition + + + {#if formErrors.alert_cooldown_seconds} + {formErrors.alert_cooldown_seconds} + {/if} +
+ +
+ + + How long the condition must persist before alerting + + + {#if formErrors.alert_time_threshold_seconds} + {formErrors.alert_time_threshold_seconds} + {/if} +
+ +
+ + +
+
+ {/snippet} + + {/if} + {/snippet} + + {#if !$enterpriseLicense} +
+ Queue Metric Alerts is an enterprise feature allowing you to monitor queues for waiting jobs. + Please upgrade to access this functionality. Learn more about our plans +
+ {:else if queueAlertConfig.length === 0} +
+

No queue alerts configured

+

Add your first alert to monitor queue conditions

+
+ {:else} +
+ + + + + + + + + + + + {#each queueAlertConfig as config, index} + + + + + + + + {/each} + +
+ + Worker Tags + Tags that identify which workers to monitor for this alert + + + + Jobs Threshold + Trigger alert when queue exceeds this many jobs + + + + Cooldown (s) + Wait time between alerts for the same condition + + + + Time Threshold (s) + How long the condition must persist before alerting + + Actions
+ {#if editingRowIndex === index} +
+ { + if (!editForm.tags_to_monitor.includes(tag)) { + editForm.tags_to_monitor = [...editForm.tags_to_monitor, tag] + } + }} + createText="Press Enter to add custom tag" + placeholder="Select or create tags..." + class="flex-1" + /> + {#if editForm.tags_to_monitor.length === 0} + + {/if} +
+ {:else} + {@const isExpanded = expandedTagRows.includes(index)} + {@const tagsToShow = + isExpanded || config.tags_to_monitor.length <= MAX_NUMBER_OF_TAGS_DISPLAYED + ? config.tags_to_monitor + : config.tags_to_monitor.slice(0, MAX_NUMBER_OF_TAGS_DISPLAYED)} + +
+ {#each tagsToShow as tag} + + {tag} + + {/each} + + {#if config.tags_to_monitor.length > MAX_NUMBER_OF_TAGS_DISPLAYED && !isExpanded} + { + expandedTagRows = [...expandedTagRows, index] + }} + > + + {config.tags_to_monitor.length} + + {/if} +
+ {/if} + 		+ {#if editingRowIndex === index} + + {:else} + {config.jobs_num_threshold} + {/if} + + {#if editingRowIndex === index} + + {:else} + {config.alert_cooldown_seconds} + {/if} + + {#if editingRowIndex === index} + + {:else} + {config.alert_time_threshold_seconds} + {/if} + + {#if editingRowIndex === index} +
+ + +
+ {:else} +
+ + +
+ {/if} +
+
+ {/if} +
diff --git a/frontend/src/lib/components/QueueMetricsDrawer.svelte b/frontend/src/lib/components/QueueMetricsDrawer.svelte index dd4ad426fa..4b05c41899 100644 --- a/frontend/src/lib/components/QueueMetricsDrawer.svelte +++ b/frontend/src/lib/components/QueueMetricsDrawer.svelte @@ -1,423 +1,26 @@ - + -
- {#if $enterpriseLicense} - {#if changesMade} -
Non applied changes
- {/if} -
- - -
+ - {#if alerts.length > 0} -
-
- - - - - - - - - - - - {#each alerts as alert, index} - - - - - - - - {/each} - -
- Queue Tags to Monitor - - - Jobs - - - Cooldown (s) - - - Time (s) - - Actions
- {#if editingIndex === index} -
- {#each alert.tags_to_monitor as tag} - - {tag} - - - {/each} -
-
- filterTags(e)} - disabled={workerTags.length === alert.tags_to_monitor.length} - class="p-1 flex-grow mr-1" - /> - -
- - - {#if filteredTags.length > 0} -
    - {#each filteredTags as tag} - {#if !alert.tags_to_monitor.includes(tag)} -
  • - -
    • - {/if} - {/each} -
- {/if} - {:else} -
- {#each alert.tags_to_monitor as tag} - {tag} - {/each} -
- {/if} - 		- {#if editingIndex === index} - - {:else} - {alert.jobs_num_threshold} - {/if} - - {#if editingIndex === index} - - {:else} - {alert.alert_cooldown_seconds} - {/if} - - {#if editingIndex === index} - - {:else} - {alert.alert_time_threshold_seconds} - {/if} - -
- {#if editingIndex === index} - - {:else} - - - {/if} -
-
-
-
- {/if} +
- -
- -
- {:else} -

- Queue Metric Alerts are an enterprise feature allowing you to monitor queues for waiting - jobs. Please upgrade to access this functionality. - Learn more about our plans. -

- {/if} -
-

Queue Metrics

-
- -
+ + +
diff --git a/frontend/src/lib/components/QueueMetricsDrawerInner.svelte b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte index 76e0b0c72b..f6cc720b88 100644 --- a/frontend/src/lib/components/QueueMetricsDrawerInner.svelte +++ b/frontend/src/lib/components/QueueMetricsDrawerInner.svelte @@ -20,6 +20,7 @@ import Skeleton from './common/skeleton/Skeleton.svelte' import DarkModeObserver from './DarkModeObserver.svelte' import Alert from './common/alert/Alert.svelte' + import { Section } from './common' let loading: boolean = true @@ -186,87 +187,89 @@ -{#if loading} - -{:else if noMetrics} -

No jobs delayed by more than 3 seconds in the last 14 days

-{:else} -
- {#if countData} - 3s)' - } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - y: { +
+ {#if loading} + + {:else if noMetrics} +

No jobs delayed by more than 3 seconds in the last 14 days

+ {:else} +
+ {#if countData} + 3s)' } - } - } - }} - /> - {/if} - {#if delayData} - 3s)' }, - tooltip: { - callbacks: { - label: function (context) { - // @ts-ignore - if (context.raw.y === 1) { - return context.dataset.label + ': 0' - } else { - // @ts-ignore - return context.dataset.label + ': ' + context.raw.y - } + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + y: { + title: { + display: true, + text: 'count' } } } - }, - scales: { - x: { - type: 'time', - min: minDate.toISOString(), - max: new Date().toISOString() - }, - - y: { - type: 'logarithmic', + }} + /> + {/if} + {#if delayData} + 3s)' }, - ticks: { - callback: (value, _) => (value === 1 ? '0' : value) + tooltip: { + callbacks: { + label: function (context) { + // @ts-ignore + if (context.raw.y === 1) { + return context.dataset.label + ': 0' + } else { + // @ts-ignore + return context.dataset.label + ': ' + context.raw.y + } + } + } + } + }, + scales: { + x: { + type: 'time', + min: minDate.toISOString(), + max: new Date().toISOString() + }, + + y: { + type: 'logarithmic', + title: { + display: true, + text: 'delay (s)' + }, + ticks: { + callback: (value, _) => (value === 1 ? '0' : value) + } } } - } - }} - /> - {/if} - - Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are - included in the graph. - -
-{/if} + }} + /> + {/if} + + Only tags for jobs that have been delayed by more than 3 seconds in the last 14 days are + included in the graph. + +
+ {/if} + diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index fd2326cb23..e4f8655157 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -12,7 +12,7 @@ PostgresTriggerService, CaptureService, type ScriptLang, - WorkerService, + WorkerService } from '$lib/gen' import { inferArgs } from '$lib/infer' import { @@ -189,7 +189,7 @@ : undefined ) const simplifiedPoll = writable(false) - + export function setPrimarySchedule(schedule: ScheduleTrigger | undefined | false) { primaryScheduleStore.set(schedule) loadTriggers() @@ -523,6 +523,7 @@ debounce_key: emptyString(script.debounce_key) ? undefined : script.debounce_key, debounce_delay_s: script.debounce_delay_s, cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, ws_error_handler_muted: script.ws_error_handler_muted, priority: script.priority, restart_unless_cancelled: script.restart_unless_cancelled, @@ -666,6 +667,7 @@ debounce_key: emptyString(script.debounce_key) ? undefined : script.debounce_key, debounce_delay_s: script.debounce_delay_s, cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, ws_error_handler_muted: script.ws_error_handler_muted, priority: script.priority, restart_unless_cancelled: script.restart_unless_cancelled, @@ -1046,7 +1048,7 @@ {#snippet content()}
-
+
{#snippet action()} {#if customUi?.settingsPanel?.metadata?.disableMute !== true} @@ -1065,13 +1067,11 @@ onSummaryChange(script.summary)} + onChange={() => onSummaryChange(script.summary)} elementProps={{ type: 'text', placeholder: 'Short summary to be displayed when listed' @@ -1101,7 +1101,6 @@
@@ -1759,7 +1759,6 @@ />
{/if} -
{#if $enterpriseLicense && initialPath != ''} diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index e77d91ca63..d042f3ec03 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -23,7 +23,7 @@ import Modal from './common/modal/Modal.svelte' import DiffEditor from './DiffEditor.svelte' import { - Clipboard, + Copy, CornerDownLeft, ExternalLink, Github, @@ -556,7 +556,7 @@ {:else} {label} + {@render labelExtra?.()} {/if} {@render header?.()} @@ -79,10 +84,7 @@ {/if} {#if eeOnly} {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
+ {/if} {/if} @@ -94,10 +96,17 @@ {/if} {#if !collapsable || !collapsed}
- {@render children?.()} + {#if description} +
{description}
+ {/if} +
+
+ {@render children?.()} +
+
{/if}
diff --git a/frontend/src/lib/components/Subsection.svelte b/frontend/src/lib/components/Subsection.svelte index b40e78e83e..bee078cf44 100644 --- a/frontend/src/lib/components/Subsection.svelte +++ b/frontend/src/lib/components/Subsection.svelte @@ -1,8 +1,9 @@ + +{#if tags.length > 0} +
+ + {#each visibleTags as tag (tag)} + + {tag} + + {/each} + + + {#if hasExtraTags} + + {#snippet trigger()} + + +{extraTags.length} + + {/snippet} + {#snippet content()} +
+ {#each extraTags as tag (tag)} + {tag} + {/each} +
+ {/snippet} + + {/if} +
+{:else} + No tags +{/if} diff --git a/frontend/src/lib/components/TagsToListenTo.svelte b/frontend/src/lib/components/TagsToListenTo.svelte index e99c67223b..f90b045438 100644 --- a/frontend/src/lib/components/TagsToListenTo.svelte +++ b/frontend/src/lib/components/TagsToListenTo.svelte @@ -1,37 +1,75 @@ worker_tags, - (w) => ((worker_tags = w.map((s) => s.replaceAll(' ', '_'))), dispatch('dirty')) - } + bind:this={multiSelect} + bind:value={() => worker_tags, (w) => (worker_tags = w.map((s) => s.replaceAll(' ', '_')))} {disabled} - class={disabled ? 'border-0' : ''} + class={twMerge(disabled ? 'border-0' : '', clazz)} allowClear={!disabled} - onCreateItem={(c) => { - worker_tags.push(c) - dispatch('dirty') - }} - createText="Press Enter to use this tag" + onCreateItem={createCustomTag} + createText={searchText ? `Create custom tag: ${searchText}` : 'Create custom tag'} /> diff --git a/frontend/src/lib/components/TooltipInner.svelte b/frontend/src/lib/components/TooltipInner.svelte index b8c435f167..28dab0595f 100644 --- a/frontend/src/lib/components/TooltipInner.svelte +++ b/frontend/src/lib/components/TooltipInner.svelte @@ -8,7 +8,7 @@
{#if markdownTooltip}
diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index d22e8c8539..0626dffabc 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -1,17 +1,32 @@ drawer?.closeDrawer()} - title={$superadmin || $devopsRole ? `Edit worker config '${name}'` : `Worker config '${name}'`} + title={canEditConfig ? `Edit worker config '${name}'` : `Worker config '${name}'`} + eeOnly > - {#if !$enterpriseLicense} - + {#if !hasEnterpriseFeatures} + Workers can still have their WORKER_TAGS, INIT_SCRIPT and WHITELIST_ENVS passed as env. Dedicated workers are an enterprise only feature. -
+ {:else if !canEditConfig} + + Only superadmin or devops role can edit the worker config. + {/if} + - { - dirty = true - if (nconfig == undefined) { - nconfig = {} - } - if (e.detail == 'dedicated') { - nconfig.dedicated_worker = '' - nconfig.worker_tags = undefined - } else { - nconfig.dedicated_worker = undefined - nconfig.worker_tags = [] - } - }} - class="mb-4" - > - {#snippet children({ item })} - - - {/snippet} - +
{#if selected == 'normal'} -
- {#if nconfig?.worker_tags != undefined} - { - dirty = true - }} - on:deletePriorityTag={(e) => { - const tag = e.detail - if (nconfig.priority_tags) { - delete nconfig.priority_tags[tag] - } - }} - bind:worker_tags={nconfig.worker_tags} - {customTags} - /> - -
- - - - - {#if defaultTagPerWorkspace} - ({ value: w.id }))} + onCreateItem={(c) => (workspaceTag = c)} + placeholder="Workspace ID" + disabled={!canEditConfig} /> - - {/if} -
+ {/if} +
+ {/if} + {/snippet} + {#if nconfig?.worker_tags != undefined} + {/if} - + + + {#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0} +
+ + {/if} + {#if nconfig !== undefined}
-
+
+ {/if} {:else if selected == 'dedicated'}
@@ -429,17 +505,15 @@ {/if} {#if nconfig?.dedicated_worker != undefined}

Workers will get killed upon detecting changes. It is assumed they are in an environment where the supervisor will restart them.

{ - dirty = true - }} + onchange={() => {}} bind:value={nconfig.dedicated_worker} />
@@ -449,52 +523,51 @@
-
-
+
{#each customEnvVars as envvar, i}
{ - dirty = true - }} + onkeypress={(e) => {}} /> { - dirty = true if (e.detail === 'dynamic') { envvar.value = undefined } }} > {#snippet children({ item })} - - + + {/snippet} - - {#if $superadmin || $devopsRole} - + startIcon={{ icon: Trash }} + iconOnly + destructive + disabled={!canEditConfig} + /> {/if}
{/each} - {#if $superadmin || $devopsRole} -
+ {#if canEditConfig} +
+ + + Set up env variables for AWS or SSL using our + { + let updated = false + aws_env_vars_preset.forEach((envvar) => { + if (!customEnvVars.some((e) => e.key === envvar)) { + updated = true + customEnvVars.push({ + key: envvar, + type: 'dynamic', + value: undefined + }) + } + }) + if (updated) { + customEnvVars = [...customEnvVars] + } + } + }, + { + displayName: `SSL (${ssl_env_vars_preset.join(', ')})`, + action: () => { + let updated = false + ssl_env_vars_preset.forEach((envvar) => { + if (!customEnvVars.some((e) => e.key === envvar)) { + updated = true + customEnvVars.push({ + key: envvar, + type: 'dynamic', + value: undefined + }) + } + }) + if (updated) { + customEnvVars = [...customEnvVars] + } + } + } + ]} + > + {#snippet buttonReplacement()} + + {/snippet} + +
{/if}
- {#if !($superadmin || $devopsRole)} -
- + {/if} +
+ {/each} + {/if} + {#if canEditConfig} +
+ - +
+ {/if} + +
+ {/if} +
-
- -
- {#snippet header()} -
- Beta - {#if nconfig.autoscaling?.enabled} - Enabled - {/if} -
- {/snippet} - (dirty = true)} - worker_tags={config?.worker_tags} - bind:config={nconfig.autoscaling} - /> -
- -
-
-
- - {#if nconfig.additional_python_paths} - {#each nconfig.additional_python_paths as _, i} -
- - {#if $superadmin || $devopsRole} - - {/if} -
- {/each} - {/if} - {#if $superadmin || $devopsRole} -
- -
- {/if} - - - {#if nconfig.pip_local_dependencies} - {#each nconfig.pip_local_dependencies as _, i} -
- - {#if $superadmin || $devopsRole} - - {/if} -
- {/each} - {/if} - {#if $superadmin || $devopsRole} -
- -
- {/if} - -
- {#if $superadmin || $devopsRole} + {#if canEditConfig}
- + Workers will get killed upon detecting any changes in this section (scripts or interval). It is assumed they are in an environment where the supervisor will restart them. @@ -737,13 +795,14 @@
-
+
Run at start of the workers. More lightweight than requiring custom worker images.
- {#snippet header()}
@@ -755,7 +814,7 @@
{ if (config) { - dirty = true const code = e.detail if (code != '') { nconfig.init_bash = code?.replace(/\r\n/g, '\n') @@ -774,17 +832,18 @@ }} />
- +
-
+
Run periodically at configurable intervals. Useful for maintenance tasks like cleaning disk space.
- {#snippet header()}
@@ -795,24 +854,24 @@ {/snippet}
- - { - dirty = true - }} - /> - Minimum: 60 seconds +
{ if (config) { - dirty = true const code = e.detail if (code != '') { nconfig.periodic_script_bash = code?.replace(/\r\n/g, '\n') @@ -832,215 +890,282 @@ }} />
- +
{#snippet actions()} -
+
- {#if dirty} -
Non applied changes
- {/if} - + await ConfigService.updateConfig({ name: 'worker__' + name, requestBody: nconfig }) + sendUserToast('Configuration set') + dispatch('reload') + }} + disabled={(!hasChanges && nconfig?.dedicated_worker == undefined) || !canEditConfig} + > + Apply changes + + {:else} + Read only + {/if}
{/snippet} -
- {#if hasWorkersWithoutIsolation(workers)} - {@const unsafeWorkers = getWorkersWithoutIsolation(workers)} -
- - {#snippet trigger()} - - {/snippet} - {#snippet content()} -
-
Workers without job isolation
-

- {unsafeWorkers.length} - {unsafeWorkers.length === 1 ? 'worker' : 'workers'} in this group - {unsafeWorkers.length === 1 ? 'is' : 'are'} running without job isolation (nsjail/unshare). -

-
- {#each unsafeWorkers as worker} - - {worker.worker} - - {/each} -
- - Learn more about job isolation → - -
- {/snippet} - -
- {/if} +
+
+
+
+ {#if selectGroup} + {@render selectGroup()} + + {:else} + Worker group: + {name} - + {/if} + + {`${pluralize(activeWorkers, 'worker')} `} + {#snippet text()}Number of active workers of this group in the last 15 seconds{/snippet} +
-
-
{pluralize(activeWorkers, 'worker')} {#if vcpus_memory?.vcpus} - {(vcpus_memory?.vcpus / 100000).toFixed(2)} vCPUs{/if} {#if vcpus_memory?.memory} - - {((vcpus_memory?.memory * 1.0) / 1024 / 1024 / 1024).toFixed(2)} GB{/if}
-
- {#if $superadmin} - - - - - {#if config} - - {/if} + - - {:else if config} - + {#snippet text()} +
+
Workers without job isolation
+

+ {unsafeWorkers.length} + {unsafeWorkers.length === 1 ? 'worker' : 'workers'} in this group + {unsafeWorkers.length === 1 ? 'is' : 'are'} running without job isolation (nsjail/unshare). +

+
+ {#each unsafeWorkers as worker} + + {worker.worker} + + {/each} +
+ + Learn more about job isolation + +
+ {/snippet} + +
{/if}
+ {#if !isAgent} +
+ {#if canEditConfig} + {#if width > 1000} + {#if config} + + {/if} + + + + {#if config} + + {/if} + {:else} + { + navigator.clipboard.writeText(YAML.stringify({ name, ...config })) + sendUserToast('Worker config copied to clipboard as YAML') + }, + disabled: !config + }, + { + displayName: 'Clean cache', + action: () => { + loadNConfig() + openClean = true + }, + disabled: !config, + type: 'delete' + }, + { + displayName: 'Delete config', + action: () => { + if (!hasEnterpriseFeatures) { + sendUserToast('Worker Management UI is an EE feature', true) + } else { + openDelete = true + } + }, + disabled: !config, + type: 'delete' + } + ]} + /> + {/if} + + {:else if config} + + {/if} +
+ {/if}
+ + {#if config?.worker_tags && config.worker_tags.length > 0} +
+
Tags:
+ +
+ {:else if config?.dedicated_worker} +
+
Dedicated to:
+
+ {config.dedicated_worker} +
+
+ {/if}
+ +{#if isAgent} +
+ + {#snippet children()} + This group is formed with agent workers, there is no associated config. {#if $superadmin || $devopsRole} + To modify the tags, generate a new JWT token .{/if} + {/snippet} + +
+{/if} diff --git a/frontend/src/lib/components/apps/components/display/AppText.svelte b/frontend/src/lib/components/apps/components/display/AppText.svelte index 7d1e09bd06..a7f98089f8 100644 --- a/frontend/src/lib/components/apps/components/display/AppText.svelte +++ b/frontend/src/lib/components/apps/components/display/AppText.svelte @@ -3,7 +3,7 @@ const bubble = createBubbler() import Tooltip from '$lib/components/Tooltip.svelte' - import { Clipboard } from 'lucide-svelte' + import { Copy } from 'lucide-svelte' import { getContext, untrack } from 'svelte' import { twMerge } from 'tailwind-merge' import { copyToClipboard, isCodeInjection } from '../../../../utils' @@ -273,7 +273,7 @@ btnClasses="!p-1" on:click={() => copyToClipboard(result)} > - +
{/if} diff --git a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte index 05b9718cd2..2efe7915c0 100644 --- a/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte +++ b/frontend/src/lib/components/apps/components/helpers/RunnableComponent.svelte @@ -3,7 +3,13 @@ import Alert from '$lib/components/common/alert/Alert.svelte' import Popover from '$lib/components/Popover.svelte' import { type ExecuteComponentData } from '$lib/gen' - import { classNames, defaultIfEmptyString, emptySchema, sendUserToast } from '$lib/utils' + import { + classNames, + defaultIfEmptyString, + emptySchema, + sendUserToast, + undefinedIfEmpty + } from '$lib/utils' import { deepEqual } from 'fast-equals' import { Bug } from 'lucide-svelte' import { createEventDispatcher, getContext, onDestroy, onMount, untrack } from 'svelte' @@ -518,9 +524,11 @@ const requestBody: ExecuteComponentData['requestBody'] = { args: nonStaticRunnableInputs, component: id, - force_viewer_static_fields: !isEditor ? undefined : staticRunnableInputs, - force_viewer_one_of_fields: !isEditor ? undefined : oneOfRunnableInputs, - force_viewer_allow_user_resources: !isEditor ? undefined : allowUserResources + force_viewer_static_fields: !isEditor ? undefined : undefinedIfEmpty(staticRunnableInputs), + force_viewer_one_of_fields: !isEditor ? undefined : undefinedIfEmpty(oneOfRunnableInputs), + force_viewer_allow_user_resources: !isEditor + ? undefined + : undefinedIfEmpty(allowUserResources) } return requestBody } diff --git a/frontend/src/lib/components/apps/components/inputs/AppDateTimeInput.svelte b/frontend/src/lib/components/apps/components/inputs/AppDateTimeInput.svelte index 3c56acc0a3..9ad56f8c40 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppDateTimeInput.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppDateTimeInput.svelte @@ -105,6 +105,11 @@ ;[resolvedConfig.defaultValue] untrack(() => handleDefault(resolvedConfig.defaultValue)) }) + function setIterValue(value: string | undefined) { + if (iterContext && listInputs) { + listInputs.set(id, value) + } + } $effect.pre(() => { if (value) { const r = formatDate(value, resolvedConfig.outputFormat) @@ -141,16 +146,14 @@ outputs?.validity.set(false) } } - if (iterContext && listInputs) { - listInputs.set(id, value) - } + untrack(() => setIterValue(value)) } else { outputs?.result.set(undefined) - if (iterContext && listInputs) { - listInputs.set(id, undefined) - } + untrack(() => setIterValue(undefined)) } - fireOnChange() + untrack(() => { + fireOnChange() + }) }) diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index 9b6f5a9b2f..81f59a5028 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -2,7 +2,7 @@ import { Alert } from '$lib/components/common' import Toggle from '$lib/components/Toggle.svelte' import { enterpriseLicense, userStore, workspaceStore } from '$lib/stores' - import { Loader2, AlertTriangle } from 'lucide-svelte' + import { Loader2 } from 'lucide-svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -13,6 +13,7 @@ import { computeSecretUrl } from './appDeploy.svelte' import { base } from '$lib/base' import { isCloudHosted } from '$lib/cloud' + import EEOnly from '$lib/components/EEOnly.svelte' let { policy, @@ -216,11 +217,9 @@
{/if} + {#if !$enterpriseLicense} -
- - EE only Enterprise Edition only feature -
+ {/if} { diff --git a/frontend/src/lib/components/apps/editor/AppExportButton.svelte b/frontend/src/lib/components/apps/editor/AppExportButton.svelte index 0bc4cf4b10..2090323725 100644 --- a/frontend/src/lib/components/apps/editor/AppExportButton.svelte +++ b/frontend/src/lib/components/apps/editor/AppExportButton.svelte @@ -7,7 +7,7 @@ import { Highlight } from 'svelte-highlight' import json from 'svelte-highlight/languages/json' import { Button } from '../../common' - import { Clipboard } from 'lucide-svelte' + import { Copy } from 'lucide-svelte' import { yaml } from 'svelte-highlight/languages' import YAML from 'yaml' import Tabs from '$lib/components/common/tabs/Tabs.svelte' @@ -45,7 +45,7 @@ )} variant="accent" size="sm" - startIcon={{ icon: Clipboard }} + startIcon={{ icon: Copy }} btnClasses="absolute top-2 right-2 w-min z-20" iconOnly /> diff --git a/frontend/src/lib/components/apps/editor/appUtilsS3.ts b/frontend/src/lib/components/apps/editor/appUtilsS3.ts index be1a581c3d..5e35f1a4bd 100644 --- a/frontend/src/lib/components/apps/editor/appUtilsS3.ts +++ b/frontend/src/lib/components/apps/editor/appUtilsS3.ts @@ -1,4 +1,3 @@ -import { defaultIfEmptyString } from '$lib/utils' import type { AppInput, EvalInputV2 } from '../inputType' import type { App, RichConfigurations } from '../types' import { collectOneOfFields } from './appUtilsCore' @@ -11,6 +10,10 @@ function filenameExprToRegex(template: string) { return `^${regexPattern}$` } +function defaultIfEmptyString(str: string | undefined, dflt: string): string { + return str === undefined || str === null || str === '' ? dflt : str! +} + function staticToRegex(str: string) { return `^${str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$` } diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte b/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte index 6fd86c9ad7..d081549dbd 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte +++ b/frontend/src/lib/components/apps/editor/componentsPanel/CssSettings.svelte @@ -1,6 +1,6 @@
-
+
{tooltip} {/if} + {#if eeOnly && !$enterpriseLicense} + + {/if}
{#if actions}
diff --git a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte index 916c3b671b..eaf3adcb20 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUploadModal.svelte @@ -32,7 +32,7 @@ {#if open} + +
+
+
+ Flow wildcard patterns +
+ +
+
{/if} {:else if mcpCreationMode && (newMcpScope !== 'folder' || selectedFolder.length > 0)} @@ -616,7 +691,9 @@ (newMcpScope === 'custom' && selectedScripts.length === 0 && selectedFlows.length === 0 && - selectedEndpoints.length === 0))} + selectedEndpoints.length === 0 && + !customScriptPatterns.trim() && + !customFlowPatterns.trim()))} variant="accent" > New token diff --git a/frontend/src/lib/components/settings/TokenDisplay.svelte b/frontend/src/lib/components/settings/TokenDisplay.svelte index 1a29fab407..5697b55b1b 100644 --- a/frontend/src/lib/components/settings/TokenDisplay.svelte +++ b/frontend/src/lib/components/settings/TokenDisplay.svelte @@ -1,105 +1,85 @@ -
+ +
+ + {#if onClose} + + {/if} +
-
+ +
{#if mcpUrl} - - - + {:else} - - - + {/if}
-
-

+ +
+ +

{displayTitle}

- -
-
- - -
-
- - {tokenOrUrl} - - -
-
-
+ +
+ + + + +
{info} - {#if mcpUrl} -
-

- Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) to access your Windmill scripts and flows as tools. -

-
- {/if}
- + + {#if mcpUrl} + +
+

+ Next steps: Use this URL in your MCP-compatible client (like Claude Desktop) + to access your Windmill scripts and flows as tools. +

+
+ {/if} +
-

\ No newline at end of file +
diff --git a/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte index edbb3329bd..468acfc423 100644 --- a/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte +++ b/frontend/src/lib/components/sidebar/CriticalAlertTable.svelte @@ -63,9 +63,9 @@ Acked
{/key} {/key} @@ -452,7 +452,7 @@ done` }} > - +
{/key}{/key}{/key}{/key} {/key} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 0a997fd43d..21f2ff106d 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -153,6 +153,20 @@
+
+ +
+

AI Providers

@@ -325,15 +339,4 @@ {#if Object.keys(aiProviders).length > 0} {/if} - -
diff --git a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte index c126616d84..9a8a2d443f 100644 --- a/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte +++ b/frontend/src/lib/components/workspaceSettings/CustomInstanceDbWizardModal.svelte @@ -172,7 +172,7 @@ if (!wasAlreadySuccessful) sendUserToast('Setup successful') else sendUserToast('Check successful') } else { - sendUserToast(result.error ?? 'An error occured', true) + sendUserToast(result.error ?? 'An error occurred', true) } } catch (e) { sendUserToast('Unexpected error, check console for details', true) diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 82edc30310..f3e9816783 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -138,7 +138,7 @@ ) ) { let confirm = await confirmationModal.ask({ - title: 'Some instance catalogs are not setup', + title: 'Some instance databases are not setup', children: 'Are you sure you want to save without setting them up ?', confirmationText: 'Save anyway' }) @@ -192,8 +192,8 @@ {#if ducklakeSettings.ducklakes.some((d) => d.catalog.resource_type === 'instance')}
- - Using an instance catalog is the fastest way to get started with Ducklake. They are public to + + Using an instance database is the fastest way to get started with Ducklake. They are public to the instance and can be re-used in other workspaces' Ducklake settings.
diff --git a/frontend/src/lib/editorLangUtils.ts b/frontend/src/lib/editorLangUtils.ts index c856d1bd8b..c52ff51f74 100644 --- a/frontend/src/lib/editorLangUtils.ts +++ b/frontend/src/lib/editorLangUtils.ts @@ -1,114 +1,114 @@ export function langToExt(lang: string): string { - switch (lang) { - case 'tsx': - return 'tsx' - case 'javascript': - return 'js' - case 'jsx': - return 'js' - case 'bunnative': - return 'ts' - case 'json': - return 'json' - case 'sql': - return 'sql' - case 'yaml': - return 'yaml' - case 'typescript': - return 'ts' - case 'python': - return 'py' - case 'go': - return 'go' - case 'bash': - return 'sh' - case 'powershell': - return 'ps1' - case 'php': - return 'php' - case 'rust': - return 'rs' - case 'deno': - return 'ts' - case 'nativets': - return 'ts' - case 'graphql': - return 'gql' - case 'css': - return 'css' - case 'ansible': - return 'yml' - case 'csharp': - return 'cs' - case 'nu': - return 'nu' - case 'java': - return 'java' - case 'svelte': - return 'svelte' - case 'vue': - return 'vue' - default: - return 'unknown' - } + switch (lang) { + case 'tsx': + return 'tsx' + case 'javascript': + return 'js' + case 'jsx': + return 'js' + case 'bunnative': + return 'ts' + case 'json': + return 'json' + case 'sql': + return 'sql' + case 'yaml': + return 'yaml' + case 'typescript': + return 'ts' + case 'python': + return 'py' + case 'go': + return 'go' + case 'bash': + return 'sh' + case 'powershell': + return 'ps1' + case 'php': + return 'php' + case 'rust': + return 'rs' + case 'deno': + return 'ts' + case 'nativets': + return 'ts' + case 'graphql': + return 'gql' + case 'css': + return 'css' + case 'ansible': + return 'yml' + case 'csharp': + return 'cs' + case 'nu': + return 'nu' + case 'java': + return 'java' + case 'svelte': + return 'svelte' + case 'vue': + return 'vue' + default: + return 'unknown' + } } export function extToLang(ext: string) { - switch (ext) { - case 'tsx': - return 'typescript' - case 'ts': - return 'typescript' - case 'js': - return 'javascript' - case 'jsx': - return 'javascript' - case 'json': - return 'json' - case 'sql': - return 'sql' - case 'yaml': - return 'yaml' - case 'py': - return 'python' - case 'go': - return 'go' - case 'sh': - return 'bash' - case 'ps1': - return 'powershell' - case 'php': - return 'php' - case 'rs': - return 'rust' - case 'gql': - return 'graphql' - case 'css': - return 'css' - case 'yml': - return 'ansible' - case 'cs': - return 'csharp' - case 'svelte': - return 'svelte' - case 'vue': - return 'vue' - case 'nu': - return 'nu' - case 'java': - return 'java' - case 'rb': - return 'ruby' - // for related places search: ADD_NEW_LANG - default: - return 'unknown' - } + switch (ext) { + case 'tsx': + return 'typescript' + case 'ts': + return 'typescript' + case 'js': + return 'javascript' + case 'jsx': + return 'javascript' + case 'json': + return 'json' + case 'sql': + return 'sql' + case 'yaml': + return 'yaml' + case 'py': + return 'python' + case 'go': + return 'go' + case 'sh': + return 'bash' + case 'ps1': + return 'powershell' + case 'php': + return 'php' + case 'rs': + return 'rust' + case 'gql': + return 'graphql' + case 'css': + return 'css' + case 'yml': + return 'ansible' + case 'cs': + return 'csharp' + case 'svelte': + return 'svelte' + case 'vue': + return 'vue' + case 'nu': + return 'nu' + case 'java': + return 'java' + case 'rb': + return 'ruby' + // for related places search: ADD_NEW_LANG + default: + return 'unknown' + } } export function createHash() { - return (Math.random() + 1).toString(36).substring(2) + return (Math.random() + 1).toString(36).substring(2) } export function createLongHash() { - return 'h' + Math.random().toString(36).substring(2) + Math.random().toString(36).substring(2) + return 'h' + Math.random().toString(36).substring(2) + Math.random().toString(36).substring(2) } diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 50884f3c26..42ab22ab4a 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -68,6 +68,15 @@ export function scriptLangToEditorLang( } } +export function extToScriptLang(lang: string): 'bun' | 'python3' | undefined { + switch (lang) { + case 'ts': + return 'bun' + case 'py': + return 'python3' + } + return undefined +} export type ScriptSchedule = { summary: string | undefined args: Record diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 14d3719463..eeb3cf8589 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -370,6 +370,13 @@ export function clickOutside( } } +export function undefinedIfEmpty(obj: any): any { + if (Object.keys(obj).length === 0) { + return undefined + } + return obj +} + export function pointerDownOutside( node: Node, options?: ClickOutsideOptions @@ -1343,6 +1350,7 @@ export type Item = { hide?: boolean | undefined extra?: Snippet id?: string + tooltip?: string } export function isObjectTooBig(obj: any): boolean { diff --git a/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte b/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte index c4d2861404..13ff9e6e6d 100644 --- a/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/apps_raw/add/+page.svelte @@ -12,7 +12,7 @@ import RawAppEditor from '$lib/components/raw_apps/RawAppEditor.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import FileEditorIcon from '$lib/components/raw_apps/FileEditorIcon.svelte' - import { react18Template, react19Template, svelte5Template, vueTemplate } from './templates' + import { react18Template, react19Template, svelte5Template } from './templates' import type { Runnable } from '$lib/components/raw_apps/rawAppPolicy' let nodraft = $page.url.searchParams.get('nodraft') @@ -136,12 +136,12 @@ name: 'Svelte 5', icon: 'svelte', files: svelte5Template - }, - { - name: 'Vue 3', - icon: 'vue', - files: vueTemplate } + // { + // name: 'Vue 3', + // icon: 'vue', + // files: vueTemplate + // } ] let templatePicker = $state(nodraft != null) let reloadCounter = $state(0) @@ -158,6 +158,12 @@ reloadCounter += 1 } templatePicker = false + // Remove nodraft from URL when a template is selected + const url = new URL(window.location.href) + if (url.searchParams.has('nodraft')) { + url.searchParams.delete('nodraft') + window.history.replaceState({}, '', url.toString()) + } }} class="w-24 h-24 flex justify-between py-5 flex-col {t.selected ? 'bg-surface-selected' diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 55c0b4f775..a989a830ee 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -75,6 +75,7 @@ let inputSelected: 'saved' | 'history' | undefined = $state(undefined) let jsonView = $state(false) let deploymentInProgress = $state(false) + let deploymentJobId: string | undefined = $state(undefined) let intervalId: number | undefined = undefined @@ -166,8 +167,11 @@ }) if (status.lock_error_logs == undefined || status.lock_error_logs != '') { deploymentInProgress = false + deploymentJobId = undefined flow.lock_error_logs = status.lock_error_logs clearInterval(intervalId) + } else if (status.job_id) { + deploymentJobId = status.job_id } } } @@ -541,6 +545,13 @@ Deployment in progress + {#if deploymentJobId} + view job + {/if} {/if} {#if flow.lock_error_logs && flow.lock_error_logs != ''} @@ -660,6 +671,9 @@ on:selected_args={(e) => { const nargs = JSON.parse(JSON.stringify(e.detail)) args = nargs + if (jsonView) { + runForm?.setCode(JSON.stringify(args ?? {}, null, '\t')) + } }} /> {/snippet} diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 0dbe481e9f..40a0a51d1d 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -89,6 +89,7 @@ let topHash: string | undefined = $state() let can_write = $state(false) let deploymentInProgress = $state(false) + let deploymentJobId: string | undefined = $state(undefined) let intervalId: number let shareModal: ShareModal | undefined = $state() let runForm: RunForm | undefined = $state() @@ -156,9 +157,12 @@ }) if (status.lock != undefined || status.lock_error_logs != undefined) { deploymentInProgress = false + deploymentJobId = undefined script.lock = status.lock script.lock_error_logs = status.lock_error_logs clearInterval(intervalId) + } else if (status.job_id) { + deploymentJobId = status.job_id } } } @@ -689,6 +693,13 @@ Deployment in progress + {#if deploymentJobId} + view job + {/if} {/if} diff --git a/frontend/src/routes/(root)/(logged)/workers/+page.svelte b/frontend/src/routes/(root)/(logged)/workers/+page.svelte index e127ba9a36..32625a5bd8 100644 --- a/frontend/src/routes/(root)/(logged)/workers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workers/+page.svelte @@ -1,14 +1,14 @@ {#if $superadmin || $devopsRole} + { + loadCustomTags() + }} + /> {/if} @@ -346,7 +448,12 @@ fixedOverflowWidgets={false} /> {#snippet actions()} - + {/snippet} @@ -385,25 +492,27 @@
{:else} - - {#if $superadmin || $devopsRole} -
-
- { - loadCustomTags() - }} + {#snippet children({ width })} + + {#if $superadmin || $devopsRole} +
+
-
- -
-
+ startIcon={{ + icon: Tags + }} + on:click={() => { + manageTagsDrawer?.openDrawer() + }} + > + Manage tags + + -
-
- -
-
- {/if} - - {#if workers != undefined} - {#if groupedWorkers.length == 0} -

No workers seem to be available

- {/if} - -

{groupWorkers?.length} Worker Groups Worker groups are groups of workers that share a config and are meant to be identical. - Worker groups are meant to be used with tags. Tags can be assigned to scripts and flows - and can be seen as dedicated queues. Only the corresponding -

-
- - {#if $superadmin || $devopsRole} -
{ + if ( + e.key === 'Enter' && + newConfigName && + newConfigName.trim() !== '' && + $enterpriseLicense + ) { + addConfig() + } + }} + onClose={() => { + newConfigName = '' + }} + bind:this={newGroupPopover} + targetId="new-group-popover-trigger" > - + {#snippet trigger()}
- - + {/snippet} + {#snippet content()}
{#if !$enterpriseLicense} -
- - EE only -
+ {/if}
- + {/snippet}
- {/if}
+ {/if} + - {#if (groupedWorkers ?? []).length > 5} -
-
Worker group:
- + {#if worker_group} + {@const config = (workerGroups ?? {})[worker_group[0]]} + {@const activeWorkers = worker_group?.[1].flatMap((x) => + x[1]?.filter((y) => (y.last_ping ?? 0) < 15) + )} + + { + loadWorkerGroups() + }} + activeWorkers={activeWorkers?.length ?? 0} + {defaultTagPerWorkspace} + {width} + shouldAutoOpenDrawer={shouldAutoOpenDrawer === worker_group[0]} + onDrawerOpened={() => { + shouldAutoOpenDrawer = '' + }} + onDeleted={handleWorkerGroupDeleted} + selectGroup={(groupedWorkers ?? []).length > 5 ? selectGroup : undefined} /> - -
- {#if worker_group?.[1].length == 0 && search} -
- No workers found. Reset the search to see all workers. + +
+ +
+
Active workers
+
+ + +
- {:else} + Worker - -
- Worker Tags - - If defined, the workers only pull jobs with the same corresponding tag - -
- - Last ping + {#if !config || !config.worker_tags || config.worker_tags.length === 0} + +
+ Worker tags + + If defined, the workers only pull jobs with the same corresponding tag + +
+ + {/if} Worker start Jobs ran {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} @@ -625,10 +724,10 @@ Memory usage
(Windmill) Limits Version - Liveness + Status {#if $superadmin || $devopsRole} - Live Shell + Repl

Open a live shell to execute bash commands on the machine where the worker @@ -639,173 +738,279 @@ {/if} - - {#each worker_group[1] as [section, workers]} - {@const hostname = section?.split(splitter)?.[0]} - - -

-
- Host: - {hostname} -
- IP: - {workers[0].ip} + + {#if worker_group?.[1].length > 0} + {#each worker_group[1] as [section, workers], groupIdx} + {@const hostname = section?.split(splitter)?.[0]} + + +
+
+ Host: + {hostname} +
+ IP: + {workers[0].ip} - {#if workers?.length > 1} - {workers?.length} Workers - {/if} + {#if workers?.length > 1} + {workers?.length} Workers + {/if} +
+ + + {#if workers} + {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }} + {@const isWorkerAlive = isWorkerMaybeAlive(last_ping)} + {@const tagMismatchInfo = getTagMismatchInfo( + custom_tags, + config?.worker_tags + )} + + + {@const underscorePos = worker.search('_')} +
+ + {#if underscorePos === -1} + {worker} + {:else} + {truncate(worker, underscorePos)} + {worker} + {/if} + + {#if config && tagMismatchInfo.hasMismatch} + + + + + + {#snippet text()} +
+
Tag configuration mismatch
+

This worker has tags that differ from the config. This + could be due to a recent update of the config or because the + worker has tags defined as environment variables.

+ + {#if tagMismatchInfo.added.length > 0} +
+
Additional tags:
+ +
+ {/if} + + {#if tagMismatchInfo.removed.length > 0} +
+
Missing tags:
+ +
+ {/if} +
+ {/snippet} + + {/if} +
+ + {#if !config || !config.worker_tags || config.worker_tags.length === 0} + + + + {/if} + {displayDate(started_at)} + {jobs_executed} + {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} + + {#if last_job_id} + + View last job + + +
+ {last_job_workspace_id} + {/if} + + + + + {/if} + +
+
+ {memory_usage + ? Math.round(memory_usage / 1024 / 1024) + 'MB' + : '--'} +
+
+ ({wm_memory_usage + ? Math.round(wm_memory_usage / 1024 / 1024) + 'MB' + : '--'}) +
+
+ + +
+
+ {vcpus ? (vcpus / 100000).toFixed(2) + ' vCPUs' : '--'} +
+
+ {memory ? Math.round(memory / 1024 / 1024) + 'MB' : '--'} +
+
+ + +
+ {wm_version.split('-')[0]} +
+ + + {@const pingText = + last_ping != undefined + ? `${last_ping + timeSinceLastPing}s ago` + : 'Unknown'} + {@const statusText = + isWorkerAlive != undefined + ? isWorkerAlive + ? 'Alive' + : 'Dead' + : 'Unknown'} +
+ + {statusText} + +
{pingText}
+
+ + {#if $superadmin || $devopsRole} + + + + {/if} + + {/each} + {/if} + {/each} + {:else if search} + + +
+ No active workers found matching the search query
- {#if workers} - {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }} - {@const isWorkerAlive = isWorkerMaybeAlive(last_ping)} - - - {@const underscorePos = worker.search('_')} - {#if underscorePos === -1} - {worker} - {:else} - {truncate(worker, underscorePos)} - {worker} - {/if} - - - {#if custom_tags && custom_tags?.length > 2} - {truncate(custom_tags?.join(', ') ?? '', 10)} - {custom_tags?.join(', ')} - {:else} - {custom_tags?.join(', ') ?? ''} - {/if} - - {last_ping != undefined ? last_ping + timeSinceLastPing : -1}s ago - {displayDate(started_at)} - {jobs_executed} - {#if (!config || config?.dedicated_worker == undefined) && ($superadmin || $devopsRole)} - - {#if last_job_id} - - View last job - -
- (workspace {last_job_workspace_id}) - {/if} - - - {displayOccupancyRate(occupancy_rate_15s)}/{displayOccupancyRate( - occupancy_rate_5m - )}/{displayOccupancyRate(occupancy_rate_30m)}/{displayOccupancyRate( - occupancy_rate - )} - - {/if} - -
-
- {memory_usage ? Math.round(memory_usage / 1024 / 1024) + 'MB' : '--'} -
-
- ({wm_memory_usage - ? Math.round(wm_memory_usage / 1024 / 1024) + 'MB' - : '--'}) -
-
- - -
-
- {vcpus ? (vcpus / 100000).toFixed(2) + ' vCPUs' : '--'} -
-
- {memory ? Math.round(memory / 1024 / 1024) + 'MB' : '--'} -
-
- - -
- {wm_version.split('-')[0]}{wm_version} -
- - - - {isWorkerAlive != undefined - ? isWorkerAlive - ? 'Alive' - : 'Dead' - : 'Unknown'} - - - {#if $superadmin || $devopsRole} - - - - {/if} - - {/each} - {/if} - {/each} + {:else} + + +
+ No active workers found for the group '{worker_group[0]}' +
+ + + {/if} - {/if} - {:else} - {@const worker_group = Object.entries(workerGroups ?? {}) - .filter((x) => !groupedWorkers.some((y) => y[0] == x[0])) - .find((x) => x[0] == selectedTab)} + {:else} + {@const worker_group = Object.entries(workerGroups ?? {}) + .filter((x) => !groupedWorkers.some((y) => y[0] == x[0])) + .find((x) => x[0] == selectedTab)} - {#if worker_group} - { - loadWorkerGroups() - }} - name={worker_group[0]} - config={worker_group[1]} - activeWorkers={0} - /> -
No workers currently in this worker group
+ {#if worker_group} + { + loadWorkerGroups() + }} + name={worker_group[0]} + config={worker_group[1]} + activeWorkers={0} + {width} + shouldAutoOpenDrawer={shouldAutoOpenDrawer === worker_group[0]} + onDrawerOpened={() => { + shouldAutoOpenDrawer = '' + }} + onDeleted={handleWorkerGroupDeleted} + /> +
No workers currently in this worker group
+ {/if} {/if} +
+
+ + {#if worker_group?.[2] === false} + {/if} -
-
- - {:else} -
- {#each new Array(4) as _} - - {/each} -
- {/if} + {:else} +
+ {#each new Array(4) as _} + + {/each} +
+ {/if} + {/snippet} {/if} + +{#snippet selectGroup()} +
+
Worker group + Worker groups are groups of workers that share a config and are meant to be identical. + Worker groups are meant to be used with tags. Tags can be assigned to scripts and flows and + can be seen as dedicated queues. Only the corresponding + +
+