diff --git a/CHANGELOG.md b/CHANGELOG.md index 98335f7532..de6fecc15d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,66 @@ # Changelog +## [1.538.0](https://github.com/windmill-labs/windmill/compare/v1.537.1...v1.538.0) (2025-09-01) + + +### Features + +* **aichat:** cache prompts when using anthropic models ([#6489](https://github.com/windmill-labs/windmill/issues/6489)) ([8b3ae19](https://github.com/windmill-labs/windmill/commit/8b3ae1984c745164bbf3938e6f8fc3cf02a0e2c1)) +* **frontend:** preprocessor snippet completion in code editor ([#6502](https://github.com/windmill-labs/windmill/issues/6502)) ([a5a9a33](https://github.com/windmill-labs/windmill/commit/a5a9a33d10fcac802834b1c046a3c116ab5b3454)) +* **perf:** add 300-second local cache for variable crypt retrieval ([#6483](https://github.com/windmill-labs/windmill/issues/6483)) ([8f46066](https://github.com/windmill-labs/windmill/commit/8f46066a6478a48411e00f5c4e1f9b9fdb49c1fc)) + + +### Bug Fixes + +* **backend:** http triggers early return ([#6501](https://github.com/windmill-labs/windmill/issues/6501)) ([c01ed2d](https://github.com/windmill-labs/windmill/commit/c01ed2d62a440b0ade624ed1def6bb2a54f5aa81)) +* **frontend:** capture/trigger UI nits ([#6494](https://github.com/windmill-labs/windmill/issues/6494)) ([8cd1c64](https://github.com/windmill-labs/windmill/commit/8cd1c6474b3c55d706683f83b40ef5569245b559)) +* **frontend:** Improve runs page ux ([#6485](https://github.com/windmill-labs/windmill/issues/6485)) ([768c600](https://github.com/windmill-labs/windmill/commit/768c60049413277e2764ef889274d4f8691412aa)) +* **frontend:** make resource a separate top-level type in schema editor for clarity ([0f6a742](https://github.com/windmill-labs/windmill/commit/0f6a74256dba0f571e9fcfa876e78971385ba541)) +* schema editor reactivity improvements ([#6496](https://github.com/windmill-labs/windmill/issues/6496)) ([6e46058](https://github.com/windmill-labs/windmill/commit/6e46058c5a9af6cd9fb8c1f08bb58dc123d5323e)) + +## [1.537.1](https://github.com/windmill-labs/windmill/compare/v1.537.0...v1.537.1) (2025-08-29) + + +### Bug Fixes + +* fix error handling of pre-processor steps ([641d565](https://github.com/windmill-labs/windmill/commit/641d5651c5b6fa898e4fc8ad29a8508f9400c6d8)) +* fix preprocessor not displaying immediately on addition ([e28c9df](https://github.com/windmill-labs/windmill/commit/e28c9df60f20b70a4dcef802839750bb843b91a4)) +* skipPreprocessor on re-running job immedaitely from UI ([e2b344e](https://github.com/windmill-labs/windmill/commit/e2b344ed02b02972366a011c3e873e0f43ab2843)) + +## [1.537.0](https://github.com/windmill-labs/windmill/compare/v1.536.0...v1.537.0) (2025-08-28) + + +### Features + +* **aichat:** allow reverting specific line for inline script suggestions ([#6480](https://github.com/windmill-labs/windmill/issues/6480)) ([0cc11b3](https://github.com/windmill-labs/windmill/commit/0cc11b3f31aeee60a9d4a231cea5d4285d7ab37e)) +* autovacuum or high intensity tables ([4ad0d25](https://github.com/windmill-labs/windmill/commit/4ad0d255f3eea303e97eab5325f89930b26f9e52)) + + +### Bug Fixes + +* fix okta and oauth0 sso settings ([73a3f4c](https://github.com/windmill-labs/windmill/commit/73a3f4cc73271759650e9246f4eb2e0efb7c7e37)) +* fix relative imports cache invalidation ([#6468](https://github.com/windmill-labs/windmill/issues/6468)) ([006f326](https://github.com/windmill-labs/windmill/commit/006f32602c7609b282f15135989c5f164c109c1c)) +* fix workflow as code behavior with multithread ([4973c86](https://github.com/windmill-labs/windmill/commit/4973c860f2c28d9bdc2af94530d90eb177234e5d)) + +## [1.536.0](https://github.com/windmill-labs/windmill/compare/v1.535.0...v1.536.0) (2025-08-27) + + +### Features + +* **aichat:** give advanced options tools to flow mode ([#6463](https://github.com/windmill-labs/windmill/issues/6463)) ([b26cea9](https://github.com/windmill-labs/windmill/commit/b26cea9d3e2f9a0acae335aad12206da491ac733)) +* email triggers extra args in 'to' header ([#6476](https://github.com/windmill-labs/windmill/issues/6476)) ([ceb9150](https://github.com/windmill-labs/windmill/commit/ceb9150f43a0ae9f8579f1984e791f69e7a05366)) + + +### Bug Fixes + +* **cli:** specific items for file resource type ([#6464](https://github.com/windmill-labs/windmill/issues/6464)) ([2066a2a](https://github.com/windmill-labs/windmill/commit/2066a2ada2f3139474527f373dc505b7e61d5182)) +* do not require locked for scheduled jobs ([41a8727](https://github.com/windmill-labs/windmill/commit/41a872725282ba4b78e8f9912bb1ac929b8557f7)) +* **frontend:** ai agent step nits ([#6469](https://github.com/windmill-labs/windmill/issues/6469)) ([2b03133](https://github.com/windmill-labs/windmill/commit/2b03133b2245bd42f3c64b915d72dd3f62eb65a4)) +* **frontend:** nats config conditional fields ([#6473](https://github.com/windmill-labs/windmill/issues/6473)) ([f90d444](https://github.com/windmill-labs/windmill/commit/f90d44469e0e4b462a5fa5160b64e99eef95c317)) +* **go:** could not read Username for 'xyz': terminal prompts disabled ([#6478](https://github.com/windmill-labs/windmill/issues/6478)) ([5808840](https://github.com/windmill-labs/windmill/commit/5808840b78e94a0b39614f37161f9def347a5352)) +* **go:** exec: "git": executable file not found ([#6475](https://github.com/windmill-labs/windmill/issues/6475)) ([475f405](https://github.com/windmill-labs/windmill/commit/475f405d0626f1c22309ee6a1b630472a89dbb30)) +* save changes made in diff mode ([#6477](https://github.com/windmill-labs/windmill/issues/6477)) ([d9ca181](https://github.com/windmill-labs/windmill/commit/d9ca181b1d8d26c175ec2a05409c45daeab887a4)) + ## [1.535.0](https://github.com/windmill-labs/windmill/compare/v1.534.1...v1.535.0) (2025-08-25) diff --git a/backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json b/backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json new file mode 100644 index 0000000000..51a1ec68cb --- /dev/null +++ b/backend/.sqlx/query-124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n w.premium,\n COALESCE(cw.is_past_due, false) as \"is_past_due!\",\n cw.max_tolerated_executions\n FROM\n workspace w\n LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id\n WHERE\n w.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "premium", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_past_due!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "max_tolerated_executions", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null, + true + ] + }, + "hash": "124e67b0cee1baa6295846db4ad6242a39dd40186f1dbb48ad3018bd9f6913ec" +} diff --git a/backend/.sqlx/query-434d8dfbc25cf7e92de51d763d3a2904ccc2e95ecc3d90b43a6394a7bb4d26ab.json b/backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json similarity index 69% rename from backend/.sqlx/query-434d8dfbc25cf7e92de51d763d3a2904ccc2e95ecc3d90b43a6394a7bb4d26ab.json rename to backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json index 0c657d6911..d643e4b8f8 100644 --- a/backend/.sqlx/query-434d8dfbc25cf7e92de51d763d3a2904ccc2e95ecc3d90b43a6394a7bb4d26ab.json +++ b/backend/.sqlx/query-2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, flow_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, flow_version) WHERE flow_version IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "434d8dfbc25cf7e92de51d763d3a2904ccc2e95ecc3d90b43a6394a7bb4d26ab" + "hash": "2367e7c0f7fbafe0971a187c0909617da55251e97180babf6ac9e8068f26d73d" } diff --git a/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json b/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json new file mode 100644 index 0000000000..6973159764 --- /dev/null +++ b/backend/.sqlx/query-4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "4221d98d76f3cb32d6be581b0f63cf7578429009bee4f648e2c1bc3784fdbefc" +} diff --git a/backend/.sqlx/query-4b8132b04e454eddfe6724c6cc3a2e60c9c24decb2a6b41125247bbf741e9c25.json b/backend/.sqlx/query-4b8132b04e454eddfe6724c6cc3a2e60c9c24decb2a6b41125247bbf741e9c25.json deleted file mode 100644 index ab5b04afc6..0000000000 --- a/backend/.sqlx/query-4b8132b04e454eddfe6724c6cc3a2e60c9c24decb2a6b41125247bbf741e9c25.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT owner, premium, usage.usage as \"usage?\", workspace_settings.customer_id, workspace_settings.plan FROM workspace LEFT JOIN workspace_settings ON workspace_settings.workspace_id = $1 LEFT JOIN usage ON usage.id = $1 AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND usage.is_workspace IS true WHERE workspace.id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "premium", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "usage?", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "customer_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "plan", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - true, - true - ] - }, - "hash": "4b8132b04e454eddfe6724c6cc3a2e60c9c24decb2a6b41125247bbf741e9c25" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json b/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json new file mode 100644 index 0000000000..1d56449d18 --- /dev/null +++ b/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa" +} diff --git a/backend/.sqlx/query-e77fcf4e0d58855542605d13177df61671334418820ca942b442adfab413cbae.json b/backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json similarity index 70% rename from backend/.sqlx/query-e77fcf4e0d58855542605d13177df61671334418820ca942b442adfab413cbae.json rename to backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json index de37a2fc04..1c2f9a9f33 100644 --- a/backend/.sqlx/query-e77fcf4e0d58855542605d13177df61671334418820ca942b442adfab413cbae.json +++ b/backend/.sqlx/query-8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, script_hash, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5) \n ON CONFLICT (workspace_id, script_hash) WHERE script_hash IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "e77fcf4e0d58855542605d13177df61671334418820ca942b442adfab413cbae" + "hash": "8d119104337bf99e9aa9dcbac0a54154267a7db96cc0fb3ebaac95635e24da29" } diff --git a/backend/.sqlx/query-ac4a375035049304db7ae084b2d1789afb8250adde6f38f458b115e787ed876f.json b/backend/.sqlx/query-ac4a375035049304db7ae084b2d1789afb8250adde6f38f458b115e787ed876f.json new file mode 100644 index 0000000000..9e3e7cb863 --- /dev/null +++ b/backend/.sqlx/query-ac4a375035049304db7ae084b2d1789afb8250adde6f38f458b115e787ed876f.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n owner, \n premium, \n usage.usage as \"usage?\", \n workspace_settings.customer_id, \n workspace_settings.plan, \n COALESCE(cw.is_past_due, false) as \"is_past_due!\", \n cw.max_tolerated_executions\n FROM workspace\n LEFT JOIN workspace_settings \n ON workspace_settings.workspace_id = $1\n LEFT JOIN usage \n ON usage.id = $1\n AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND usage.is_workspace IS true\n LEFT JOIN cloud_workspace_settings cw\n ON cw.workspace_id = $1\n WHERE workspace.id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "owner", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "premium", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "usage?", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "customer_id", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "plan", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_past_due!", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "max_tolerated_executions", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + null, + true + ] + }, + "hash": "ac4a375035049304db7ae084b2d1789afb8250adde6f38f458b115e787ed876f" +} diff --git a/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json b/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json deleted file mode 100644 index 641c94c555..0000000000 --- a/backend/.sqlx/query-b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job_queue, v2_job_runtime, v2_job_status", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "b9b38d63af3670d1f11d5cbb82a8008a9479bf4b3d7231371ebf26382ecde365" -} diff --git a/backend/.sqlx/query-ce6f3e803909d55c19169c77d4111bffc0fc93032943369015971373f1f2af68.json b/backend/.sqlx/query-ce6f3e803909d55c19169c77d4111bffc0fc93032943369015971373f1f2af68.json new file mode 100644 index 0000000000..3b5c65095a --- /dev/null +++ b/backend/.sqlx/query-ce6f3e803909d55c19169c77d4111bffc0fc93032943369015971373f1f2af68.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) \n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, $4, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets \n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ce6f3e803909d55c19169c77d4111bffc0fc93032943369015971373f1f2af68" +} diff --git a/backend/.sqlx/query-d768bbc46f8a9c4289b918c88ca50aa180b29bbe931d948f6e61976f71b7cdb9.json b/backend/.sqlx/query-d768bbc46f8a9c4289b918c88ca50aa180b29bbe931d948f6e61976f71b7cdb9.json deleted file mode 100644 index 22b38a5217..0000000000 --- a/backend/.sqlx/query-d768bbc46f8a9c4289b918c88ca50aa180b29bbe931d948f6e61976f71b7cdb9.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT premium FROM workspace WHERE id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "premium", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "d768bbc46f8a9c4289b918c88ca50aa180b29bbe931d948f6e61976f71b7cdb9" -} diff --git a/backend/.sqlx/query-f04632c3a8e0d7c5b48cdd26a99bb1dc5bd12df221f82405d663b8f15f5c0c3a.json b/backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json similarity index 69% rename from backend/.sqlx/query-f04632c3a8e0d7c5b48cdd26a99bb1dc5bd12df221f82405d663b8f15f5c0c3a.json rename to backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json index aa8d76b04f..ace3edec94 100644 --- a/backend/.sqlx/query-f04632c3a8e0d7c5b48cdd26a99bb1dc5bd12df221f82405d663b8f15f5c0c3a.json +++ b/backend/.sqlx/query-f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path) DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", + "query": "INSERT INTO deployment_metadata (workspace_id, path, app_version, callback_job_ids, deployment_msg) VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, path, app_version) WHERE app_version IS NOT NULL DO UPDATE SET callback_job_ids = $4, deployment_msg = $5", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "f04632c3a8e0d7c5b48cdd26a99bb1dc5bd12df221f82405d663b8f15f5c0c3a" + "hash": "f37140fcdc721a8b199471b30c2baf124affa2eaf56c801c8dac3264c584f981" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5866575aa6..f0fe796f51 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1541072f81945fa1251f8795ef6c92c4282d74d59f88498ae7d4bf00f0ebdad9" +checksum = "d025db5d9f52cbc413b167136afb3d8aeea708c0d8884783cf6253be5e22f6f2" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1018,9 +1018,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.4" +version = "0.61.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" +checksum = "eaa31b350998e703e9826b2104dd6f63be0508666e1aba88137af060e8944047" dependencies = [ "aws-smithy-types", ] @@ -1895,10 +1895,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.34" +version = "1.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +checksum = "590f9024a68a8c40351881787f1934dc11afd69090f5edb6831464694d836ea3" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -2004,9 +2005,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.45" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -2014,9 +2015,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.44" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -2105,9 +2106,9 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.1.4" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" +checksum = "3f8e18d0dca9578507f13f9803add0df13362b02c501c1c17734f0dbb52eaf0b" dependencies = [ "unicode-segmentation", "unicode-width 0.2.1", @@ -4449,9 +4450,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" dependencies = [ "powerfmt", "serde", @@ -5130,7 +5131,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set 0.5.3", - "regex-automata 0.4.10", + "regex-automata", "regex-syntax 0.8.6", ] @@ -5141,7 +5142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" dependencies = [ "bit-set 0.8.0", - "regex-automata 0.4.10", + "regex-automata", "regex-syntax 0.8.6", ] @@ -5240,6 +5241,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e178e4fba8a2726903f6ba98a6d221e76f9c12c650d5dc0e6afdc50677b49650" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -5833,9 +5840,9 @@ dependencies = [ [[package]] name = "getopts" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cba6ae63eb948698e300f645f87c70f76630d505f23b8907cf1e193ee85048c1" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ "unicode-width 0.2.1", ] @@ -5863,7 +5870,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasi 0.14.3+wasi-0.2.4", "wasm-bindgen", ] @@ -5929,7 +5936,7 @@ dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.10", + "regex-automata", "regex-syntax 0.8.6", ] @@ -6877,9 +6884,9 @@ dependencies = [ [[package]] name = "if_chain" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" +checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" [[package]] name = "ignore" @@ -6891,7 +6898,7 @@ dependencies = [ "globset", "log", "memchr", - "regex-automata 0.4.10", + "regex-automata", "same-file", "walkdir", "winapi-util", @@ -7811,7 +7818,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53304fff6ab1e597661eee37e42ea8c47a146fca280af902bb76bff8a896e523" dependencies = [ - "nu-ansi-term 0.50.1", + "nu-ansi-term", ] [[package]] @@ -7988,11 +7995,11 @@ checksum = "204651f31b0a6a7b2128d2b92c372cd94607b210c3a6b6e542c57a8cfd4db996" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -8581,16 +8588,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nu-ansi-term" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" -dependencies = [ - "overload", - "winapi", -] - [[package]] name = "nu-ansi-term" version = "0.50.1" @@ -9293,12 +9290,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "ownedbytes" version = "0.9.0" @@ -9877,9 +9868,9 @@ dependencies = [ [[package]] name = "potential_utf" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ "zerovec", ] @@ -10253,9 +10244,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases 0.2.1", @@ -10264,7 +10255,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls 0.23.29", - "socket2 0.5.10", + "socket2 0.6.0", "thiserror 2.0.16", "tokio", "tracing", @@ -10273,9 +10264,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", "getrandom 0.3.3", @@ -10294,16 +10285,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases 0.2.1", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -10613,19 +10604,10 @@ checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.10", + "regex-automata", "regex-syntax 0.8.6", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - [[package]] name = "regex-automata" version = "0.4.10" @@ -10643,12 +10625,6 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943f41321c63ef1c92fd763bfe054d2668f7f225a5c29f0105903dc2fc04ba30" -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.7.5" @@ -13509,12 +13485,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.41" +version = "0.3.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +checksum = "8ca967379f9d8eb8058d86ed467d81d03e81acd45757e4ca341c24affbe8e8e3" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde", @@ -13524,15 +13499,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" +checksum = "a9108bb380861b07264b950ded55a44a14a4adc68b9f5efd85aafc3aa4d40a68" [[package]] name = "time-macros" -version = "0.2.22" +version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +checksum = "7182799245a7264ce590b349d90338f1c1affad93d2639aed5f8f69c090b334c" dependencies = [ "num-conv", "time-core", @@ -14155,14 +14130,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", - "nu-ansi-term 0.46.0", + "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "serde", "serde_json", "sharded-slab", @@ -14770,11 +14745,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.2+wasi-0.2.4" +version = "0.14.3+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] @@ -15129,7 +15104,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "axum", @@ -15183,7 +15158,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "argon2", @@ -15300,7 +15275,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.535.0" +version = "1.538.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15315,7 +15290,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.535.0" +version = "1.538.0" dependencies = [ "chrono", "serde", @@ -15328,7 +15303,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "axum", @@ -15347,7 +15322,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "async-recursion", @@ -15427,7 +15402,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.535.0" +version = "1.538.0" dependencies = [ "regex", "serde", @@ -15442,7 +15417,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "bytes", @@ -15466,7 +15441,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.535.0" +version = "1.538.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15478,7 +15453,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.535.0" +version = "1.538.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15487,7 +15462,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "lazy_static", @@ -15499,7 +15474,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "serde_json", @@ -15511,7 +15486,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "gosyn", @@ -15523,7 +15498,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "lazy_static", @@ -15535,7 +15510,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "serde_json", @@ -15547,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "nu-parser", @@ -15558,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15569,7 +15544,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15581,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "async-recursion", @@ -15604,7 +15579,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "lazy_static", @@ -15618,7 +15593,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15635,7 +15610,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "lazy_static", @@ -15649,7 +15624,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "lazy_static", @@ -15667,7 +15642,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15692,7 +15667,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "serde_json", @@ -15702,7 +15677,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "async-recursion", @@ -15735,7 +15710,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.535.0" +version = "1.538.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15720,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.535.0" +version = "1.538.0" dependencies = [ "anyhow", "async-recursion", @@ -16325,13 +16300,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.3", -] +checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" [[package]] name = "writeable" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7168cade28..f1335287db 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.535.0" +version = "1.538.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ ] [workspace.package] -version = "1.535.0" +version = "1.538.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 7e96e8cdec..dfa2a01850 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -15a7592ca66b93b9760d49e58b23c090ead06fe2 +d16e52d570f10dfdabb04a9061fe7ebdbf5a2aa1 \ No newline at end of file diff --git a/backend/migrations/20250826163254_runnable_notify_on_insert.down.sql b/backend/migrations/20250826163254_runnable_notify_on_insert.down.sql new file mode 100644 index 0000000000..8b16eb1081 --- /dev/null +++ b/backend/migrations/20250826163254_runnable_notify_on_insert.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TRIGGER script_insert_trigger ON script; diff --git a/backend/migrations/20250826163254_runnable_notify_on_insert.up.sql b/backend/migrations/20250826163254_runnable_notify_on_insert.up.sql new file mode 100644 index 0000000000..06ab07ff55 --- /dev/null +++ b/backend/migrations/20250826163254_runnable_notify_on_insert.up.sql @@ -0,0 +1,6 @@ +-- Add up migration script here +CREATE TRIGGER script_insert_trigger +AFTER INSERT ON script +FOR EACH ROW +WHEN (NEW.lock IS NOT NULL) +EXECUTE FUNCTION notify_runnable_version_change('script'); diff --git a/backend/migrations/20250827102435_add_team_plan_status.down.sql b/backend/migrations/20250827102435_add_team_plan_status.down.sql new file mode 100644 index 0000000000..980b70d425 --- /dev/null +++ b/backend/migrations/20250827102435_add_team_plan_status.down.sql @@ -0,0 +1,7 @@ +-- Add down migration script here +DROP FUNCTION notify_team_plan_status_change; +DROP TRIGGER notify_team_plan_status_change ON cloud_workspace_settings; + +ALTER TABLE cloud_workspace_settings + DROP COLUMN is_past_due, + DROP COLUMN max_tolerated_executions; \ No newline at end of file diff --git a/backend/migrations/20250827102435_add_team_plan_status.up.sql b/backend/migrations/20250827102435_add_team_plan_status.up.sql new file mode 100644 index 0000000000..1b2007bfe7 --- /dev/null +++ b/backend/migrations/20250827102435_add_team_plan_status.up.sql @@ -0,0 +1,18 @@ +-- Add up migration script here +ALTER TABLE cloud_workspace_settings + ADD COLUMN is_past_due BOOLEAN NOT NULL DEFAULT FALSE, + ADD COLUMN max_tolerated_executions INTEGER; + +CREATE OR REPLACE FUNCTION notify_team_plan_status_change() +RETURNS TRIGGER AS $$ +BEGIN + PERFORM pg_notify('notify_workspace_premium_change', NEW.workspace_id); -- reuse the same channel as the one used for workspace premium change => clear cache + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + + +CREATE TRIGGER team_plan_status_change_trigger +AFTER UPDATE OF is_past_due, max_tolerated_executions ON cloud_workspace_settings +FOR EACH ROW +EXECUTE FUNCTION notify_team_plan_status_change(); \ No newline at end of file diff --git a/backend/migrations/20250827152300_workspace_key_cache.down.sql b/backend/migrations/20250827152300_workspace_key_cache.down.sql new file mode 100644 index 0000000000..23f95616ff --- /dev/null +++ b/backend/migrations/20250827152300_workspace_key_cache.down.sql @@ -0,0 +1,4 @@ +-- Remove workspace key cache invalidation trigger + +DROP TRIGGER workspace_key_change_trigger ON workspace_key; +DROP FUNCTION notify_workspace_key_change(); \ No newline at end of file diff --git a/backend/migrations/20250827152300_workspace_key_cache.up.sql b/backend/migrations/20250827152300_workspace_key_cache.up.sql new file mode 100644 index 0000000000..13f374625f --- /dev/null +++ b/backend/migrations/20250827152300_workspace_key_cache.up.sql @@ -0,0 +1,19 @@ +-- Add workspace key cache invalidation trigger + +CREATE OR REPLACE FUNCTION notify_workspace_key_change() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM pg_notify('notify_workspace_key_change', OLD.workspace_id); + RETURN OLD; + ELSE + PERFORM pg_notify('notify_workspace_key_change', NEW.workspace_id); + RETURN NEW; + END IF; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER workspace_key_change_trigger +AFTER INSERT OR UPDATE OF key OR DELETE ON workspace_key +FOR EACH ROW +EXECUTE FUNCTION notify_workspace_key_change(); \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 9b80e58eef..1dce88b3dc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -854,10 +854,15 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Workspace envs change detected, invalidating workspace envs cache: {}", workspace_id); windmill_common::variables::CUSTOM_ENVS_CACHE.remove(workspace_id); }, + "notify_workspace_key_change" => { + let workspace_id = n.payload(); + tracing::info!("Workspace key change detected, invalidating workspace key cache: {}", workspace_id); + windmill_common::variables::WORKSPACE_CRYPT_CACHE.remove(workspace_id); + }, "notify_workspace_premium_change" => { let workspace_id = n.payload(); tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); - windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id); + windmill_common::workspaces::TEAM_PLAN_CACHE.remove(workspace_id); }, "notify_runnable_version_change" => { let payload = n.payload(); @@ -1268,6 +1273,7 @@ async fn listen_pg(url: &str) -> Option { "notify_global_setting_change", "notify_webhook_change", "notify_workspace_envs_change", + "notify_workspace_key_change", "notify_runnable_version_change", "notify_token_invalidation", ]; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ea79738215..c4c2265268 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -8,7 +8,7 @@ use std::{ atomic::{AtomicU16, Ordering}, Arc, Mutex, }, - time::Duration, + time::{Duration, Instant}, }; use chrono::{DateTime, NaiveDateTime, Utc}; @@ -149,7 +149,9 @@ lazy_static::lazy_static! { static ref QUEUE_RUNNING_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true"); - static ref STALE_JOB_TRESHOLD_MINUTES: Option = std::env::var("STALE_JOB_TRESHOLD_MINUTES") + //legacy typo + static ref STALE_JOB_THRESHOLD_MINUTES: Option = std::env::var("STALE_JOB_THRESHOLD_MINUTES").or_else(|_| std::env::var("STALE_JOB_THRESHOLD_MINUTES")) + .ok() .and_then(|x| x.parse::().ok()); } @@ -1530,6 +1532,20 @@ pub async fn monitor_db( } }; + // run every hour + let vacuum_queue_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(60) { + if let Some(db) = conn.as_sql() { + let instant = Instant::now(); + tracing::info!("vacuuming tables"); + if let Err(e) = vacuuming_tables(&db).await { + tracing::error!("Error vacuuming v2_job: {:?}", e); + } + tracing::info!("vacuum tables done in {}s", instant.elapsed().as_secs()); + } + } + }; + let expired_items_f = async { if server_mode && !initial_load { if let Some(db) = conn.as_sql() { @@ -1607,6 +1623,7 @@ pub async fn monitor_db( expired_items_f, zombie_jobs_f, stale_jobs_f, + vacuum_queue_f, expose_queue_metrics_f, verify_license_key_f, worker_groups_alerts_f, @@ -1619,6 +1636,13 @@ pub async fn monitor_db( ); } +async fn vacuuming_tables(db: &Pool) -> error::Result<()> { + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics") + .execute(db) + .await?; + Ok(()) +} + pub async fn expose_queue_metrics(db: &Pool) { let last_check = sqlx::query_scalar!( "SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1" @@ -1874,7 +1898,7 @@ pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> { } async fn stale_job_cancellation(db: &Pool) { - if let Some(threshold) = *STALE_JOB_TRESHOLD_MINUTES { + if let Some(threshold) = *STALE_JOB_THRESHOLD_MINUTES { let stale_jobs = sqlx::query!( "SELECT v2_job_queue.id, v2_job.tag, v2_job_queue.scheduled_for, v2_job_queue.workspace_id FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id WHERE running = false AND scheduled_for < now() - ($1 || ' minutes')::interval", threshold.to_string() diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b58c74fbee..85d7555e1b 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -4679,30 +4679,42 @@ mod job_payload { test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; } - #[sqlx::test(fixtures("base", "hello"))] async fn test_dependencies_payload(db: Pool) { initialize_tracing().await; let server = ApiServer::start(db.clone()).await; let port = server.addr.port(); - let test = || async { - let result = RunJob::from(JobPayload::Dependencies { - path: "f/system/hello".to_string(), - hash: ScriptHash(123412), - language: ScriptLang::Deno, - dedicated_worker: None, - }) - .run_until_complete(&db, port) - .await - .json_result() - .unwrap(); + let result = RunJob::from(JobPayload::Dependencies { + path: "f/system/hello".to_string(), + hash: ScriptHash(123412), + language: ScriptLang::Deno, + dedicated_worker: None, + }) + .run_until_complete(&db, port) + .await + .json_result() + .unwrap(); - assert_eq!( - result.get("status").unwrap(), - &json!("Successful lock file generation") - ); - }; - test_for_versions(VERSION_FLAGS.iter().cloned(), test).await; + assert_eq!( + result.get("status").unwrap(), + &json!("Successful lock file generation") + ); + } + + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dependencies_payload_min_1_427(db: Pool) { + *MIN_VERSION_IS_AT_LEAST_1_427.write().await = true; + test_dependencies_payload(db).await; + } + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dependencies_payload_min_1_432(db: Pool) { + *MIN_VERSION_IS_AT_LEAST_1_432.write().await = true; + test_dependencies_payload(db).await; + } + #[sqlx::test(fixtures("base", "hello"))] + async fn test_dependencies_payload_min_1_440(db: Pool) { + *MIN_VERSION_IS_AT_LEAST_1_440.write().await = true; + test_dependencies_payload(db).await; } // Just test that deploying a flow work as expected. diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 29597b13c7..2fa7ff8c9a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.535.0 + version: 1.538.0 title: Windmill API contact: @@ -2034,6 +2034,11 @@ paths: - workspace parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: skip_subscription_fetch + in: query + description: skip fetching subscription status from stripe + schema: + type: boolean responses: "200": @@ -2051,9 +2056,14 @@ paths: type: string status: type: string + is_past_due: + type: boolean + max_tolerated_executions: + type: number required: - premium - owner + - is_past_due /w/{workspace}/workspaces/threshold_alert: get: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 44a6b2986e..a41840404d 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -156,6 +156,7 @@ impl AIRequestConfig { let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL || matches!(provider, AIProvider::AzureOpenAI); let is_anthropic = matches!(provider, AIProvider::Anthropic); + let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let url = if is_azure && method != Method::GET { if base_url.ends_with("/deployments") { @@ -167,6 +168,9 @@ impl AIRequestConfig { } else { format!("{}/{}", base_url, path) } + } else if is_anthropic_sdk { + let truncated_base_url = base_url.trim_end_matches("/v1"); + format!("{}/{}", truncated_base_url, path) } else { format!("{}/{}", base_url, path) }; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index c29e4e6f02..5332bcb6c4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -3741,7 +3741,7 @@ async fn batch_rerun_handle_job( PushArgsOwned { extra: None, args }, ) .await; - if let Ok(uuid) = result { + if let Ok((uuid, _)) = result { return Ok(uuid.to_string()); } } @@ -3798,7 +3798,7 @@ pub async fn run_flow_by_path( ) .await?; - let uuid = + let (uuid, _) = run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args).await?; Ok((StatusCode::CREATED, uuid.to_string())) @@ -3812,7 +3812,7 @@ pub async fn run_flow_by_path_inner( flow_path: StripPath, run_query: RunJobQuery, args: PushArgsOwned, -) -> error::Result { +) -> error::Result<(Uuid, Option)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -3828,6 +3828,7 @@ pub async fn run_flow_by_path_inner( has_preprocessor, on_behalf_of_email, edited_by, + early_return, .. } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; @@ -3889,7 +3890,7 @@ pub async fn run_flow_by_path_inner( ) .await?; tx.commit().await?; - Ok(uuid) + Ok((uuid, early_return)) } #[cfg(not(feature = "enterprise"))] diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 571ecdfac9..b474e710db 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -592,6 +592,7 @@ pub async fn transform_json_value<'c>( job.flow_step_id.clone(), job.root_job.map(|x| x.to_string()), Some(job.scheduled_for.clone()), + None, ) .await; diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index de2533a954..f0b2c4c483 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -28,16 +28,13 @@ use axum::{ }; use hyper::StatusCode; use itertools::Itertools; +use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; use serde_json::json; use serde_json::value::RawValue; use sql_builder::prelude::*; use sqlx::{FromRow, Postgres, Transaction}; -use std::{ - collections::{hash_map::DefaultHasher, HashMap}, - hash::{Hash, Hasher}, - sync::Arc, -}; +use std::{collections::HashMap, sync::Arc}; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_worker::process_relative_imports; @@ -45,6 +42,8 @@ use windmill_worker::process_relative_imports; use windmill_common::{ assets::{clear_asset_usage, insert_asset_usage, AssetUsageKind, AssetWithAltAccessType}, error::to_anyhow, + scripts::hash_script, + utils::WarnAfterExt, worker::CLOUD_HOSTED, }; @@ -378,12 +377,6 @@ async fn get_top_hub_scripts( Ok::<_, Error>((status_code, headers, response)) } -fn hash_script(ns: &NewScript) -> i64 { - let mut dh = DefaultHasher::new(); - ns.hash(&mut dh); - dh.finish() as i64 -} - async fn create_snapshot_script( authed: ApiAuthed, Extension(user_db): Extension, @@ -1369,8 +1362,9 @@ async fn toggle_workspace_error_handler( async fn get_tokened_raw_script_by_path( Extension(user_db): Extension, Extension(db): Extension, - Path((w_id, token, path)): Path<(String, String, StripPath)>, Extension(cache): Extension>, + Path((w_id, token, path)): Path<(String, String, StripPath)>, + Query(query): Query, ) -> Result { let authed = cache .get_authed(Some(w_id.clone()), &token) @@ -1381,6 +1375,7 @@ async fn get_tokened_raw_script_by_path( Extension(user_db), Extension(db), Path((w_id, path)), + Query(query), ) .await; } @@ -1389,13 +1384,21 @@ async fn get_empty_ts_script_by_path() -> String { return String::new(); } +#[derive(Deserialize)] +struct RawScriptByPathQuery { + // used to make cache immutable with respect to importer + cache_key: Option, + // used specifically for python to cache folders on import success to avoid extra db calls on package fetch + cache_folders: Option, +} async fn raw_script_by_path( authed: ApiAuthed, Extension(user_db): Extension, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, ) -> Result { - raw_script_by_path_internal(path, user_db, db, authed, w_id, false).await + raw_script_by_path_internal(path, user_db, db, authed, w_id, false, query).await } async fn raw_script_by_path_unpinned( @@ -1403,8 +1406,9 @@ async fn raw_script_by_path_unpinned( Extension(user_db): Extension, Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, ) -> Result { - raw_script_by_path_internal(path, user_db, db, authed, w_id, true).await + raw_script_by_path_internal(path, user_db, db, authed, w_id, true, query).await } lazy_static::lazy_static! { @@ -1412,6 +1416,12 @@ lazy_static::lazy_static! { std::env::var("DEBUG_RAW_SCRIPT_ENDPOINTS").is_ok(); } +lazy_static::lazy_static! { + pub static ref RAW_SCRIPT_CACHE: Cache = Cache::new(1000); + pub static ref CACHE_FOLDERS_PATH: Cache = Cache::new(1000); + +} + async fn raw_script_by_path_internal( path: StripPath, user_db: UserDB, @@ -1419,9 +1429,20 @@ async fn raw_script_by_path_internal( authed: ApiAuthed, w_id: String, unpin: bool, + query: RawScriptByPathQuery, ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("scripts:read:{}", path))?; + let cache_path = query + .cache_key + .map(|x| format!("{w_id}:{path}:{x}{}", if unpin { ":unpinned" } else { "" })); + if let Some(cache_path) = cache_path.clone() { + let cached_content = RAW_SCRIPT_CACHE.get(&cache_path); + if let Some(cached_content) = cached_content { + return Ok(cached_content); + } + } + if !path.ends_with(".py") && !path.ends_with(".ts") && !path.ends_with(".go") @@ -1439,6 +1460,27 @@ async fn raw_script_by_path_internal( .trim_end_matches(".ts") .trim_end_matches(".go") .trim_end_matches(".sh"); + + // folder cache is only useful for python given it needs to recuse over all intermediate folders to find the package. + // When a script exists in a folder, we can cache the fact that the folder exists to avoid extra db calls. + let mut split_path = path.split("/").collect::>(); + let folder_path = if query.cache_folders.is_some() && split_path.len() > 2 { + Some(format!("{w_id}:{path}/")) + } else { + None + }; + + let has_folder_cache = folder_path.is_some(); + if let Some(cache_folders) = folder_path { + let cached_content = CACHE_FOLDERS_PATH.get(&cache_folders); + if let Some(cached_ts) = cached_content { + if cached_ts >= chrono::Utc::now().timestamp() - 300 { + // 5 minutes + return Ok("WINDMILL_IS_FOLDER".to_string()); + } + } + } + let mut tx = user_db.begin(&authed).await?; let content_o = sqlx::query_scalar!( @@ -1447,6 +1489,7 @@ async fn raw_script_by_path_internal( w_id ) .fetch_optional(&mut *tx) + .warn_after_seconds(5) .await?; tx.commit().await?; @@ -1457,6 +1500,7 @@ async fn raw_script_by_path_internal( w_id ) .fetch_one(&db) + .warn_after_seconds(5) .await? .unwrap_or(false); @@ -1490,11 +1534,24 @@ async fn raw_script_by_path_internal( let content = not_found_if_none(content_o, "Script", path)?; - if unpin { - return Ok(remove_pinned_imports(&content)?); + let content = if unpin { + remove_pinned_imports(&content)? } else { - return Ok(content); + content + }; + + if has_folder_cache { + while split_path.len() >= 2 { + split_path.pop(); + let npath = split_path.join("/"); + CACHE_FOLDERS_PATH.insert(format!("{w_id}:{npath}/"), chrono::Utc::now().timestamp()); + } } + + if let Some(cache_path) = cache_path { + RAW_SCRIPT_CACHE.insert(cache_path, content.clone()); + } + Ok(content) } async fn exists_script_by_path( diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs index 723a966bca..1c2bdc029c 100644 --- a/backend/windmill-api/src/trigger_helpers.rs +++ b/backend/windmill-api/src/trigger_helpers.rs @@ -472,12 +472,12 @@ async fn trigger_runnable_inner( error_handler_path: Option<&str>, error_handler_args: Option<&sqlx::types::Json>>>, trigger_path: String, -) -> Result<(Uuid, Option)> { +) -> Result<(Uuid, Option, Option)> { let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); - let (uuid, delete_after_use) = if is_flow { + let (uuid, delete_after_use, early_return) = if is_flow { let run_query = RunJobQuery::default(); let path = StripPath(runnable_path.to_string()); - let uuid = run_flow_by_path_inner( + let (uuid, early_return) = run_flow_by_path_inner( authed, db.clone(), user_db, @@ -487,9 +487,9 @@ async fn trigger_runnable_inner( args, ) .await?; - (uuid, None) + (uuid, None, early_return) } else { - trigger_script_internal( + let (uuid, delete_after_use) = trigger_script_internal( db, user_db, authed, @@ -501,10 +501,11 @@ async fn trigger_runnable_inner( error_handler_args, trigger_path, ) - .await? + .await?; + (uuid, delete_after_use, None) }; - Ok((uuid, delete_after_use)) + Ok((uuid, delete_after_use, early_return)) } #[allow(dead_code)] @@ -521,7 +522,7 @@ pub async fn trigger_runnable( error_handler_args: Option<&sqlx::types::Json>>>, trigger_path: String, ) -> Result { - let (uuid, _) = trigger_runnable_inner( + let (uuid, _, _) = trigger_runnable_inner( db, user_db, authed, @@ -553,7 +554,7 @@ pub async fn trigger_runnable_and_wait_for_result( trigger_path: String, ) -> Result { let username = authed.username.clone(); - let (uuid, delete_after_use) = trigger_runnable_inner( + let (uuid, delete_after_use, early_return) = trigger_runnable_inner( db, user_db, authed, @@ -568,7 +569,8 @@ pub async fn trigger_runnable_and_wait_for_result( ) .await?; let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), None, &username).await?; + run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) + .await?; if delete_after_use.unwrap_or(false) { delete_job_metadata_after_use(&db, uuid).await?; @@ -592,7 +594,7 @@ pub async fn trigger_runnable_and_wait_for_raw_result( trigger_path: String, ) -> Result> { let username = authed.username.clone(); - let (uuid, delete_after_use) = trigger_runnable_inner( + let (uuid, delete_after_use, early_return) = trigger_runnable_inner( db, user_db, authed, @@ -607,23 +609,6 @@ pub async fn trigger_runnable_and_wait_for_raw_result( ) .await?; - let early_return = if is_flow { - sqlx::query_scalar!( - r#"SELECT flow_version.value->>'early_return' as early_return - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 and flow.workspace_id = $2"#, - runnable_path, - workspace_id, - ) - .fetch_optional(db) - .await? - .flatten() - } else { - None - }; - let (result, success) = run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) .await diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index 919396b7ec..f7476d9f26 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -24,7 +24,11 @@ use serde_json::Value; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ - db::UserDB, error::{Error, JsonResult, Result}, utils::{not_found_if_none, paginate, Pagination, StripPath}, variables::{ + db::UserDB, + error::{Error, JsonResult, Result}, + scripts::ScriptHash, + utils::{not_found_if_none, paginate, Pagination, StripPath, WarnAfterExt}, + variables::{ build_crypt, get_reserved_variables, ContextualVariable, CreateVariable, ListableVariable, }, worker::CLOUD_HOSTED, @@ -74,6 +78,7 @@ async fn list_contextual_variables( Some("c".to_string()), Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), Some(chrono::offset::Utc::now()), + Some(ScriptHash(1234567890)), ) .await .to_vec(), @@ -693,6 +698,7 @@ pub async fn get_value_internal<'c>( LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id ) .fetch_optional(&mut *tx) + .warn_after_seconds(5) .await?; let variable = if let Some(variable) = variable_o { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 78052b8ed8..fb0ea74f26 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -35,7 +35,7 @@ use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::{build_crypt, decrypt, encrypt}; +use windmill_common::variables::{build_crypt, decrypt, encrypt, WORKSPACE_CRYPT_CACHE}; use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; #[cfg(feature = "enterprise")] use windmill_common::workspaces::GitRepositorySettings; @@ -415,7 +415,9 @@ async fn is_premium( ) -> JsonResult { require_admin(authed.is_admin, &authed.username)?; #[cfg(feature = "cloud")] - let premium = windmill_common::workspaces::is_premium_workspace(&_db, &_w_id).await; + let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) + .await + .premium; #[cfg(not(feature = "cloud"))] let premium = false; Ok(Json(premium)) @@ -1926,6 +1928,8 @@ async fn set_encryption_key( .execute(&db) .await?; + WORKSPACE_CRYPT_CACHE.remove(w_id.as_str()); + if !request.skip_reencrypt.unwrap_or(false) { let new_encryption_key = build_crypt(&db, w_id.as_str()).await?; @@ -1948,7 +1952,13 @@ async fn set_encryption_key( if !variable.is_secret { continue; } - let decrypted_value = decrypt(&previous_encryption_key, variable.value)?; + let decrypted_value = + decrypt(&previous_encryption_key, variable.value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })?; let new_encrypted_value = encrypt(&new_encryption_key, decrypted_value.as_str()); sqlx::query!( "UPDATE variable SET value = $1 WHERE workspace_id = $2 AND path = $3", diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 4b529873a4..ba5c6c006e 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -654,6 +654,12 @@ pub async fn get_logs_from_store( return None; } +lazy_static::lazy_static! { + static ref TAGS_ARE_SENSITIVE: bool = std::env::var("TAGS_ARE_SENSITIVE").map( + |v| v.parse().unwrap() + ).unwrap_or(false); +} + pub async fn check_tag_available_for_workspace_internal( db: &DB, w_id: &str, @@ -691,10 +697,14 @@ pub async fn check_tag_available_for_workspace_internal( ))); } - return Err(error::Error::BadRequest(format!( + if *TAGS_ARE_SENSITIVE { + return Err(Error::BadRequest(format!("{tag} is not available to you"))); + } else { + return Err(error::Error::BadRequest(format!( "Only super admins are allowed to use tags that are not included in the allowed CUSTOM_TAGS: {:?}", custom_tags_per_w ))); + } } return Ok(()); diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 7633e60e25..c2a753e1a9 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -649,3 +649,9 @@ pub struct HubScript { pub schema: Box, pub summary: Option, } + +pub fn hash_script(ns: &NewScript) -> i64 { + let mut dh = std::hash::DefaultHasher::new(); + ns.hash(&mut dh); + dh.finish() as i64 +} diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 6486819e61..771678bb2f 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -265,14 +265,18 @@ pub fn create_directory_sync(directory_path: &str) { .expect("could not create dir"); } +#[track_caller] pub fn not_found_if_none>(opt: Option, kind: &str, name: U) -> Result { if let Some(o) = opt { Ok(o) } else { + let loc = Location::caller(); Err(Error::NotFound(format!( - "{} not found at name {}", + "{} not found at name {} ({}:{})", kind, - name.as_ref() + name.as_ref(), + loc.file().split("/").last().unwrap_or_default(), + loc.line() ))) } } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1dfa1438d8..f43292755e 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -7,6 +7,8 @@ */ use crate::error; +use crate::scripts::ScriptHash; +use crate::utils::WarnAfterExt; use crate::worker::Connection; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; @@ -77,13 +79,38 @@ pub struct CreateVariable { } pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result { - let key = get_workspace_key(w_id, db).await?; - let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", key, salt) + // Check cache first (300-second staleness) + let cached_key_o = WORKSPACE_CRYPT_CACHE.get(w_id).and_then(|(ts, key)| { + if ts > chrono::Utc::now().timestamp() - 300 { + Some(key) + } else { + None + } + }); + + let crypt = if let Some(cached_key) = cached_key_o { + cached_key } else { - key + let key = get_workspace_key(w_id, db).await?; + tracing::info!( + "crypt for workspace {} with key {} expired, refetching", + w_id, + key + ); + let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { + format!("{}{}", key, salt) + } else { + key + }; + let ncrypt = magic_crypt::new_magic_crypt!(crypt_key, 256); + WORKSPACE_CRYPT_CACHE.insert( + w_id.to_string(), + (chrono::Utc::now().timestamp(), ncrypt.clone()), + ); + ncrypt }; - Ok(magic_crypt::new_magic_crypt!(crypt_key, 256)) + + Ok(crypt) } pub async fn build_crypt_with_key_suffix( @@ -106,8 +133,10 @@ pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result)> = Cache::new(100); + pub static ref WORKSPACE_CRYPT_CACHE: Cache = Cache::new(1000); + } pub async fn get_reserved_variables( @@ -181,6 +212,7 @@ pub async fn get_reserved_variables( step_id: Option, root_flow_id: Option, scheduled_for: Option>, + runnable_id: Option, ) -> Vec { let state_path = { let trigger = if schedule_path.is_some() { @@ -336,7 +368,13 @@ pub async fn get_reserved_variables( ContextualVariable { name: "WM_WORKER_GROUP".to_string(), value: WORKER_GROUP.clone(), - description: "name of the worker group the job is running on".to_string(), + description: "Name of the worker group the job is running on".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_RUNNABLE_ID".to_string(), + value: runnable_id.map(|x| x.to_string()).unwrap_or_else(|| "".to_string()), + description: "Hash of the script. Useful as cache key for cache that should be runnable specific.".to_string(), is_custom: false, }, ].into_iter().chain(custom_envs.into_iter().map(|(name, value)| ContextualVariable { diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5d7d3d9967..c120e3c39a 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -86,22 +86,47 @@ impl Default for GitSyncSettings { } } +#[derive(Clone)] +pub struct TeamPlanStatus { + pub premium: bool, + pub is_past_due: bool, + pub max_tolerated_executions: Option, +} + lazy_static::lazy_static! { - pub static ref IS_PREMIUM_CACHE: Cache = Cache::new(5000); + pub static ref TEAM_PLAN_CACHE: Cache = Cache::new(5000); } #[cfg(feature = "cloud")] -pub async fn is_premium_workspace(_db: &crate::DB, _w_id: &str) -> bool { - let cached = IS_PREMIUM_CACHE.get(_w_id); +pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> TeamPlanStatus { + let cached = TEAM_PLAN_CACHE.get(_w_id); if let Some(cached) = cached { return cached; } - let premium = sqlx::query_scalar!("SELECT premium FROM workspace WHERE id = $1", _w_id) - .fetch_one(_db) - .await - .unwrap_or(false); - IS_PREMIUM_CACHE.insert(_w_id.to_string(), premium); - premium + let team_plan_info = sqlx::query_as!( + TeamPlanStatus, + r#" + SELECT + w.premium, + COALESCE(cw.is_past_due, false) as "is_past_due!", + cw.max_tolerated_executions + FROM + workspace w + LEFT JOIN cloud_workspace_settings cw ON cw.workspace_id = w.id + WHERE + w.id = $1 + "#, + _w_id + ) + .fetch_one(_db) + .await + .unwrap_or_else(|_| TeamPlanStatus { + premium: false, + is_past_due: false, + max_tolerated_executions: None, + }); + TEAM_PLAN_CACHE.insert(_w_id.to_string(), team_plan_info.clone()); + team_plan_info } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 09b2a39695..3667239119 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1419,8 +1419,9 @@ fn apply_completed_job_cloud_usage( let email2 = email.clone(); tokio::task::spawn(async move { let additional_usage = _duration / 1000; - let premium_workspace = - windmill_common::workspaces::is_premium_workspace(&db, &w_id).await; + let premium_workspace = windmill_common::workspaces::get_team_plan_status(&db, &w_id) + .await + .premium; tokio::time::timeout(std::time::Duration::from_secs(10), async move { let _ = sqlx::query!( "INSERT INTO usage (id, is_workspace, month_, usage) @@ -3636,8 +3637,8 @@ pub async fn push<'c, 'd>( ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - let premium_workspace = - windmill_common::workspaces::is_premium_workspace(_db, workspace_id).await; + let team_plan_status = + windmill_common::workspaces::get_team_plan_status(_db, workspace_id).await; // we track only non flow steps let (workspace_usage, user_usage) = if !matches!( job_payload, @@ -3655,7 +3656,7 @@ pub async fn push<'c, 'd>( .await .map_err(|e| Error::internal_err(format!("updating usage: {e:#}")))?; - let user_usage = if !premium_workspace { + let user_usage = if !team_plan_status.premium { Some(sqlx::query_scalar!( "INSERT INTO usage (id, is_workspace, month_, usage) VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) @@ -3678,7 +3679,7 @@ pub async fn push<'c, 'd>( Ok((None, None)) }?; - if !premium_workspace { + if !team_plan_status.premium || team_plan_status.is_past_due { let is_super_admin = sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email) .fetch_optional(_db) @@ -3686,7 +3687,8 @@ pub async fn push<'c, 'd>( .unwrap_or(false); if !is_super_admin { - if email != ERROR_HANDLER_USER_EMAIL + if !team_plan_status.premium + && email != ERROR_HANDLER_USER_EMAIL && email != SCHEDULE_ERROR_HANDLER_USER_EMAIL && email != SCHEDULE_RECOVERY_HANDLER_USER_EMAIL && email != "worker@windmill.dev" @@ -3765,43 +3767,53 @@ pub async fn push<'c, 'd>( .flatten() .unwrap_or(1) }; + if team_plan_status.premium { + // team plan is premium but past due, we check if the workspace has exceeded the max tolerated executions + if team_plan_status.max_tolerated_executions.is_none() + || workspace_usage > team_plan_status.max_tolerated_executions.unwrap() + { + return Err(error::Error::QuotaExceeded(format!( + "Workspace {workspace_id} team plan is past due and isn't allowed to run any more jobs. Please fix your payment method in the workspace settings." + ))); + } + } else { + if workspace_usage > MAX_FREE_EXECS + && !matches!(job_payload, JobPayload::Dependencies { .. }) + && !matches!(job_payload, JobPayload::FlowDependencies { .. }) + && !matches!(job_payload, JobPayload::AppDependencies { .. }) + { + return Err(error::Error::QuotaExceeded(format!( + "Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces." + ))); + } - if workspace_usage > MAX_FREE_EXECS - && !matches!(job_payload, JobPayload::Dependencies { .. }) - && !matches!(job_payload, JobPayload::FlowDependencies { .. }) - && !matches!(job_payload, JobPayload::AppDependencies { .. }) - { - return Err(error::Error::QuotaExceeded(format!( - "Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces." - ))); - } + let in_queue_workspace = sqlx::query_scalar!( + "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", + workspace_id + ) + .fetch_one(_db) + .await? + .unwrap_or(0); - let in_queue_workspace = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", - workspace_id - ) - .fetch_one(_db) - .await? - .unwrap_or(0); + if in_queue_workspace > MAX_FREE_EXECS as i64 { + return Err(error::Error::QuotaExceeded(format!( + "Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces." + ))); + } - if in_queue_workspace > MAX_FREE_EXECS as i64 { - return Err(error::Error::QuotaExceeded(format!( - "Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces." - ))); - } - - let concurrent_runs_workspace = sqlx::query_scalar!( + let concurrent_runs_workspace = sqlx::query_scalar!( "SELECT COUNT(id) FROM v2_job_queue WHERE running = true AND workspace_id = $1", - workspace_id - ) - .fetch_one(_db) - .await? - .unwrap_or(0); + workspace_id + ) + .fetch_one(_db) + .await? + .unwrap_or(0); - if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS as i64 { - return Err(error::Error::QuotaExceeded(format!( - "Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces." - ))); + if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS as i64 { + return Err(error::Error::QuotaExceeded(format!( + "Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces." + ))); + } } } } diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 7f07b6f65e..ae3653881a 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -156,7 +156,7 @@ pub async fn push_scheduled_job<'c>( &mut *tx, &schedule.workspace_id, &schedule.script_path, - true, + false, ) .await?; diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index 3d40c93b42..646ec64029 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -2,6 +2,7 @@ import sys import os from importlib.abc import MetaPathFinder, Loader from importlib.machinery import ModuleSpec, SourceFileLoader +import urllib.response class WindmillLoader(Loader): @@ -27,7 +28,15 @@ class WindmillFinder(MetaPathFinder): if l <= 2: return ModuleSpec(name, WindmillLoader(name)) elif l > 2: + script_path = "/".join(splitted) + folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1]) + fullpath = folder + "/" + splitted[-1] + ".py" + + if os.path.exists(fullpath): + return ModuleSpec(name, SourceFileLoader(name, fullpath)) + + import urllib.parse import urllib.request @@ -35,20 +44,29 @@ class WindmillFinder(MetaPathFinder): "Authorization": f"Bearer {os.environ.get('WM_TOKEN')}", "User-Agent": "windmill/beta" } - url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py" + + query_params = "?cache_folders=true" + runnable_id = os.environ.get('WM_RUNNABLE_ID') + if runnable_id: + query_params += f"&cache_key={runnable_id}" + url = f"{os.environ.get('BASE_INTERNAL_URL')}/api/w/{os.environ.get('WM_WORKSPACE')}/scripts/raw/p/{script_path}.py{query_params}" req = urllib.request.Request(url, None, headers) try: with urllib.request.urlopen(req) as response: - r = response.read().decode("utf-8") - folder = os.getcwd() + "/tmp/" + "/".join(splitted[:-1]) - fullpath = folder + "/" + splitted[-1] + ".py" - os.makedirs(folder, exist_ok=True) - with open(fullpath, "w+") as f: - f.write(r) - return ModuleSpec(name, SourceFileLoader(name, fullpath)) - except: - # raise ImportError(f"Script {script_path} not found") + os.makedirs(folder, exist_ok=True) + r = response.read().decode("utf-8") + if r == "WINDMILL_IS_FOLDER": + return ModuleSpec(name, WindmillLoader(name)) + with open(fullpath, "w+") as f: + f.write(r) + return ModuleSpec(name, SourceFileLoader(name, fullpath)) + except urllib.error.HTTPError as e: + if e.code != 404: + print(f"Error fetching script {script_path}: HTTP {e.code} - {e.reason}") + return ModuleSpec(name, WindmillLoader(name)) + except Exception as e: + print(f"Error fetching script {script_path}: {e}") return ModuleSpec(name, WindmillLoader(name)) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 9cad2ebba7..d0ad991bf2 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1589,6 +1589,7 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 2b7523ecad..1df44c34df 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -464,6 +464,7 @@ pub async fn get_reserved_variables( job.flow_step_id.clone(), job.flow_innermost_root_job.clone().map(|x| x.to_string()), Some(job.scheduled_for.clone()), + job.runnable_id, ) .await .to_vec(); @@ -680,11 +681,12 @@ pub async fn resolve_job_timeout( let mut warn_msg: Option = None; #[cfg(feature = "cloud")] let cloud_premium_workspace = *CLOUD_HOSTED - && windmill_common::workspaces::is_premium_workspace( + && windmill_common::workspaces::get_team_plan_status( _conn.as_sql().expect("cloud cannot use http connection"), _w_id, ) - .await; + .await + .premium; #[cfg(not(feature = "cloud"))] let cloud_premium_workspace = false; diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 4cc6649525..607e6375d2 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -544,6 +544,7 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 8fe187b8dd..201da5b959 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -571,6 +571,8 @@ pub async fn install_go_dependencies( child_cmd .current_dir(job_dir) .env_clear() + .env("HOME", HOME_ENV.as_str()) + .env("PATH", PATH_ENV.as_str()) .env("GOPATH", { #[cfg(unix)] { @@ -584,6 +586,18 @@ pub async fn install_go_dependencies( .args(vec!["mod", mod_command]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + if let Some(ref goprivate) = *GOPRIVATE { + child_cmd.env("GOPRIVATE", goprivate); + } + + // TODO: Remove if no incidents reported + if !std::env::var("WMDEBUG_NO_GOPROXY_ON_TIDY").ok().is_some() { + if let Some(ref goproxy) = *GOPROXY { + child_cmd.env("GOPROXY", goproxy); + } + } + // If annotation used we want to call tidy with special flag to pin go to 1.22 // The reason for this that at some point we had to jump from go 1.22 to 1.25 and this addds backward compatibility. if anns.go1_22_compat && mod_command == "tidy" { diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index d5bf2add3b..8b0b2119b5 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -520,7 +520,7 @@ function get_from_env(name) {{ .map(|a| { format!("let {a} = get_from_env(\"{a}\");\n",) }) .join(""), if expr.contains("error") && transform_context.contains(&"previous_result".to_string()) { - "let error = previous_result.error" + "let error = previous_result.error;" } else { "" }, diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 3972de49e7..ace218c207 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -63,7 +63,9 @@ mod worker_lockfiles; mod worker_utils; pub use worker::*; -pub use worker_lockfiles::process_relative_imports; +pub use worker_lockfiles::{ + process_relative_imports, trigger_dependents_to_recompute_dependencies, +}; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index eb6a6e7f30..70f980052b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -2145,6 +2145,7 @@ pub async fn start_worker( None, None, None, + None, ) .await .to_vec(); @@ -2264,6 +2265,7 @@ for line in sys.stdin: None, None, None, + None, ) .await; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 37c6e3fc70..f4874f6456 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1027,7 +1027,7 @@ pub async fn update_flow_status_after_job_completion_internal( .ok_or_else(|| Error::internal_err(format!("requiring flow to be in the queue")))?; tx.commit().await?; - if matches!(module_step, Step::PreprocessorStep) { + if matches!(module_step, Step::PreprocessorStep) && success { let tag_and_concurrency_key = get_tag_and_concurrency(&flow, db).await; let require_args = tag_and_concurrency_key.as_ref().is_some_and(|x| { x.tag.as_ref().is_some_and(|t| t.contains("$args")) @@ -1144,9 +1144,7 @@ pub async fn update_flow_status_after_job_completion_internal( "error while updating args in preprocessing step: {e:#}" )) })?; - if success { - return Ok(UpdateFlowStatusAfterJobCompletion::PreprocessingStep); - } + return Ok(UpdateFlowStatusAfterJobCompletion::PreprocessingStep); } let job_root = flow_job diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 38e3edec4d..358e93e0a1 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -17,7 +17,7 @@ use windmill_common::error::Result; use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; -use windmill_common::scripts::ScriptHash; +use windmill_common::scripts::{hash_script, NewScript, ScriptHash}; #[cfg(feature = "python")] use windmill_common::worker::PythonAnnotations; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection}; @@ -38,6 +38,11 @@ use windmill_parser_py_imports::parse_relative_imports; use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PushIsolationLevel}; +lazy_static::lazy_static! { + // TODO: To be removed in future versions + static ref WMDEBUG_NO_HASH_CHANGE_ON_DJ: bool = std::env::var("WMDEBUG_NO_HASH_CHANGE_ON_DJ").is_ok(); +} + use crate::common::OccupancyMetrics; use crate::csharp_executor::generate_nuget_lockfile; @@ -251,6 +256,7 @@ pub async fn handle_dependency_job( .is_some_and(|y| y.to_string().as_str() == "true") }) .unwrap_or(false); + let npm_mode = if job .script_lang .as_ref() @@ -338,30 +344,140 @@ pub async fn handle_dependency_job( )); } - let hash = job.runnable_id.unwrap_or(ScriptHash(0)); + let current_hash = job.runnable_id.unwrap_or(ScriptHash(0)); let w_id = &job.workspace_id; - sqlx::query!( - "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", - &content, - &hash.0, - w_id - ) - .execute(db) - .await?; - - // `lock` has been updated; invalidate the cache. - cache::script::invalidate(hash); - let (deployment_message, parent_path) = get_deployment_msg_and_parent_path_from_args(job.args.clone()); + let script_info = sqlx::query_as::<_, windmill_common::scripts::Script>( + "SELECT * FROM script WHERE hash = $1 AND workspace_id = $2", + ) + .bind(¤t_hash.0) + .bind(w_id) + .fetch_one(db) + .await?; + + // DependencyJob can be triggered only from 2 places: + // 1. create_script function in windmill-api/src/scripts.rs + // 2. trigger_dependents_to_recompute_dependencies (in this file) + // + // First will **always** produce script with null in `lock` + // where Second will **always** do with lock being not null + let deployed_hash = if script_info.lock.is_some() && !*WMDEBUG_NO_HASH_CHANGE_ON_DJ { + let mut tx = db.begin().await?; + // This entire section exists to solve following problem: + // + // 2 workers, one script that depend on another in python + // run the original script on both workers + // you update the dependenecy of a relative import, + // run it again until you ran it on both, normally it should fail on one of those + // + // It happens because every worker has cached their own script versions. + // However usual dependency job does not update hash of the script (and cache is keyed by the hash). + // This logical branch will create new script which will update the hash and automatically invalidate cache. + // + // IMPORTANT: This will **only** be triggered by another DependencyJob. It will never be triggered by script (re)deployement + + let ns = NewScript { + path: script_info.path, + parent_hash: Some(current_hash), + summary: script_info.summary, + description: script_info.description, + content: script_info.content, + schema: script_info.schema, + is_template: Some(script_info.is_template), + // TODO: Make it either None everywhere (particularely when raw reqs are calculated) + // Or handle this case and conditionally make Some (only with raw reqs) + lock: None, + language: script_info.language, + kind: Some(script_info.kind), + tag: script_info.tag, + draft_only: script_info.draft_only, + envs: script_info.envs, + concurrent_limit: script_info.concurrent_limit, + concurrency_time_window_s: script_info.concurrency_time_window_s, + cache_ttl: script_info.cache_ttl, + dedicated_worker: script_info.dedicated_worker, + ws_error_handler_muted: script_info.ws_error_handler_muted, + priority: script_info.priority, + timeout: script_info.timeout, + delete_after_use: script_info.delete_after_use, + restart_unless_cancelled: script_info.restart_unless_cancelled, + deployment_message: deployment_message.clone(), + concurrency_key: script_info.concurrency_key, + visible_to_runner_only: script_info.visible_to_runner_only, + no_main_func: script_info.no_main_func, + codebase: script_info.codebase, + has_preprocessor: script_info.has_preprocessor, + on_behalf_of_email: script_info.on_behalf_of_email, + assets: script_info.assets, + }; + + let new_hash = hash_script(&ns); + + sqlx::query!(" + INSERT INTO script + (workspace_id, hash, path, parent_hashes, summary, description, content, \ + created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ + codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets) + + SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ + content, created_by, schema, is_template, extra_perms, $4, language, kind, tag, \ + draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ + dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, \ + codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets + + FROM script WHERE hash = $2 AND workspace_id = $3; + ", + new_hash, current_hash.0, w_id, &content).execute(db).await?; + + // Archive current + sqlx::query!( + "UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2", + current_hash.0, + w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + ScriptHash(new_hash) + } else { + // We do not create new row for this update + // That means we can keep current hash and just update lock + sqlx::query!( + "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", + &content, + ¤t_hash.0, + w_id + ) + .execute(db) + .await?; + + // `lock` has been updated; invalidate the cache. + // Since only worker that ran this Dependency Job has the cache + // we do not need to think about invalidating cache for other workers. + cache::script::invalidate(current_hash); + + if *WMDEBUG_NO_HASH_CHANGE_ON_DJ { + tracing::warn!("WMDEBUG_NO_HASH_CHANGE_ON_DJ usually should not be used. Behavior might be unstable. Please contact Windmill Team for support.") + } + + current_hash + }; + if let Err(e) = handle_deployment_metadata( &job.permissioned_as_email, &job.created_by, &db, &w_id, DeployedObject::Script { - hash, + hash: deployed_hash, path: script_path.to_string(), parent_path: parent_path.clone(), }, @@ -502,7 +618,7 @@ pub async fn process_relative_imports( Ok(()) } -async fn trigger_dependents_to_recompute_dependencies( +pub async fn trigger_dependents_to_recompute_dependencies( w_id: &str, script_path: &str, deployment_message: Option, @@ -535,6 +651,10 @@ async fn trigger_dependents_to_recompute_dependencies( args.insert("deployment_message".to_string(), to_raw_value(&dm)); } if let Some(ref p_path) = parent_path { + // NOTE: + // it's not used but maybe one day it will be useful. allows more back-compatibility for the workers when we need it + // also very useful for debugging/observability + // it adds that information to the job args so you can see from the runs page args.insert("common_dependency_path".to_string(), to_raw_value(&p_path)); } @@ -542,6 +662,7 @@ async fn trigger_dependents_to_recompute_dependencies( "already_visited".to_string(), to_raw_value(&already_visited), ); + let kind = s.importer_kind.clone().unwrap_or_default(); let job_payload = if kind == "script" { let r = get_latest_deployed_hash_for_path(db, w_id, s.importer_path.as_str()).await; @@ -577,7 +698,7 @@ async fn trigger_dependents_to_recompute_dependencies( Ok(Some(version)) => JobPayload::FlowDependencies { path: s.importer_path.clone(), dedicated_worker: None, - version: version, + version, }, Ok(None) => { tracing::error!( diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 4da318d37b..d654e5af65 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -308,7 +308,7 @@ pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: tokio::task::spawn( (async move { tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); - if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status") + if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status, job_perms") .execute(&db2) .await { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index aa2dbfd3d3..ba5aff4e4e 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.535.0"; +export const VERSION = "v1.538.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/resource/resource.ts b/cli/src/commands/resource/resource.ts index 3c62665842..4a62a28b70 100644 --- a/cli/src/commands/resource/resource.ts +++ b/cli/src/commands/resource/resource.ts @@ -11,6 +11,8 @@ import { colors, Command, log, SEP, Table } from "../../../deps.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { Resource } from "../../../gen/types.gen.ts"; import { readInlinePathSync } from "../../utils/utils.ts"; +import { isBranchSpecificFile } from "../../core/specific_items.ts"; +import { getCurrentGitBranch } from "../../utils/git.ts"; export interface ResourceFile { value: any; @@ -23,7 +25,8 @@ export async function pushResource( workspace: string, remotePath: string, resource: ResourceFile | Resource | undefined, - localResource: ResourceFile + localResource: ResourceFile, + originalLocalPath?: string ): Promise { remotePath = removeType(remotePath, "resource"); try { @@ -35,21 +38,49 @@ export async function pushResource( // flow doesn't exist } - if (localResource.value["content"]?.startsWith("!inline ")) { - const basePath = localResource.value["content"].split(" ")[1]; - localResource.value["content"] = readInlinePathSync(basePath); - } + // Helper function to resolve inline content + const resolveInlineContent = async () => { + if (localResource.value["content"]?.startsWith("!inline ")) { + const basePath = localResource.value["content"].split(" ")[1]; + + // If we're processing a branch-specific metadata file, read from branch-specific resource file + + let pathToRead = basePath; + + if (originalLocalPath && isBranchSpecificFile(originalLocalPath)) { + const currentBranch = getCurrentGitBranch(); + if (currentBranch) { + // Directly construct branch-specific resource file path + const resourcePathSegments = basePath.split("."); + if (resourcePathSegments.length >= 4 && resourcePathSegments[resourcePathSegments.length - 3] === "resource" && resourcePathSegments[resourcePathSegments.length - 2] === "file") { + const fileBaseParts = resourcePathSegments.slice(0, -3); + const fileExt = resourcePathSegments.slice(-3); + pathToRead = [...fileBaseParts, currentBranch, ...fileExt].join("."); + } + } + } + + localResource.value["content"] = readInlinePathSync(pathToRead); + } + }; + if (resource) { if (isSuperset(localResource, resource)) { return; } + // Only resolve inline content if we're actually updating + await resolveInlineContent(); + await wmill.updateResource({ workspace: workspace, path: remotePath.replaceAll(SEP, "/"), requestBody: { ...localResource }, }); } else { + // New resource - resolve inline content + await resolveInlineContent(); + if (localResource.is_oauth) { log.info( colors.yellow( @@ -89,7 +120,8 @@ async function push(opts: PushOptions, filePath: string, remotePath: string) { workspace.workspaceId, remotePath, undefined, - parseFromFile(filePath) + parseFromFile(filePath), + filePath // Pass the local file path for branch-specific inline content resolution ); log.info(colors.bold.underline.green(`Resource ${remotePath} pushed`)); } diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index a2240498ab..9c4171a97e 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -15,6 +15,8 @@ import { } from "../../../deps.ts"; import { deepEqual } from "../../utils/utils.ts"; import * as wmill from "../../../gen/services.gen.ts"; +import * as specificItems from "../../core/specific_items.ts"; +import { getCurrentGitBranch } from "../../utils/git.ts"; import { defaultScriptMetadata, @@ -102,12 +104,24 @@ async function push(opts: PushOptions, filePath: string) { export async function findResourceFile(path: string) { const splitPath = path.split("."); - const contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json"; - const contentBasePathYAML = splitPath[0] + "." + splitPath[1] + ".yaml"; + let contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json"; + let contentBasePathYAML = splitPath[0] + "." + splitPath[1] + ".yaml"; + + // Check for branch-specific metadata files first + const currentBranch = getCurrentGitBranch(); + + const candidates = [contentBasePathJSON, contentBasePathYAML]; + + if (currentBranch) { + // Add branch-specific candidates at the beginning (higher priority) + const branchSpecificJSON = specificItems.toBranchSpecificPath(contentBasePathJSON, currentBranch); + const branchSpecificYAML = specificItems.toBranchSpecificPath(contentBasePathYAML, currentBranch); + candidates.unshift(branchSpecificJSON, branchSpecificYAML); + } const validCandidates = ( await Promise.all( - [contentBasePathJSON, contentBasePathYAML].map((x) => { + candidates.map((x) => { return Deno.stat(x) .catch(() => undefined) .then((x) => x?.isFile) @@ -580,7 +594,7 @@ export function filePathExtensionFromContentType( return ".java"; } else if (language === "ruby") { return ".rb"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -611,7 +625,7 @@ export const exts = [ ".playbook.yml", ".java", ".rb" - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG ]; export function removeExtensionToPath(path: string): string { diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index f7c4833273..a7b2fd4660 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -1930,11 +1930,21 @@ export async function push( await Deno.readTextFile(resourceFilePath) ); + // For branch-specific resources, push to the base path on the workspace server + // This ensures branch-specific files are stored with their base names in the workspace + let serverPath = resourceFilePath; + const currentBranch = getCurrentGitBranch(); + + if (currentBranch && isBranchSpecificFile(resourceFilePath)) { + serverPath = fromBranchSpecificPath(resourceFilePath, currentBranch); + } + await pushResource( workspace.workspaceId, - resourceFilePath, + serverPath, undefined, - newObj + newObj, + resourceFilePath ); if (stateTarget) { await Deno.writeTextFile(stateTarget, change.after); @@ -1945,6 +1955,12 @@ export async function push( const oldObj = parseFromPath(change.path, change.before); const newObj = parseFromPath(change.path, change.after); + // Check if this is a branch-specific item and get the original branch-specific path + let originalBranchSpecificPath: string | undefined; + if (specificItems && isSpecificItem(change.path, specificItems)) { + originalBranchSpecificPath = getBranchSpecificPath(change.path, specificItems); + } + await pushObj( workspace.workspaceId, change.path, @@ -1952,7 +1968,8 @@ export async function push( newObj, opts.plainSecrets ?? false, alreadySynced, - opts.message + opts.message, + originalBranchSpecificPath ); if (stateTarget) { @@ -1986,6 +2003,17 @@ export async function push( ); } const obj = parseFromPath(change.path, change.content); + + // Determine the actual local file path for this change + // For branch-specific items, we read from branch-specific files but push to base server paths + let localFilePath = change.path; + if (specificItems && isSpecificItem(change.path, specificItems)) { + const branchSpecificPath = getBranchSpecificPath(change.path, specificItems); + if (branchSpecificPath) { + localFilePath = branchSpecificPath; + } + } + await pushObj( workspace.workspaceId, change.path, @@ -1993,7 +2021,8 @@ export async function push( obj, opts.plainSecrets ?? false, [], - opts.message + opts.message, + localFilePath // Pass the actual local file path ); if (stateTarget) { diff --git a/cli/src/commands/trigger/trigger.ts b/cli/src/commands/trigger/trigger.ts index c9cd0e013d..7e3afd9288 100644 --- a/cli/src/commands/trigger/trigger.ts +++ b/cli/src/commands/trigger/trigger.ts @@ -15,7 +15,10 @@ import { isSuperset, parseFromFile, removeType, + TRIGGER_TYPES, } from "../../types.ts"; +import { fromBranchSpecificPath, isBranchSpecificFile } from "../../core/specific_items.ts"; +import { getCurrentGitBranch } from "../../utils/git.ts"; import { requireLogin } from "../../core/auth.ts"; import { validatePath, resolveWorkspace } from "../../core/context.ts"; @@ -222,25 +225,29 @@ async function list(opts: GlobalOptions) { } function checkIfValidTrigger(kind: string | undefined): kind is TriggerType { - if ( - kind && - [ - "http", - "websocket", - "kafka", - "nats", - "postgres", - "mqtt", - "sqs", - "gcp", - ].includes(kind) - ) { + if (kind && (TRIGGER_TYPES as readonly string[]).includes(kind)) { return true; } else { return false; } } +function extractTriggerKindFromPath(filePath: string): string | undefined { + let pathToAnalyze = filePath; + + // If this is a branch-specific file, convert it to the base path first + if (isBranchSpecificFile(filePath)) { + const currentBranch = getCurrentGitBranch(); + if (currentBranch) { + pathToAnalyze = fromBranchSpecificPath(filePath, currentBranch); + } + } + + // Now extract trigger type from the base path: "something.kafka_trigger.yaml" -> "kafka" + const triggerMatch = pathToAnalyze.match(/\.(\w+)_trigger\.yaml$/); + return triggerMatch ? triggerMatch[1] : undefined; +} + async function push(opts: GlobalOptions, filePath: string, remotePath: string) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); @@ -256,7 +263,7 @@ async function push(opts: GlobalOptions, filePath: string, remotePath: string) { console.log(colors.bold.yellow("Pushing trigger...")); - const triggerKind = filePath.split(".")[1].split("_")[0]; + const triggerKind = extractTriggerKindFromPath(filePath); if (!checkIfValidTrigger(triggerKind)) { throw new Error("Invalid trigger kind: " + triggerKind); } diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 5f5c352862..b64d48b915 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -44,6 +44,7 @@ export interface SyncOptions { commonSpecificItems?: { variables?: string[]; resources?: string[]; + triggers?: string[]; }; } & { [branchName: string]: SyncOptions & { @@ -54,6 +55,7 @@ export interface SyncOptions { specificItems?: { variables?: string[]; resources?: string[]; + triggers?: string[]; }; }; }; @@ -62,6 +64,7 @@ export interface SyncOptions { commonSpecificItems?: { variables?: string[]; resources?: string[]; + triggers?: string[]; }; } & { [branchName: string]: SyncOptions & { @@ -72,6 +75,7 @@ export interface SyncOptions { specificItems?: { variables?: string[]; resources?: string[]; + triggers?: string[]; }; }; }; diff --git a/cli/src/core/context.ts b/cli/src/core/context.ts index 3fcfdc6510..92a6276aa3 100644 --- a/cli/src/core/context.ts +++ b/cli/src/core/context.ts @@ -109,15 +109,11 @@ async function tryResolveWorkspace( return { isError: false, value: e }; } - const defaultWorkspace = await getActiveWorkspace(opts); - if (!defaultWorkspace) { - return { - isError: true, - error: colors.red.underline("No workspace given and no default set."), - }; - } - - return { isError: false, value: defaultWorkspace }; + // Only check for explicit workspace, don't fallback to active workspace here + return { + isError: true, + error: colors.red.underline("No explicit workspace given."), + }; } async function tryResolveBranchWorkspace( @@ -259,6 +255,9 @@ async function tryResolveBranchWorkspace( export async function resolveWorkspace( opts: GlobalOptions ): Promise { + const cache = (opts as any).__secret_workspace; + if (cache) return cache; + if (opts.baseUrl) { if (opts.workspace && opts.token) { let normalizedBaseUrl: string; @@ -328,20 +327,28 @@ export async function resolveWorkspace( } } - // Try explicit workspace flag first (should override branch-based resolution) + // Try explicit workspace flag first (highest priority) const res = await tryResolveWorkspace(opts); if (!res.isError) { return res.value; } - // Fall back to branch-based resolution if no explicit workspace + // Try branch-based resolution (medium priority) const branchWorkspace = await tryResolveBranchWorkspace(opts); if (branchWorkspace) { + (opts as any).__secret_workspace = branchWorkspace; return branchWorkspace; } - // If both failed, show the original error from explicit workspace resolution - log.info(colors.red.bold(res.error)); + // Fall back to active workspace (lowest priority) + const activeWorkspace = await getActiveWorkspace(opts); + if (activeWorkspace) { + (opts as any).__secret_workspace = activeWorkspace; + return activeWorkspace; + } + + // If everything failed, show error + log.info(colors.red.bold("No workspace given and no default set.")); return Deno.exit(-1); } diff --git a/cli/src/core/specific_items.ts b/cli/src/core/specific_items.ts index ac0696ccf0..2e9b5052f0 100644 --- a/cli/src/core/specific_items.ts +++ b/cli/src/core/specific_items.ts @@ -1,10 +1,59 @@ import { minimatch } from "../../deps.ts"; import { getCurrentGitBranch, isGitRepository } from "../utils/git.ts"; +import { isFileResource } from "../utils/utils.ts"; import { SyncOptions } from "./conf.ts"; +import { TRIGGER_TYPES } from "../types.ts"; export interface SpecificItemsConfig { variables?: string[]; resources?: string[]; + triggers?: string[]; +} + +// Define all branch-specific file types (computed lazily) +function getBranchSpecificTypes() { + return { + variable: '.variable.yaml', + resource: '.resource.yaml', + // Generate trigger patterns from the list + ...Object.fromEntries( + TRIGGER_TYPES.map(t => [`${t}_trigger`, `.${t}_trigger.yaml`]) + ) + } as const; +} + +/** + * Check if a path ends with any trigger type + */ +function isTriggerFile(path: string): boolean { + return TRIGGER_TYPES.some(type => path.endsWith(`.${type}_trigger.yaml`)); +} + +/** + * Extract the file type suffix from a path + */ +function getFileTypeSuffix(path: string): string | null { + for (const [_, suffix] of Object.entries(getBranchSpecificTypes())) { + if (path.endsWith(suffix)) { + return suffix; + } + } + + const resourceFileMatch = path.match(/(\\.resource\\.file\\..+)$/); + if (resourceFileMatch) { + return resourceFileMatch[1]; + } + + return null; +} + +/** + * Build regex pattern for all supported yaml file types + */ +function buildYamlTypePattern(): string { + const basicTypes = ['variable', 'resource']; + const triggerTypes = TRIGGER_TYPES.map(t => `${t}_trigger`); + return `((${basicTypes.join('|')})|(${triggerTypes.join('|')}))`; } /** @@ -39,6 +88,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions): SpecificI if (commonItems?.resources) { merged.resources = [...commonItems.resources]; } + if (commonItems?.triggers) { + merged.triggers = [...commonItems.triggers]; + } // Add branch-specific items (extending common items) if (branchItems?.variables) { @@ -47,6 +99,9 @@ export function getSpecificItemsForCurrentBranch(config: SyncOptions): SpecificI if (branchItems?.resources) { merged.resources = [...(merged.resources || []), ...branchItems.resources]; } + if (branchItems?.triggers) { + merged.triggers = [...(merged.triggers || []), ...branchItems.triggers]; + } return merged; } @@ -75,6 +130,21 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig return specificItems.resources ? matchesPatterns(path, specificItems.resources) : false; } + // Check for any trigger type + if (isTriggerFile(path)) { + return specificItems.triggers ? matchesPatterns(path, specificItems.triggers) : false; + } + + // Check for resource files using the standard detection function + if (isFileResource(path)) { + // Extract the base path without the file extension to match against patterns + const basePathMatch = path.match(/^(.+?)\.resource\.file\./); + if (basePathMatch && specificItems.resources) { + const basePath = basePathMatch[1] + '.resource.yaml'; + return matchesPatterns(basePath, specificItems.resources); + } + } + return false; } @@ -82,14 +152,24 @@ export function isSpecificItem(path: string, specificItems: SpecificItemsConfig * Convert a base path to a branch-specific path */ export function toBranchSpecificPath(basePath: string, branchName: string): string { - // Extract the extension (e.g., ".variable.yaml" or ".resource.yaml") - const extensionMatch = basePath.match(/(\.(variable|resource)\.yaml)$/); - if (!extensionMatch) { - return basePath; // Return unchanged if no recognized extension - } + // Check for resource file pattern (e.g., .resource.file.ini) + const resourceFileMatch = basePath.match(/^(.+?)(\.resource\.file\..+)$/); - const extension = extensionMatch[1]; - const pathWithoutExtension = basePath.substring(0, basePath.length - extension.length); + let extension: string; + let pathWithoutExtension: string; + + if (resourceFileMatch) { + // Handle resource files + extension = resourceFileMatch[2]; + pathWithoutExtension = resourceFileMatch[1]; + } else { + const suffix = getFileTypeSuffix(basePath); + if (!suffix) { + return basePath; + } + extension = suffix; + pathWithoutExtension = basePath.substring(0, basePath.length - extension.length); + } // Sanitize branch name to be filesystem-safe const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_'); @@ -108,17 +188,29 @@ export function toBranchSpecificPath(basePath: string, branchName: string): stri export function fromBranchSpecificPath(branchSpecificPath: string, branchName: string): string { // Sanitize branch name the same way as in toBranchSpecificPath const sanitizedBranchName = branchName.replace(/[\/\\:*?"<>|.]/g, '_'); - - // Pattern: path.sanitizedBranchName.extension const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const pattern = new RegExp(`\\.${escapedBranchName}(\\.(variable|resource)\\.yaml)$`); - const match = branchSpecificPath.match(pattern); - if (!match) { + // Check for resource file pattern first + const resourceFilePattern = new RegExp(`\\.${escapedBranchName}(\\.resource\\.file\\..+)$`); + const resourceFileMatch = branchSpecificPath.match(resourceFilePattern); + + if (resourceFileMatch) { + const extension = resourceFileMatch[1]; + const pathWithoutBranchAndExtension = branchSpecificPath.substring( + 0, + branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length + ); + return `${pathWithoutBranchAndExtension}${extension}`; + } + + const yamlPattern = new RegExp(`\\.${escapedBranchName}(\\.${buildYamlTypePattern()}\\.yaml)$`); + const yamlMatch = branchSpecificPath.match(yamlPattern); + + if (!yamlMatch) { return branchSpecificPath; // Return unchanged if not a branch-specific path } - const extension = match[1]; + const extension = yamlMatch[1]; const pathWithoutBranchAndExtension = branchSpecificPath.substring( 0, branchSpecificPath.length - `.${sanitizedBranchName}${extension}`.length @@ -166,10 +258,14 @@ export function isCurrentBranchFile(path: string): boolean { return false; } + // Sanitize branch name to match what would be used in file naming + const sanitizedBranchName = currentBranch.replace(/[\/\\:*?"<>|.]/g, '_'); + const escapedBranchName = sanitizedBranchName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // Use cached pattern or create and cache new one let pattern = branchPatternCache.get(currentBranch); if (!pattern) { - pattern = new RegExp(`\\.${currentBranch}\\.(variable|resource)\\.yaml$`); + pattern = new RegExp(`\\.${escapedBranchName}\\.${buildYamlTypePattern()}\\.yaml$|\\.${escapedBranchName}\\.resource\\.file\\..+$`); branchPatternCache.set(currentBranch, pattern); } @@ -181,6 +277,6 @@ export function isCurrentBranchFile(path: string): boolean { * Used to identify and skip files from other branches during sync operations */ export function isBranchSpecificFile(path: string): boolean { - // Pattern: *.branchName.variable.yaml or *.branchName.resource.yaml - return /\.[^.]+\.(variable|resource)\.yaml$/.test(path); + const yamlTypePattern = buildYamlTypePattern(); + return new RegExp(`\\.[^.]+\\.${yamlTypePattern}\\.yaml$|\\.[^.]+\\.resource\\.file\\..+$`).test(path); } diff --git a/cli/src/main.ts b/cli/src/main.ts index ef8c27d7c1..630a895259 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.535.0"; +export const VERSION = "1.538.0"; const command = new Command() .name("wmill") diff --git a/cli/src/types.ts b/cli/src/types.ts index 57c1c9b8e7..123f132662 100644 --- a/cli/src/types.ts +++ b/cli/src/types.ts @@ -45,6 +45,17 @@ export interface DifferenceChange { export type Difference = DifferenceCreate | DifferenceRemove | DifferenceChange; +export const TRIGGER_TYPES = [ + 'http', + 'websocket', + 'kafka', + 'nats', + 'postgres', + 'mqtt', + 'sqs', + 'gcp' +] as const; + export type GlobalOptions = { baseUrl: string | undefined; workspace: string | undefined; @@ -111,6 +122,17 @@ export function showConflict(path: string, local: string, remote: string) { log.info("\n"); } +/** + * Pushes an object to the workspace server based on its type + * @param workspace - The workspace ID to push to + * @param p - The server path (base path for branch-specific items) + * @param befObj - The previous object state (for updates) + * @param newObj - The new object state to push + * @param plainSecrets - Whether to store secrets in plain text + * @param alreadySynced - Array to track already synced items + * @param message - Optional commit/update message + * @param originalLocalPath - The original local file path (used for branch-specific resource file resolution) + */ export async function pushObj( workspace: string, p: string, @@ -118,7 +140,8 @@ export async function pushObj( newObj: any, plainSecrets: boolean, alreadySynced: string[], - message?: string + message?: string, + originalLocalPath?: string ) { const typeEnding = getTypeStrFromPath(p); @@ -135,7 +158,7 @@ export async function pushObj( } else if (typeEnding === "resource") { if (!alreadySynced.includes(p)) { alreadySynced.push(p); - await pushResource(workspace, p, befObj, newObj); + await pushResource(workspace, p, befObj, newObj, originalLocalPath || p); } } else if (typeEnding === "resource-type") { await pushResourceType(workspace, p, befObj, newObj); diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index 96bdfcebb0..cb8b808725 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -136,10 +136,12 @@ export function sleep(ms: number) { export function isFileResource(path: string): boolean { const splitPath = path.split("."); + + // Check for pattern: *.resource.file.* (handles both base and branch-specific) return ( splitPath.length >= 4 && - splitPath[1] == "resource" && - splitPath[2] == "file" + splitPath[splitPath.length - 3] == "resource" && + splitPath[splitPath.length - 2] == "file" ); } diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 28b8ba6c03..91bdc296d8 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,15 +1,16 @@ { "name": "windmill-components", - "version": "1.535.0", + "version": "1.538.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.535.0", + "version": "1.538.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { + "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", "@codingame/monaco-vscode-editor-api": "~20.2.1", @@ -25,7 +26,7 @@ "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", - "@windmill-labs/svelte-dnd-action": "^0.9.48", + "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", "ag-charts-community": "^9.0.1", @@ -48,7 +49,7 @@ "highlight.js": "^11.8.0", "idb": "^8.0.2", "lru-cache": "^11.1.0", - "lucide-svelte": "^0.399.0", + "lucide-svelte": "^0.540.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~20.2.1", "monaco-editor-wrapper": "6.12.0", @@ -56,7 +57,7 @@ "monaco-languageclient": "9.11.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.87.1", + "openai": "^5.16.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -186,6 +187,15 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.60.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.60.0.tgz", + "integrity": "sha512-9zu/TXaUy8BZhXedDtt1wT3H4LOlpKDO1/ftiFpeR3N1PCr3KJFKkxxlQWWt1NNp08xSwUNJ3JNY8yhl8av6eQ==", + "license": "MIT", + "bin": { + "anthropic-ai-sdk": "bin/cli" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.6.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz", @@ -3640,20 +3650,12 @@ "version": "20.19.10", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.10.tgz", "integrity": "sha512-iAFpG6DokED3roLSP0K+ybeDdIX6Bc0Vd3mLW5uDqThPWtNos3E+EqOM11mPQHKzfWHqEBuLjIlsBQQ8CsISmQ==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/normalize-package-data": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", @@ -3898,6 +3900,7 @@ "version": "0.9.48", "resolved": "https://registry.npmjs.org/@windmill-labs/svelte-dnd-action/-/svelte-dnd-action-0.9.48.tgz", "integrity": "sha512-A6pWayH3nOi79DZohTscGj5t2PuNtHlKM5WIAj9WlVDx5pS1A+MsyfuNZi8oCpj2590wFmQl+58G4y4xZqDa1Q==", + "license": "MIT", "peerDependencies": { "svelte": ">=3.23.0 || ^5.0.0-next.0" } @@ -3953,17 +3956,6 @@ "svelte": "^3.57.0 || ^4.0.0 || ^5.0.0" } }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, "node_modules/abstract-leveldown": { "version": "6.2.3", "resolved": "https://registry.npmjs.org/abstract-leveldown/-/abstract-leveldown-6.2.3.tgz", @@ -4028,17 +4020,6 @@ "ag-grid-community": "31.3.4" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -4210,7 +4191,8 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dev": true }, "node_modules/autoprefixer": { "version": "10.4.21", @@ -4802,6 +4784,7 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -5426,6 +5409,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dev": true, "engines": { "node": ">=0.4.0" } @@ -5742,6 +5726,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", @@ -6171,14 +6156,6 @@ "node": ">=0.10.0" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/eventemitter3": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-2.0.3.tgz", @@ -6405,6 +6382,7 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.2.tgz", "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -6415,23 +6393,6 @@ "node": ">= 6" } }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/fraction.js": { "version": "4.3.7", "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", @@ -7129,14 +7090,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/idb": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", @@ -7895,9 +7848,10 @@ "optional": true }, "node_modules/lucide-svelte": { - "version": "0.399.0", - "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.399.0.tgz", - "integrity": "sha512-NQ8AxNMKbIJsx7HV//gnAsIY1wJfb3rbXSK2S/ZDjIldvAEdzGngpUT8T8Q8zHYUuii0bavAmVARN8giR4vvpA==", + "version": "0.540.0", + "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.540.0.tgz", + "integrity": "sha512-jedJgKrrsT1B6mAMHGMmk6dIZKhorLq4KO7v+j7hYfeGR0LRe3W+SvwpWF1E6WT2/AY9IAm8KT/G+Nzf+0lW3g==", + "license": "ISC", "peerDependencies": { "svelte": "^3 || ^4 || ^5.0.0-next.42" } @@ -8784,6 +8738,7 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dev": true, "engines": { "node": ">= 0.6" } @@ -8792,6 +8747,7 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, "dependencies": { "mime-db": "1.52.0" }, @@ -9141,44 +9097,6 @@ "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", "optional": true }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/node-fetch-native": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.6.tgz", @@ -9363,18 +9281,10 @@ } }, "node_modules/openai": { - "version": "4.100.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.100.0.tgz", - "integrity": "sha512-9soq/wukv3utxcuD7TWFqKdKp0INWdeyhUCvxwrne5KwnxaCp4eHL4GdT/tMFhYolxgNhxFzg5GFwM331Z5CZg==", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.16.0.tgz", + "integrity": "sha512-hoEH8ZNvg1HXjU9mp88L/ZH8O082Z8r6FHCXGiWAzVRrEv443aI57qhch4snu07yQydj+AUAWLenAiBXhu89Tw==", + "license": "Apache-2.0", "bin": { "openai": "bin/cli" }, @@ -9391,19 +9301,6 @@ } } }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.101", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.101.tgz", - "integrity": "sha512-Ykg7fcE3+cOQlLUv2Ds3zil6DVjriGQaSN/kEpl5HQ3DIGM6W0F2n9+GkWV4bRt7KjLymgzNdTnSKCbFUUJ7Kw==", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" - }, "node_modules/openapi-types": { "version": "12.1.3", "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", @@ -12381,11 +12278,6 @@ "node": ">=6" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -12521,6 +12413,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, "license": "MIT" }, "node_modules/unified": { @@ -12956,33 +12849,11 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "engines": { - "node": ">= 14" - } - }, "node_modules/web-worker": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", "integrity": "sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==" }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/wheel": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wheel/-/wheel-1.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index e13e9c41fb..dec955bee1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.535.0", + "version": "1.538.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -77,6 +77,7 @@ }, "type": "module", "dependencies": { + "@anthropic-ai/sdk": "^0.60.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~20.2.1", "@codingame/monaco-vscode-editor-api": "~20.2.1", @@ -92,7 +93,7 @@ "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", - "@windmill-labs/svelte-dnd-action": "^0.9.48", + "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", "ag-charts-community": "^9.0.1", @@ -115,7 +116,7 @@ "highlight.js": "^11.8.0", "idb": "^8.0.2", "lru-cache": "^11.1.0", - "lucide-svelte": "^0.399.0", + "lucide-svelte": "^0.540.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~20.2.1", "monaco-editor-wrapper": "6.12.0", @@ -123,7 +124,7 @@ "monaco-languageclient": "9.11.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.87.1", + "openai": "^5.16.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -544,4 +545,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} \ No newline at end of file +} diff --git a/frontend/src/global.d.ts b/frontend/src/global.d.ts index 3cf0168db4..bd06974068 100644 --- a/frontend/src/global.d.ts +++ b/frontend/src/global.d.ts @@ -1,7 +1,7 @@ /// -declare type Item = import('svelte-dnd-action').Item -declare type DndEvent = import('svelte-dnd-action').DndEvent +declare type Item = import('@windmill-labs/svelte-dnd-action').Item +declare type DndEvent = import('@windmill-labs/svelte-dnd-action').DndEvent declare namespace svelte.JSX { interface HTMLAttributes { onconsider?: (event: CustomEvent> & { target: EventTarget & T }) => void diff --git a/frontend/src/lib/components/ArgInfo.svelte b/frontend/src/lib/components/ArgInfo.svelte index feadd419cd..be042b95e5 100644 --- a/frontend/src/lib/components/ArgInfo.svelte +++ b/frontend/src/lib/components/ArgInfo.svelte @@ -95,14 +95,14 @@
{#if JSON.stringify(value).length > 120} {/if} -
+
diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index a15c461c12..7897832188 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -937,11 +937,21 @@ selected={oneOfSelected} on:selected={({ detail }) => { oneOfSelected = detail - const prevValueKeys = Object.keys( + const selectedObjProperties = oneOf?.find((o) => o.title == detail)?.properties ?? {} - ) + const newValueKeys = Object.keys(selectedObjProperties) const toKeep = {} - for (const key of prevValueKeys) { + for (const key of newValueKeys) { + // Check if there is a select (enum) in the newly selected oneOf and if the current value is not in the enum, skip it + if ( + !['kind', 'label'].includes(key) && + selectedObjProperties[key]?.enum && + value && + value[key] !== undefined && + !selectedObjProperties[key].enum.includes(value[key]) + ) { + continue + } toKeep[key] = value[key] } const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind')) @@ -982,8 +992,11 @@ } } bind:args={value} - dndType={`nested-${title}`} - hiddenArgs={['label', 'kind']} + hiddenArgs={[ + oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + ]} on:reorder={(e) => { if (oneOf && oneOf[objIdx]) { const keys = e.detail @@ -1076,20 +1089,14 @@ {disablePortal} {disabled} {prettifyHeader} - bind:schema={ - () => ({ - properties, - $schema: '', - required: nestedRequired ?? [], - type: 'object', - order - }), - (newSchema) => { - dispatch('nestedChange') - } - } + schema={{ + properties, + $schema: '', + required: nestedRequired ?? [], + type: 'object', + order + }} bind:args={value} - dndType={`nested-${title}`} on:reorder={(e) => { const keys = e.detail order = keys diff --git a/frontend/src/lib/components/Auth0Setting.svelte b/frontend/src/lib/components/Auth0Setting.svelte index 5d8dc9ba5a..1116ea0d27 100644 --- a/frontend/src/lib/components/Auth0Setting.svelte +++ b/frontend/src/lib/components/Auth0Setting.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/ConcurrentJobsChart.svelte b/frontend/src/lib/components/ConcurrentJobsChart.svelte index 351a4caa25..c401df4be6 100644 --- a/frontend/src/lib/components/ConcurrentJobsChart.svelte +++ b/frontend/src/lib/components/ConcurrentJobsChart.svelte @@ -13,16 +13,24 @@ Tooltip } from 'chart.js' import type { CompletedJob, ExtendedJobs } from '$lib/gen' - import { createEventDispatcher } from 'svelte' import { getDbClockNow } from '$lib/forLater' import { Line } from '$lib/components/chartjs-wrappers/chartJs' - export let extendedJobs: ExtendedJobs | undefined = undefined - export let maxIsNow: boolean = false - export let minTimeSet: string | undefined = undefined - export let maxTimeSet: string | undefined = undefined + interface Props { + extendedJobs?: ExtendedJobs | undefined + maxIsNow?: boolean + minTimeSet?: string | undefined + maxTimeSet?: string | undefined + onZoom: (zoom: { min: Date; max: Date }) => void + } - const dispatch = createEventDispatcher() + let { + extendedJobs = undefined, + maxIsNow = false, + minTimeSet = undefined, + maxTimeSet = undefined, + onZoom + }: Props = $props() function calculateTimeSeries(extendedJobs: ExtendedJobs): AggregatedInterval[] { const timeline = new Map() @@ -57,7 +65,7 @@ } const s = timeline.get(startTime)! s.count += 1 - s.id_started.push('unknoww') + s.id_started.push('unknown') if (j.duration_ms != undefined) { const jc = j as CompletedJob const endTime = startTime + jc.duration_ms @@ -103,9 +111,6 @@ } type AggregatedInterval = { time: Date; count: number; msg?: string } - let intervals: AggregatedInterval[] | undefined = undefined - $: intervals = extendedJobs ? calculateTimeSeries(extendedJobs) : undefined - ChartJS.register( Title, Tooltip, @@ -118,31 +123,12 @@ TimeScale ) - $: data = { - datasets: [ - { - borderColor: '#4ade80', - backgroundColor: '#f8717100', - pointRadius: 0, - label: 'running', - showLine: true, - stepped: true, - data: - intervals?.map((job) => ({ - x: job.time as any, - y: job.count, - id: job.msg - })) ?? [] - } - ] - } - const zoomOptions = { pan: { enabled: true, modifierKey: 'ctrl' as 'ctrl', onPanComplete: ({ chart }) => { - dispatch('zoom', { + onZoom({ min: addSeconds(new Date(chart.scales.x.min), -1), max: addSeconds(new Date(chart.scales.x.max), 1) }) @@ -154,17 +140,13 @@ }, mode: 'x' as 'x', onZoom: ({ chart }) => { - dispatch('zoom', { + onZoom({ min: addSeconds(new Date(chart.scales.x.min), -1), max: addSeconds(new Date(chart.scales.x.max), 1) }) } } } - let minTime = addSeconds(new Date(), -300) - let maxTime = getDbClockNow() - - $: computeMinMaxTime(intervals, minTimeSet, maxTimeSet) function minJobTime(intervals: AggregatedInterval[]): Date { return intervals[0].time @@ -181,15 +163,13 @@ let minTimeSetDate = minTimeSet ? new Date(minTimeSet) : undefined let maxTimeSetDate = maxTimeSet ? new Date(maxTimeSet) : undefined if (minTimeSetDate && maxTimeSetDate) { - minTime = minTimeSetDate - maxTime = maxTimeSetDate - return + return { min: minTimeSetDate, max: maxTimeSetDate } } if (intervals == undefined || intervals?.length == 0) { - minTime = minTimeSetDate ?? addSeconds(new Date(), -300) - maxTime = maxTimeSetDate ?? getDbClockNow() - return + const minTime = minTimeSetDate ?? addSeconds(new Date(), -300) + const maxTime = maxTimeSetDate ?? getDbClockNow() + return { min: minTime, max: maxTime } } const maxJob = maxIsNow ? getDbClockNow() : maxJobTime(intervals) @@ -197,12 +177,11 @@ const diff = (maxJob.getTime() - minJob.getTime()) / 20000 - minTime = minTimeSetDate ?? addSeconds(minJob, -diff) - if (maxIsNow) { - maxTime = maxTimeSetDate ?? maxJob - } else { - maxTime = maxTimeSetDate ?? addSeconds(maxJob, diff) - } + const minTime = minTimeSetDate ?? addSeconds(minJob, -diff) + const maxTime = maxIsNow + ? (maxTimeSetDate ?? maxJob) + : (maxTimeSetDate ?? addSeconds(maxJob, diff)) + return { min: minTime, max: maxTime } } function addSeconds(date: Date, seconds: number): Date { @@ -210,7 +189,30 @@ return date } - $: options = { + const intervals = $derived(extendedJobs ? calculateTimeSeries(extendedJobs) : undefined) + + let data = $derived({ + datasets: [ + { + borderColor: '#4ade80', + backgroundColor: '#f8717100', + pointRadius: 0, + label: 'running', + showLine: true, + stepped: true, + data: + intervals?.map((job) => ({ + x: job.time as any, + y: job.count, + id: job.msg + })) ?? [] + } + ] + }) + + const minMaxTimes = $derived(computeMinMaxTime(intervals, minTimeSet, maxTimeSet)) + + let options = $derived({ responsive: true, maintainAspectRatio: false, plugins: { @@ -232,8 +234,8 @@ grid: { display: false }, - min: minTime, - max: maxTime + min: minMaxTimes.min, + max: minMaxTimes.max }, y: { grid: { @@ -254,7 +256,7 @@ intersect: false, mode: 'index' } - } as any + } as any)
diff --git a/frontend/src/lib/components/DiffEditor.svelte b/frontend/src/lib/components/DiffEditor.svelte index 524c7e264c..186712f214 100644 --- a/frontend/src/lib/components/DiffEditor.svelte +++ b/frontend/src/lib/components/DiffEditor.svelte @@ -32,6 +32,7 @@ defaultModified?: string readOnly?: boolean buttons?: ButtonProp[] + modifiedModel?: meditor.ITextModel } let { @@ -44,7 +45,8 @@ defaultOriginal = undefined, defaultModified = undefined, readOnly = false, - buttons = [] + buttons = [], + modifiedModel }: Props = $props() let diffEditor: meditor.IStandaloneDiffEditor | undefined = $state(undefined) @@ -72,11 +74,8 @@ lineNumbersMinChars: 2, scrollbar: { alwaysConsumeMouseWheel: false } }) - if ( - defaultOriginal !== undefined && - defaultModified !== undefined && - defaultLang !== undefined - ) { + + if (defaultLang !== undefined) { setupModel(defaultLang, defaultOriginal, defaultModified, defaultModifiedLang) } } @@ -87,16 +86,12 @@ modified?: string, modifiedLang?: string ) { + const o = meditor.createModel(original ?? '', lang) + const m = modifiedModel ?? meditor.createModel(modified ?? '', modifiedLang ?? lang) diffEditor?.setModel({ - original: meditor.createModel('', lang), - modified: meditor.createModel('', modifiedLang ?? lang) + original: o, + modified: m }) - if (original) { - setOriginal(original) - } - if (modified) { - setModified(modified) - } } export function setOriginal(code: string) { @@ -113,6 +108,15 @@ defaultModified = code } + export function setModifiedModel(model: meditor.ITextModel) { + const curr = diffEditor?.getModel() + if (!curr) return + diffEditor?.setModel({ + original: curr.original, + modified: model + }) + } + export function getModified(): string { return diffEditor?.getModel()?.modified.getValue() ?? '' } diff --git a/frontend/src/lib/components/DropdownSelect.svelte b/frontend/src/lib/components/DropdownSelect.svelte new file mode 100644 index 0000000000..7a31f3435b --- /dev/null +++ b/frontend/src/lib/components/DropdownSelect.svelte @@ -0,0 +1,38 @@ + + + + {#snippet buttonReplacement()} +
+
+ + {selectedDisplayName ?? items.find((item) => item.id === selected)?.displayName ?? ''} + + + {@render extraLabel?.()} +
+ +
+ {/snippet} + diff --git a/frontend/src/lib/components/DropdownV2Inner.svelte b/frontend/src/lib/components/DropdownV2Inner.svelte index c8236eed5f..15ec3b5ada 100644 --- a/frontend/src/lib/components/DropdownV2Inner.svelte +++ b/frontend/src/lib/components/DropdownV2Inner.svelte @@ -47,7 +47,7 @@ aiDescription={item.displayName} > {#if item.icon} - + {/if}

{item.displayName} diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index a3dfc5a168..edaa506dc4 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -14,13 +14,18 @@ import FlowPropertyEditor from './schema/FlowPropertyEditor.svelte' import PropertyEditor from './schema/PropertyEditor.svelte' import SimpleEditor from './SimpleEditor.svelte' - import { createEventDispatcher, tick, untrack } from 'svelte' + import { createEventDispatcher, untrack } from 'svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import Label from './Label.svelte' import { sendUserToast } from '$lib/toast' import Toggle from './Toggle.svelte' - import { DynamicSelect, emptyString } from '$lib/utils' + import { + DynamicSelect, + emptyString, + generateRandomString, + readFieldsRecursively + } from '$lib/utils' import Popover from './meltComponents/Popover.svelte' import SchemaFormDnd from './schema/SchemaFormDND.svelte' import { deepEqual } from 'fast-equals' @@ -48,7 +53,6 @@ isAppInput?: boolean displayWebhookWarning?: boolean onlyMaskPassword?: boolean - dndType?: string | undefined editTab: | 'inputEditor' | 'history' @@ -75,6 +79,8 @@ addProperty?: import('svelte').Snippet runButton?: import('svelte').Snippet extraTab?: import('svelte').Snippet + schemaFormClassName?: string + onChange?: (args: Record) => void } let { @@ -91,7 +97,6 @@ isAppInput = false, displayWebhookWarning = false, onlyMaskPassword = false, - dndType = undefined, editTab, previewSchema = undefined, editPanelInitialSize = undefined, @@ -110,7 +115,9 @@ openEditTab, addProperty, runButton, - extraTab + extraTab, + schemaFormClassName = undefined, + onChange = undefined }: Props = $props() $effect.pre(() => { @@ -125,6 +132,13 @@ } }) + $effect(() => { + if (onChange) { + readFieldsRecursively(args) + onChange(args ?? {}) + } + }) + $effect(() => { if (schema && dynSelectCode !== undefined && dynSelectLang !== undefined) { if (dynSelectCode && dynSelectCode.trim()) { @@ -154,9 +168,10 @@ let variableEditor: VariableEditor | undefined = $state(undefined) let keys: string[] = $state( - Array.isArray(schema?.order) + (Array.isArray(schema?.order) ? [...schema.order] : (Object.keys(schema?.properties ?? {}) ?? Object.keys(schema?.properties ?? {})) + ).filter((x) => !hiddenArgs?.includes(x)) ) function alignOrderWithProperties(schema: { @@ -187,37 +202,30 @@ return hasChanged } function onSchemaChange() { - let editSchema = false if (alignOrderWithProperties(schema)) { - console.log('alignOrderWithProperties', JSON.stringify(schema, null, 2)) - editSchema = true + // console.log('alignOrderWithProperties', JSON.stringify(schema, null, 2)) } - let lkeys = schema?.order ?? Object.keys(schema?.properties ?? {}) + let lkeys = (schema?.order ?? Object.keys(schema?.properties ?? {})).filter( + (x) => !hiddenArgs?.includes(x) + ) if (schema?.properties && !deepEqual(lkeys, keys)) { keys = [...lkeys] - editSchema = true if (opened == undefined) { opened = keys[0] } } - if (editSchema) { - schema = schema - } } let opened: string | undefined = $state(untrack(() => keys[0])) function computeSelected(property: any) { if (!opened) return '' - return property.type !== 'object' - ? property.type - : property.format === 'resource-s3_object' - ? 'S3' - : property.format?.startsWith('dynselect-') - ? 'dynselect' - : property.oneOf && property.oneOf.length >= 2 - ? 'oneOf' - : 'object' + if (property.type !== 'object') return property.type + if (property.format === 'resource-s3_object') return 'S3' + if (property.format?.startsWith('dynselect-')) return 'dynselect' + if (property.oneOf && property.oneOf.length >= 2) return 'oneOf' + if (property.format?.startsWith('resource-')) return 'resource' + return 'object' } export function openField(key: string) { @@ -248,30 +256,31 @@ // clear the input el.value = oldName } else { + let newSchema = $state.snapshot(schema) if (args) { args[newName] = args[oldName] delete args[oldName] } - schema.properties[newName] = schema.properties[oldName] - delete schema.properties[oldName] + newSchema.properties[newName] = newSchema.properties[oldName] + delete newSchema.properties[oldName] - if (schema.required?.includes(oldName)) { - schema.required = schema.required?.map((x) => (x === oldName ? newName : x)) + if (newSchema.required?.includes(oldName)) { + newSchema.required = newSchema.required?.map((x) => (x === oldName ? newName : x)) } // Replace the old name with the new name in the order array - if (schema.order) { - const index = schema.order.indexOf(oldName) + if (newSchema.order) { + const index = newSchema.order.indexOf(oldName) if (index !== -1) { - schema.order[index] = newName + newSchema.order[index] = newName } } opened = newName - schema = $state.snapshot(schema) - dispatch('change', schema) + schema = newSchema + sendUserToast('Argument renamed') } } @@ -347,10 +356,11 @@ ['Number', 'number'], ['Integer', 'integer'], ['Object', 'object'], + ['Resource', 'resource'], ['OneOf', 'oneOf'], ['Array', 'array'], ['Boolean', 'boolean'], - ['S3 Object', 'S3'] + ['S3', 'S3'] ] if (showDynSelectOpt) { typeOptions.push(['DynSelect', 'dynselect']) @@ -368,6 +378,8 @@ const code = generateFn(functionName) dynSelectCode = dynSelectCode ? dynSelectCode.concat(code) : code } + + let dndType = $state(generateRandomString())

@@ -404,15 +416,15 @@ class="min-h-0 overflow-y-auto grow rounded-md {runButton ? 'flex flex-col gap-2' : ''}" > (previewSchema ? previewSchema : schema), (newSchema) => { schema = newSchema - tick().then(() => dispatch('change', schema)) } } - {dndType} + {hiddenArgs} {disableDnd} {onlyMaskPassword} bind:args @@ -420,11 +432,16 @@ opened = e.detail }} on:reorder={(e) => { + let order = e.detail + let newProperties = {} + for (let key of order) { + newProperties[key] = schema.properties[key] + } schema = { ...schema, + properties: newProperties, order: e.detail } - tick().then(() => dispatch('change', schema)) }} helperScript={{ type: 'inline', @@ -436,9 +453,6 @@ {diff} on:acceptChange on:rejectChange - on:nestedChange={() => { - dispatch('change', schema) - }} {shouldDispatchChanges} bind:isValid noVariablePicker={noVariablePicker || customUi?.disableVariablePicker === true} @@ -446,8 +460,8 @@ {@render runButton?.()} -
- {#if dynSelectFunctions.length > 0} + {#if dynSelectFunctions.length > 0} +
- {/if} -
+
+ {/if}
@@ -514,7 +528,7 @@ {#if jsonEnabled && customUi?.jsonOnly != true}
{#if addPropertyInEditorTab} - + {#snippet trigger()}
@@ -629,7 +647,7 @@
{#if opened === argName}
- {#if !hiddenArgs.includes(argName) && Object.keys(schema?.properties ?? {}).includes(argName)} + {#if Object.keys(schema?.properties ?? {}).includes(argName)} {#if typeof args == 'object' && schema?.properties[argName]} { - schema = $state.snapshot(schema) - dispatch('change', schema) - }} > {#snippet typeeditor()} {#if isFlowInput || isAppInput} @@ -664,6 +678,7 @@ bind:selected={ () => computeSelected(schema.properties[opened ?? '']), (v) => { + const isResource = v == 'resource' const isS3 = v == 'S3' const isOneOf = v == 'oneOf' const isDynSelect = v == 'dynselect' @@ -684,13 +699,18 @@ nullable: undefined, required: undefined } - if (isS3) { schema.properties[argName] = { ...emptyProperty, type: 'object', format: 'resource-s3_object' } + } else if (isResource) { + schema.properties[argName] = { + ...emptyProperty, + type: 'object', + format: 'resource-' + } } else if (isDynSelect) { const functionName = argName.replace(/\s+/g, '_') schema.properties[argName] = { @@ -742,13 +762,9 @@ type: v } } + schema.properties = schema.properties } } - on:selected={(e) => { - schema = schema - dispatch('change', schema) - dispatch('schemaChange') - }} > {#snippet children({ item })} {#each typeOptions as x} @@ -762,6 +778,9 @@ {#if isFlowInput || isAppInput} { + dndType = generateRandomString() + }} bind:defaultValue={schema.properties[argName].default} {variableEditor} {itemPicker} @@ -792,12 +811,6 @@ (x) => x !== argName ) } - dispatch('change', schema) - }} - on:schemaChange={(e) => { - schema = $state.snapshot(schema) - dispatch('change', schema) - dispatch('schemaChange') }} /> {/if} @@ -822,7 +835,6 @@ on:change={() => { try { schema = JSON.parse(schemaString) - dispatch('change', schema) error = '' } catch (err) { error = err.message diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index ae35b9fcfb..0bc4a4d57c 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -150,11 +150,12 @@ import { writable } from 'svelte/store' import { formatResourceTypes } from './copilot/chat/script/core' import FakeMonacoPlaceHolder from './FakeMonacoPlaceHolder.svelte' - import { editorPositionMap, readFieldsRecursively } from '$lib/utils' + import { editorPositionMap } from '$lib/utils' import { extToLang, langToExt } from '$lib/editorLangUtils' import { aiChatManager } from './copilot/chat/AIChatManager.svelte' import type { Selection } from 'monaco-editor' import { getDbSchemas } from './apps/components/display/dbtable/utils' + import { PYTHON_PREPROCESSOR_MODULE_CODE, TS_PREPROCESSOR_MODULE_CODE } from '$lib/script_helpers' // import EditorTheme from './EditorTheme.svelte' let divEl: HTMLDivElement | null = $state(null) @@ -185,6 +186,7 @@ key?: string | undefined class?: string | undefined moduleId?: string + enablePreprocessorSnippet?: boolean } let { @@ -211,7 +213,8 @@ loadAsync = false, key = undefined, class: clazz = undefined, - moduleId = undefined + moduleId = undefined, + enablePreprocessorSnippet = false }: Props = $props() $effect.pre(() => { @@ -544,9 +547,7 @@ let sqlSchemaCompletor: IDisposable | undefined = undefined - async function updateSchema() { - const newSchemaRes = lang === 'graphql' ? args?.api : args?.database - + async function updateSchema(newSchemaRes: string | undefined) { if (typeof newSchemaRes === 'string') { const resourcePath = newSchemaRes.replace('$res:', '') dbSchema = $dbSchemas[resourcePath] @@ -667,6 +668,55 @@ } } + let preprocessorCompletor: IDisposable | undefined = undefined + function addPreprocessorCompletions(lang: 'typescript' | 'python') { + if (preprocessorCompletor) { + preprocessorCompletor.dispose() + } + const preprocessorCode = + lang === 'typescript' ? TS_PREPROCESSOR_MODULE_CODE : PYTHON_PREPROCESSOR_MODULE_CODE + preprocessorCompletor = languages.registerCompletionItemProvider(lang, { + provideCompletionItems: function (model, position) { + const word = model.getWordUntilPosition(position) + + if (word.word.length >= 3 && 'preprocessor'.startsWith(word.word)) { + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn + } + return { + suggestions: [ + { + label: 'preprocessor (windmill)', + kind: languages.CompletionItemKind.Function, + insertTextRules: languages.CompletionItemInsertTextRule.InsertAsSnippet, + insertText: preprocessorCode, + range, + additionalTextEdits: [ + { + range: { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: 0, + endColumn: word.startColumn + }, + text: '' + } + ] + } + ] + } + } + + return { + suggestions: [] + } + } + }) + } + let reviewingChanges = $state(writable(false)) let aiChatEditorHandler: AIChatEditorHandler | undefined = $state(undefined) @@ -676,7 +726,21 @@ let selectedCode = $state('') export function reviewAndApplyCode(code: string, applyAll: boolean = false) { - aiChatEditorHandler?.reviewAndApply(code, applyAll) + aiChatEditorHandler?.reviewChanges(code, { applyAll, mode: 'apply' }) + } + + export function reviewAppliedCode( + originalCode: string, + opts?: { onFinishedReview?: () => void } + ) { + aiChatEditorHandler?.reviewChanges(originalCode, { + mode: 'revert', + onFinishedReview: opts?.onFinishedReview + }) + } + + export function getAiChatEditorHandler() { + return aiChatEditorHandler } function addChatHandler(editor: meditor.IStandaloneCodeEditor) { @@ -1556,6 +1620,7 @@ sqlSchemaCompletor && sqlSchemaCompletor.dispose() autocompletor && autocompletor.dispose() sqlTypeCompletor && sqlTypeCompletor.dispose() + preprocessorCompletor && preprocessorCompletor.dispose() timeoutModel && clearTimeout(timeoutModel) loadTimeout && clearTimeout(loadTimeout) aiChatEditorHandler?.clear() @@ -1607,10 +1672,20 @@ ? untrack(() => addSqlTypeCompletions()) : sqlTypeCompletor?.dispose() }) + $effect(() => { - console.log('updating schema', lang, $dbSchemas) - readFieldsRecursively(args) - lang && $dbSchemas && untrack(() => updateSchema()) + initialized && (lang === 'typescript' || lang === 'python') && enablePreprocessorSnippet + ? untrack(() => addPreprocessorCompletions(lang as 'typescript' | 'python')) + : preprocessorCompletor?.dispose() + }) + + let lastArg = undefined + $effect(() => { + let newArg = lang === 'graphql' ? args?.api : args?.database + if (newArg !== lastArg) { + lastArg = newArg + $dbSchemas && untrack(() => updateSchema(newArg)) + } }) $effect(() => { console.log('updating db schema completions', dbSchema, lang) @@ -1684,10 +1759,20 @@ {#if $reviewingChanges} { - aiChatEditorHandler?.acceptAll() + const mode = aiChatEditorHandler?.getReviewMode?.() + if (mode === 'revert') { + aiChatEditorHandler?.keepAll() + } else { + aiChatEditorHandler?.acceptAll() + } }} onRejectAll={() => { - aiChatEditorHandler?.rejectAll() + const mode = aiChatEditorHandler?.getReviewMode?.() + if (mode === 'revert') { + aiChatEditorHandler?.revertAll() + } else { + aiChatEditorHandler?.rejectAll() + } }} /> {/if} diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index a2012f73c3..493ace3201 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -50,6 +50,7 @@ AI_TOOL_MESSAGE_PREFIX, getToolCallId } from './graph/renderers/nodes/AIToolNode.svelte' + import JobAssetsViewer from './assets/JobAssetsViewer.svelte' let { flowStateStore, @@ -1048,7 +1049,11 @@ } } }) - let selected = $derived(isListJob ? 'sequence' : 'graph') as 'sequence' | 'graph' | 'logs' + let selected = $derived(isListJob ? 'sequence' : 'graph') as + | 'sequence' + | 'graph' + | 'logs' + | 'assets' let animateLogsTab = $state(false) @@ -1076,6 +1081,23 @@ return '' } + + // Set all tabs content to the same height to prevent layout jumps + let tabsHeight = $state({ + sequenceHeight: 0, + logsHeight: 0, + assetsHeight: 0, + graphHeight: 0 + }) + + let minTabHeight = $derived( + Math.max( + tabsHeight.sequenceHeight, + tabsHeight.logsHeight, + tabsHeight.assetsHeight, + tabsHeight.graphHeight + ) + ) @@ -1154,12 +1176,17 @@ : ''}>Logs Details + Assets {:else}
{/if} {/if} -
+
{#if isListJob} {@const sliceFrom = globalIterationBounds[buildSubflowKey(flowJobIds?.moduleId ?? '', prefix)] @@ -1508,7 +1535,11 @@
Empty flow
{/if}
-
+
+ {#if selected == 'assets' && render} +
+ +
+ {/if}
{#if render} {#if job.raw_flow && !isListJob} -
+
diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index 52603575a7..cd0c121a93 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -85,7 +85,7 @@ ${Object.entries(args) {#each Object.entries(args).sort((a, b) => a[0].localeCompare(b[0])) as [arg, value]} {arg} - + {/each} {:else if args} diff --git a/frontend/src/lib/components/ObjectTypeNarrowing.svelte b/frontend/src/lib/components/ObjectTypeNarrowing.svelte deleted file mode 100644 index 6c0ed70116..0000000000 --- a/frontend/src/lib/components/ObjectTypeNarrowing.svelte +++ /dev/null @@ -1,21 +0,0 @@ - - - { - resource = e.detail - format = resource != undefined ? `resource-${resource}` : undefined - dispatch('change', format) - }} - value={resource} - nonePickable={true} -/> diff --git a/frontend/src/lib/components/OktaSetting.svelte b/frontend/src/lib/components/OktaSetting.svelte index 5a1836ec7d..7ad4cb01b4 100644 --- a/frontend/src/lib/components/OktaSetting.svelte +++ b/frontend/src/lib/components/OktaSetting.svelte @@ -1,4 +1,5 @@ diff --git a/frontend/src/lib/components/Portal.svelte b/frontend/src/lib/components/Portal.svelte index 5ea9c44e95..801188de2e 100644 --- a/frontend/src/lib/components/Portal.svelte +++ b/frontend/src/lib/components/Portal.svelte @@ -1,4 +1,4 @@ - -