diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a5bf52ecc..ab3f168064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,44 @@ # Changelog +## [1.680.0](https://github.com/windmill-labs/windmill/compare/v1.679.0...v1.680.0) (2026-04-08) + + +### Features + +* add CLI workspace merge command and enhance fork with datatable/color support ([#8756](https://github.com/windmill-labs/windmill/issues/8756)) ([4342c18](https://github.com/windmill-labs/windmill/commit/4342c1854134500d3b2bc46280f9885ee84e2c9e)) +* add scheduled job deletion with configurable retention period ([#8753](https://github.com/windmill-labs/windmill/issues/8753)) ([2d18a68](https://github.com/windmill-labs/windmill/commit/2d18a680991babe317ca315bbce40e6ce733afda)) +* add status indicator dots to parallel loop iteration picker ([#8761](https://github.com/windmill-labs/windmill/issues/8761)) ([470b8aa](https://github.com/windmill-labs/windmill/commit/470b8aa5f1870e26fea022c1e2a9f48471d8a205)) + + +### Bug Fixes + +* move alert config from config table to global_settings ([#8762](https://github.com/windmill-labs/windmill/issues/8762)) ([fa66870](https://github.com/windmill-labs/windmill/commit/fa668707c0ee7f261d78e145666b1073471259fd)) +* resolve esbuild host/binary version mismatch in app sync push ([#8765](https://github.com/windmill-labs/windmill/issues/8765)) ([e36d440](https://github.com/windmill-labs/windmill/commit/e36d440a251a43ea888e3ce378d0bb8ed8f42e11)) +* skip serializing ws_specific on resources when false ([#8764](https://github.com/windmill-labs/windmill/issues/8764)) ([c69f10d](https://github.com/windmill-labs/windmill/commit/c69f10d20dd064f0c329934096c2945424ff81f2)) + +## [1.679.0](https://github.com/windmill-labs/windmill/compare/v1.678.0...v1.679.0) (2026-04-07) + + +### Features + +* Fork datatables ([#8339](https://github.com/windmill-labs/windmill/issues/8339)) ([3d4f4c6](https://github.com/windmill-labs/windmill/commit/3d4f4c6c38155396e9b2236a6a7a7ad4e02da877)) + +## [1.678.0](https://github.com/windmill-labs/windmill/compare/v1.677.0...v1.678.0) (2026-04-07) + + +### Features + +* accept any content type on webhooks/http triggers with fallback ([#8743](https://github.com/windmill-labs/windmill/issues/8743)) ([208a597](https://github.com/windmill-labs/windmill/commit/208a597d599b4d203f7ab817a5d8ce2c06f79d0a)) +* add download all logs button for flow jobs ([#8748](https://github.com/windmill-labs/windmill/issues/8748)) ([d938625](https://github.com/windmill-labs/windmill/commit/d938625785ba301fbd2c5f3d001c320eab1c504c)) + + +### Bug Fixes + +* delete raw_script_temp rows before workspace deletion to avoid FK violation ([#8752](https://github.com/windmill-labs/windmill/issues/8752)) ([8b9523e](https://github.com/windmill-labs/windmill/commit/8b9523e03c82c5a095b7cb2d5f70a87b7bbc8608)) +* Fix FlowTimeline duplicate key ([#8754](https://github.com/windmill-labs/windmill/issues/8754)) ([2413dbe](https://github.com/windmill-labs/windmill/commit/2413dbefe3cc3b65c28bea437cd4471cf7e9ecba)) +* remove span.enter() in dedicated worker to prevent tracing panic ([#8749](https://github.com/windmill-labs/windmill/issues/8749)) ([db55e8e](https://github.com/windmill-labs/windmill/commit/db55e8efb0c9ae198ca5ac7013439a94dfe9f550)) +* restore ai agent tool deletion ([#8744](https://github.com/windmill-labs/windmill/issues/8744)) ([2f7ba9e](https://github.com/windmill-labs/windmill/commit/2f7ba9edac1a57dfc0eb3417574c72292855fc56)) + ## [1.677.0](https://github.com/windmill-labs/windmill/compare/v1.676.0...v1.677.0) (2026-04-06) diff --git a/Dockerfile b/Dockerfile index d312d3a823..e11cf9cecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -162,11 +162,19 @@ ENV PATH /usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH RUN apt-get update \ - && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini \ + && apt-get install -y --no-install-recommends netbase tzdata ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common tini gnupg lsb-release \ && if echo "$features" | grep -q "ee"; then apt-get install -y --no-install-recommends libsasl2-modules-gssapi-mit krb5-user; fi \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + RUN if [ "$WITH_GIT" = "true" ]; then \ apt-get update -y \ && apt-get install -y git \ diff --git a/backend/.sqlx/query-007fa93171b244490b94464938b9f95aca4e91bccde6da93cb151799b3398049.json b/backend/.sqlx/query-007fa93171b244490b94464938b9f95aca4e91bccde6da93cb151799b3398049.json new file mode 100644 index 0000000000..24aa1f17ee --- /dev/null +++ b/backend/.sqlx/query-007fa93171b244490b94464938b9f95aca4e91bccde6da93cb151799b3398049.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "007fa93171b244490b94464938b9f95aca4e91bccde6da93cb151799b3398049" +} diff --git a/backend/.sqlx/query-035e29e775bfc5b236100135e1d94a4baf2b617b86f0c3c74ba9a00b859993f6.json b/backend/.sqlx/query-035e29e775bfc5b236100135e1d94a4baf2b617b86f0c3c74ba9a00b859993f6.json new file mode 100644 index 0000000000..9a28b7ee24 --- /dev/null +++ b/backend/.sqlx/query-035e29e775bfc5b236100135e1d94a4baf2b617b86f0c3c74ba9a00b859993f6.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO job_delete_schedule (job_id, workspace_id, delete_at) VALUES ($1, $2, now() + make_interval(secs => $3::double precision)) ON CONFLICT (job_id) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "035e29e775bfc5b236100135e1d94a4baf2b617b86f0c3c74ba9a00b859993f6" +} diff --git a/backend/.sqlx/query-0ca770234f3e38be3fb1c280d82e9c06440168806fe605e38808bdcb400d4034.json b/backend/.sqlx/query-0ca770234f3e38be3fb1c280d82e9c06440168806fe605e38808bdcb400d4034.json new file mode 100644 index 0000000000..2dcdba586b --- /dev/null +++ b/backend/.sqlx/query-0ca770234f3e38be3fb1c280d82e9c06440168806fe605e38808bdcb400d4034.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0ca770234f3e38be3fb1c280d82e9c06440168806fe605e38808bdcb400d4034" +} diff --git a/backend/.sqlx/query-1325d08f0d34a5e706543585186dac4ee099aefcd3ed51f10f3298a2c006da80.json b/backend/.sqlx/query-1325d08f0d34a5e706543585186dac4ee099aefcd3ed51f10f3298a2c006da80.json new file mode 100644 index 0000000000..80bfa41872 --- /dev/null +++ b/backend/.sqlx/query-1325d08f0d34a5e706543585186dac4ee099aefcd3ed51f10f3298a2c006da80.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM raw_script_temp WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "1325d08f0d34a5e706543585186dac4ee099aefcd3ed51f10f3298a2c006da80" +} diff --git a/backend/.sqlx/query-20ca0b26a0d2e9cfb9fb8e7066d6d5f3a154de0f76ce0d12ee678bcabd7b445c.json b/backend/.sqlx/query-20ca0b26a0d2e9cfb9fb8e7066d6d5f3a154de0f76ce0d12ee678bcabd7b445c.json new file mode 100644 index 0000000000..002bdef847 --- /dev/null +++ b/backend/.sqlx/query-20ca0b26a0d2e9cfb9fb8e7066d6d5f3a154de0f76ce0d12ee678bcabd7b445c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE ws_specific SET path = $1 WHERE workspace_id = $2 AND item_kind = 'resource' AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "20ca0b26a0d2e9cfb9fb8e7066d6d5f3a154de0f76ce0d12ee678bcabd7b445c" +} diff --git a/backend/.sqlx/query-21331baf02c3c798bcc215443dad16eb66acac5f40c20529b595154f4e6fd754.json b/backend/.sqlx/query-21331baf02c3c798bcc215443dad16eb66acac5f40c20529b595154f4e6fd754.json new file mode 100644 index 0000000000..fe6da898d8 --- /dev/null +++ b/backend/.sqlx/query-21331baf02c3c798bcc215443dad16eb66acac5f40c20529b595154f4e6fd754.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_schedule_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_schedule_clean', '[]'::jsonb) || $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "21331baf02c3c798bcc215443dad16eb66acac5f40c20529b595154f4e6fd754" +} diff --git a/backend/.sqlx/query-2c256552a430877c42224055aeb81df33d88ff295483cb28369eda42ce58afec.json b/backend/.sqlx/query-2c256552a430877c42224055aeb81df33d88ff295483cb28369eda42ce58afec.json deleted file mode 100644 index a7b1b3793b..0000000000 --- a/backend/.sqlx/query-2c256552a430877c42224055aeb81df33d88ff295483cb28369eda42ce58afec.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of_email, assets, modules\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of_email, assets, modules\n FROM script\n WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "2c256552a430877c42224055aeb81df33d88ff295483cb28369eda42ce58afec" -} diff --git a/backend/.sqlx/query-2d236ba78748c6160abec9ae1ad11e9e90063e2c583e2145f6d5b86fbf981861.json b/backend/.sqlx/query-2d236ba78748c6160abec9ae1ad11e9e90063e2c583e2145f6d5b86fbf981861.json new file mode 100644 index 0000000000..ae433869b2 --- /dev/null +++ b/backend/.sqlx/query-2d236ba78748c6160abec9ae1ad11e9e90063e2c583e2145f6d5b86fbf981861.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "2d236ba78748c6160abec9ae1ad11e9e90063e2c583e2145f6d5b86fbf981861" +} diff --git a/backend/.sqlx/query-3ea2e1994d576e7a1b32ab276820dcff8e4e3b83f0d6a78fa0120c73fe90f345.json b/backend/.sqlx/query-3ea2e1994d576e7a1b32ab276820dcff8e4e3b83f0d6a78fa0120c73fe90f345.json new file mode 100644 index 0000000000..1be67a5b73 --- /dev/null +++ b/backend/.sqlx/query-3ea2e1994d576e7a1b32ab276820dcff8e4e3b83f0d6a78fa0120c73fe90f345.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "3ea2e1994d576e7a1b32ab276820dcff8e4e3b83f0d6a78fa0120c73fe90f345" +} diff --git a/backend/.sqlx/query-4a623e8789073abd92c52615077b89ed24398f59205c64290fa52ab19b3e44a3.json b/backend/.sqlx/query-4a623e8789073abd92c52615077b89ed24398f59205c64290fa52ab19b3e44a3.json new file mode 100644 index 0000000000..e94fe23a05 --- /dev/null +++ b/backend/.sqlx/query-4a623e8789073abd92c52615077b89ed24398f59205c64290fa52ab19b3e44a3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4a623e8789073abd92c52615077b89ed24398f59205c64290fa52ab19b3e44a3" +} diff --git a/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json b/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json new file mode 100644 index 0000000000..99b8051298 --- /dev/null +++ b/backend/.sqlx/query-4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "4d03f08c86588d3fba178340a3b64f375f82e8904920d51454f10e04fc24b62a" +} diff --git a/backend/.sqlx/query-56cb2ff34e93ea7ebe9e3b4c98f086793ba0ad2ad00c4361fef4f183557c50fd.json b/backend/.sqlx/query-56cb2ff34e93ea7ebe9e3b4c98f086793ba0ad2ad00c4361fef4f183557c50fd.json new file mode 100644 index 0000000000..5db3015cc6 --- /dev/null +++ b/backend/.sqlx/query-56cb2ff34e93ea7ebe9e3b4c98f086793ba0ad2ad00c4361fef4f183557c50fd.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "56cb2ff34e93ea7ebe9e3b4c98f086793ba0ad2ad00c4361fef4f183557c50fd" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json b/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json new file mode 100644 index 0000000000..3b1d7fae8c --- /dev/null +++ b/backend/.sqlx/query-5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e.json @@ -0,0 +1,83 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE job_tree AS (\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n '' as path_label, 0 as depth,\n j.id::text as id_path,\n ''::text as parent_module_type\n FROM v2_job j\n WHERE j.id = $2 AND j.workspace_id = $1\n UNION ALL\n SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job,\n CASE\n WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '')\n ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '')\n END,\n jt.depth + 1,\n jt.id_path || '/' || j.id::text,\n COALESCE((\n SELECT m->'value'->>'type'\n FROM v2_job parent_j\n LEFT JOIN flow f ON f.path = parent_j.runnable_path\n AND f.workspace_id = parent_j.workspace_id\n LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id\n CROSS JOIN LATERAL jsonb_array_elements(\n COALESCE(parent_j.raw_flow, f.value, fn.flow)->'modules'\n ) m\n WHERE parent_j.id = jt.id\n AND m->>'id' = j.flow_step_id\n LIMIT 1\n ), '')::text\n FROM v2_job j\n JOIN job_tree jt ON j.parent_job = jt.id\n WHERE j.workspace_id = $1\n ),\n with_sibling_index AS (\n SELECT jt.*,\n ROW_NUMBER() OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ORDER BY jt.id\n ) as sibling_index,\n COUNT(*) OVER (\n PARTITION BY jt.parent_job, jt.flow_step_id\n ) as sibling_count\n FROM job_tree jt\n )\n SELECT w.id, w.kind, w.flow_step_id, w.path_label,\n w.sibling_index::int as sibling_index,\n w.sibling_count::int as sibling_count,\n w.depth::int as depth,\n w.parent_module_type,\n coalesce(job_logs.logs, '') as logs,\n COALESCE(job_logs.log_offset, 0) as log_offset,\n job_logs.log_file_index\n FROM with_sibling_index w\n LEFT JOIN job_logs ON job_logs.job_id = w.id\n ORDER BY w.id_path ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "path_label", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "sibling_index", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "sibling_count", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "depth", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "parent_module_type", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 9, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "log_file_index", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + true + ] + }, + "hash": "5b95f464d492227ffe8a449844da00a6539788f9be647d6b52f01cbb2967473e" +} diff --git a/backend/.sqlx/query-5ed468f47150ab26cced7ebbd17ea111ab8926ba67c875974d2c9f4ef366d212.json b/backend/.sqlx/query-5ed468f47150ab26cced7ebbd17ea111ab8926ba67c875974d2c9f4ef366d212.json new file mode 100644 index 0000000000..abeac825ec --- /dev/null +++ b/backend/.sqlx/query-5ed468f47150ab26cced7ebbd17ea111ab8926ba67c875974d2c9f4ef366d212.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM background_task_state\n WHERE name LIKE $1\n AND owner != $2\n AND running = true\n AND updated_at > $3\n ) AS \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5ed468f47150ab26cced7ebbd17ea111ab8926ba67c875974d2c9f4ef366d212" +} diff --git a/backend/.sqlx/query-954e832d2587506a4bb0cb3a4fb45658026de7da4680eca1ea6b08f4b5e33800.json b/backend/.sqlx/query-954e832d2587506a4bb0cb3a4fb45658026de7da4680eca1ea6b08f4b5e33800.json new file mode 100644 index 0000000000..72e8f575eb --- /dev/null +++ b/backend/.sqlx/query-954e832d2587506a4bb0cb3a4fb45658026de7da4680eca1ea6b08f4b5e33800.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "954e832d2587506a4bb0cb3a4fb45658026de7da4680eca1ea6b08f4b5e33800" +} diff --git a/backend/.sqlx/query-9c5f9fc1c2fdf35e98c78180a319f2cfdfe09b4eef3375a6170454d3d52e8dbc.json b/backend/.sqlx/query-9c5f9fc1c2fdf35e98c78180a319f2cfdfe09b4eef3375a6170454d3d52e8dbc.json new file mode 100644 index 0000000000..71ffe65d7e --- /dev/null +++ b/backend/.sqlx/query-9c5f9fc1c2fdf35e98c78180a319f2cfdfe09b4eef3375a6170454d3d52e8dbc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM v2_job WHERE root_job = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9c5f9fc1c2fdf35e98c78180a319f2cfdfe09b4eef3375a6170454d3d52e8dbc" +} diff --git a/backend/.sqlx/query-a25200e046d6e15bf5a5f81d6cc9622bdb7ad8009fa6e67606296c1a8a1a92b4.json b/backend/.sqlx/query-a25200e046d6e15bf5a5f81d6cc9622bdb7ad8009fa6e67606296c1a8a1a92b4.json new file mode 100644 index 0000000000..0b8295cf85 --- /dev/null +++ b/backend/.sqlx/query-a25200e046d6e15bf5a5f81d6cc9622bdb7ad8009fa6e67606296c1a8a1a92b4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_schedule_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_schedule_clean', '[]'::jsonb) || $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "a25200e046d6e15bf5a5f81d6cc9622bdb7ad8009fa6e67606296c1a8a1a92b4" +} diff --git a/backend/.sqlx/query-a969194571dd3f12e628ce0f01b0ddc09bbcf4506eff2290664c295d16fec4ae.json b/backend/.sqlx/query-a969194571dd3f12e628ce0f01b0ddc09bbcf4506eff2290664c295d16fec4ae.json deleted file mode 100644 index 527e41e133..0000000000 --- a/backend/.sqlx/query-a969194571dd3f12e628ce0f01b0ddc09bbcf4506eff2290664c295d16fec4ae.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO script\n (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels)\n\n SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels\n\n FROM script WHERE hash = $2 AND workspace_id = $3;\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a969194571dd3f12e628ce0f01b0ddc09bbcf4506eff2290664c295d16fec4ae" -} diff --git a/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json b/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json deleted file mode 100644 index 0e87ff0632..0000000000 --- a/backend/.sqlx/query-b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_status\n SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1)\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "b01160fe44d69834ac08bbf60feacb3e3caa02a04b084da44cdcb9103794b39e" -} diff --git a/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json b/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json new file mode 100644 index 0000000000..d6aa0a4360 --- /dev/null +++ b/backend/.sqlx/query-b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb)\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "b2903da4186589d9df7d4e245f29138047b78f32d270a91a433cd6a04ecb48ea" +} diff --git a/backend/.sqlx/query-c19df45b092a246141e52770e70e02293b0a5eac859cbdc45f07c08957f00e00.json b/backend/.sqlx/query-c19df45b092a246141e52770e70e02293b0a5eac859cbdc45f07c08957f00e00.json new file mode 100644 index 0000000000..620842de3c --- /dev/null +++ b/backend/.sqlx/query-c19df45b092a246141e52770e70e02293b0a5eac859cbdc45f07c08957f00e00.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE resource\n SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text))\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c19df45b092a246141e52770e70e02293b0a5eac859cbdc45f07c08957f00e00" +} diff --git a/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json b/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json new file mode 100644 index 0000000000..3abe38f3cb --- /dev/null +++ b/backend/.sqlx/query-da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET datatable = jsonb_set(\n jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)),\n ARRAY['datatables', $2, 'forked_from'], $4::jsonb\n )\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "da6f37f91f2a24cf9ce8a1759adc5e9aebca35a0245598000c5d741dc6ede2d5" +} diff --git a/backend/.sqlx/query-790d79ec7abe6ebe1092afd9de4c5fc383272d2057ac5b47a7425f095f4e8788.json b/backend/.sqlx/query-dafc503a5f3adc5c7db7c11096775cacfffd2d3173dfdddcf37589cba356791e.json similarity index 79% rename from backend/.sqlx/query-790d79ec7abe6ebe1092afd9de4c5fc383272d2057ac5b47a7425f095f4e8788.json rename to backend/.sqlx/query-dafc503a5f3adc5c7db7c11096775cacfffd2d3173dfdddcf37589cba356791e.json index 29885406c0..e327857d3f 100644 --- a/backend/.sqlx/query-790d79ec7abe6ebe1092afd9de4c5fc383272d2057ac5b47a7425f095f4e8788.json +++ b/backend/.sqlx/query-dafc503a5f3adc5c7db7c11096775cacfffd2d3173dfdddcf37589cba356791e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41)", "describe": { "columns": [], "parameters": { @@ -77,6 +77,7 @@ "Bool", "Bool", "Int4", + "Int4", "Varchar", "Bool", "Varchar", @@ -95,5 +96,5 @@ }, "nullable": [] }, - "hash": "790d79ec7abe6ebe1092afd9de4c5fc383272d2057ac5b47a7425f095f4e8788" + "hash": "dafc503a5f3adc5c7db7c11096775cacfffd2d3173dfdddcf37589cba356791e" } diff --git a/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json b/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json new file mode 100644 index 0000000000..c2eb36edb4 --- /dev/null +++ b/backend/.sqlx/query-df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "df13aa5740f5475165b934f26d90b0f7c9ed4f2aa5b2d54817e7630ca1d37753" +} diff --git a/backend/.sqlx/query-e1886af775f70f7ad3949e35d2e274818884b2ce38226d5d8f9b8774dda6d5dc.json b/backend/.sqlx/query-e1886af775f70f7ad3949e35d2e274818884b2ce38226d5d8f9b8774dda6d5dc.json new file mode 100644 index 0000000000..114d2dd984 --- /dev/null +++ b/backend/.sqlx/query-e1886af775f70f7ad3949e35d2e274818884b2ce38226d5d8f9b8774dda6d5dc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_delete_schedule\n WHERE job_id IN (\n SELECT job_id FROM job_delete_schedule\n WHERE delete_at <= now()\n ORDER BY delete_at\n LIMIT $1\n FOR UPDATE SKIP LOCKED\n )\n RETURNING job_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "job_id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e1886af775f70f7ad3949e35d2e274818884b2ce38226d5d8f9b8774dda6d5dc" +} diff --git a/backend/.sqlx/query-edcda7e58fd1c586b5fb7aadedfde6b9dcffbc859426295c58719f6a8a544936.json b/backend/.sqlx/query-edcda7e58fd1c586b5fb7aadedfde6b9dcffbc859426295c58719f6a8a544936.json new file mode 100644 index 0000000000..517b691759 --- /dev/null +++ b/backend/.sqlx/query-edcda7e58fd1c586b5fb7aadedfde6b9dcffbc859426295c58719f6a8a544936.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path) VALUES ($1, 'resource', $2) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "edcda7e58fd1c586b5fb7aadedfde6b9dcffbc859426295c58719f6a8a544936" +} diff --git a/backend/.sqlx/query-f0fcc355334f4d715b366e1b3be88b2b5e316536efae31da09217176b59db030.json b/backend/.sqlx/query-f0fcc355334f4d715b366e1b3be88b2b5e316536efae31da09217176b59db030.json new file mode 100644 index 0000000000..bddbe43ad8 --- /dev/null +++ b/backend/.sqlx/query-f0fcc355334f4d715b366e1b3be88b2b5e316536efae31da09217176b59db030.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (\n workspace_id, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of_email, assets, modules\n )\n SELECT\n $1, hash, path, parent_hashes, summary, description, content,\n created_by, created_at, archived, schema, deleted, is_template,\n extra_perms, lock, lock_error_logs, language, kind, tag, draft_only,\n envs, concurrent_limit, concurrency_time_window_s, cache_ttl,\n dedicated_worker, ws_error_handler_muted, priority, timeout,\n delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key,\n visible_to_runner_only, auto_kind, codebase, has_preprocessor,\n on_behalf_of_email, assets, modules\n FROM script\n WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "f0fcc355334f4d715b366e1b3be88b2b5e316536efae31da09217176b59db030" +} diff --git a/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json b/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json new file mode 100644 index 0000000000..f0080ed07c --- /dev/null +++ b/backend/.sqlx/query-f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f85208a1f14536125db503d5cf375737a5855560a7f7c4a105461af73778a078" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d9e1840170..58adf9273b 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -5644,9 +5644,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a043dc74da1e37d6afe657061213aa6f425f855399a11d3463c6ecccc4dfda1f" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "fastwebsockets" @@ -9612,9 +9612,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] name = "openssl-src" -version = "300.5.5+3.5.5" +version = "300.6.0+3.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709" +checksum = "a8e8cbfd3a4a8c8f089147fd7aaa33cf8c7450c4d09f8f80698a0cf093abeff4" dependencies = [ "cc", ] @@ -16007,7 +16007,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-nats", @@ -16086,7 +16086,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16099,7 +16099,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "argon2", @@ -16240,7 +16240,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16263,7 +16263,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16276,7 +16276,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16302,7 +16302,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.677.0" +version = "1.680.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -16312,7 +16312,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16329,7 +16329,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "base64 0.22.1", @@ -16352,7 +16352,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16375,7 +16375,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16391,7 +16391,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16411,7 +16411,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16431,7 +16431,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16445,7 +16445,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-nats", @@ -16476,7 +16476,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16501,7 +16501,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "flate2", @@ -16519,7 +16519,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16541,7 +16541,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16561,7 +16561,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16591,7 +16591,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16618,7 +16618,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.677.0" +version = "1.680.0" dependencies = [ "lazy_static", "serde", @@ -16630,7 +16630,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.677.0" +version = "1.680.0" dependencies = [ "argon2", "axum 0.8.4", @@ -16655,7 +16655,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16669,7 +16669,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.677.0" +version = "1.680.0" dependencies = [ "axum 0.8.4", "chrono", @@ -16696,12 +16696,13 @@ dependencies = [ "windmill-git-sync", "windmill-object-store", "windmill-queue", + "windmill-store", "windmill-types", ] [[package]] name = "windmill-audit" -version = "1.677.0" +version = "1.680.0" dependencies = [ "chrono", "lazy_static", @@ -16715,7 +16716,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "axum 0.8.4", @@ -16734,7 +16735,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.677.0" +version = "1.680.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -16837,7 +16838,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.677.0" +version = "1.680.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -16856,7 +16857,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.677.0" +version = "1.680.0" dependencies = [ "regex", "serde", @@ -16871,7 +16872,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16895,7 +16896,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "futures", @@ -16912,7 +16913,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.677.0" +version = "1.680.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16928,7 +16929,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -16949,7 +16950,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -16980,7 +16981,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-oauth2", @@ -17004,7 +17005,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-stream", @@ -17038,7 +17039,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "futures", @@ -17056,7 +17057,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.677.0" +version = "1.680.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -17065,7 +17066,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "lazy_static", @@ -17077,7 +17078,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde_json", @@ -17089,7 +17090,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "gosyn", @@ -17101,7 +17102,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "lazy_static", @@ -17113,7 +17114,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde_json", @@ -17125,7 +17126,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "nu-parser", @@ -17136,7 +17137,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17147,7 +17148,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17159,7 +17160,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17170,7 +17171,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-recursion", @@ -17192,7 +17193,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde_json", @@ -17204,7 +17205,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "lazy_static", @@ -17218,7 +17219,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -17235,7 +17236,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "lazy_static", @@ -17248,7 +17249,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde", @@ -17260,7 +17261,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "lazy_static", @@ -17278,7 +17279,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17294,7 +17295,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "rustpython-ast", @@ -17310,7 +17311,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "serde", @@ -17321,7 +17322,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-recursion", @@ -17358,7 +17359,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "const_format", @@ -17396,7 +17397,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.677.0" +version = "1.680.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17407,7 +17408,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-recursion", @@ -17436,7 +17437,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17460,7 +17461,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17493,7 +17494,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17513,7 +17514,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17547,7 +17548,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17583,7 +17584,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17606,7 +17607,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17630,7 +17631,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-nats", @@ -17654,7 +17655,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17689,7 +17690,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17717,7 +17718,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-trait", @@ -17740,7 +17741,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17759,7 +17760,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.677.0" +version = "1.680.0" dependencies = [ "anyhow", "async-once-cell", @@ -17870,7 +17871,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.677.0" +version = "1.680.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ceee118e59..4d1d2f8df4 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.677.0" +version = "1.680.0" authors.workspace = true edition.workspace = true @@ -85,7 +85,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.677.0" +version = "1.680.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b554862c0c..fd2e07ccdd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8e5b77ef1f07b5b6620e540fea580f1a3d7f7d8b +9ea1259fa6a95e6b24d1523b2f6ec6ae0042ecfe diff --git a/backend/migrations/20260407000001_resource_ws_specific.down.sql b/backend/migrations/20260407000001_resource_ws_specific.down.sql new file mode 100644 index 0000000000..94a25ecb6f --- /dev/null +++ b/backend/migrations/20260407000001_resource_ws_specific.down.sql @@ -0,0 +1 @@ +ALTER TABLE resource DROP COLUMN IF EXISTS ws_specific; diff --git a/backend/migrations/20260407000001_resource_ws_specific.up.sql b/backend/migrations/20260407000001_resource_ws_specific.up.sql new file mode 100644 index 0000000000..67cfcb037b --- /dev/null +++ b/backend/migrations/20260407000001_resource_ws_specific.up.sql @@ -0,0 +1,3 @@ +-- Add ws_specific flag to resource table. +-- When true, the resource is excluded from workspace diff comparisons (e.g. auto-created resources during fork). +ALTER TABLE resource ADD COLUMN IF NOT EXISTS ws_specific BOOLEAN NOT NULL DEFAULT false; diff --git a/backend/migrations/20260407154106_add_delete_after_secs.down.sql b/backend/migrations/20260407154106_add_delete_after_secs.down.sql new file mode 100644 index 0000000000..04dadfcdd5 --- /dev/null +++ b/backend/migrations/20260407154106_add_delete_after_secs.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS job_delete_schedule; +ALTER TABLE script DROP COLUMN IF EXISTS delete_after_secs; diff --git a/backend/migrations/20260407154106_add_delete_after_secs.up.sql b/backend/migrations/20260407154106_add_delete_after_secs.up.sql new file mode 100644 index 0000000000..713f513cce --- /dev/null +++ b/backend/migrations/20260407154106_add_delete_after_secs.up.sql @@ -0,0 +1,9 @@ +ALTER TABLE script ADD COLUMN delete_after_secs INTEGER; + +CREATE TABLE job_delete_schedule ( + job_id UUID PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL, + delete_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX idx_job_delete_schedule_delete_at ON job_delete_schedule (delete_at); diff --git a/backend/migrations/20260408140410_move_alert_to_global_settings.down.sql b/backend/migrations/20260408140410_move_alert_to_global_settings.down.sql new file mode 100644 index 0000000000..ece61a3ec9 --- /dev/null +++ b/backend/migrations/20260408140410_move_alert_to_global_settings.down.sql @@ -0,0 +1,6 @@ +-- Move alert config back from global_settings to the config table. +INSERT INTO config (name, config) +SELECT 'alert__job_queue_waiting', value FROM global_settings WHERE name = 'alert_job_queue_waiting' +ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config; + +DELETE FROM global_settings WHERE name = 'alert_job_queue_waiting'; diff --git a/backend/migrations/20260408140410_move_alert_to_global_settings.up.sql b/backend/migrations/20260408140410_move_alert_to_global_settings.up.sql new file mode 100644 index 0000000000..62e860f811 --- /dev/null +++ b/backend/migrations/20260408140410_move_alert_to_global_settings.up.sql @@ -0,0 +1,7 @@ +-- Move alert config from the generic config table to global_settings +-- where it belongs alongside other instance-level settings. +INSERT INTO global_settings (name, value) +SELECT 'alert_job_queue_waiting', config FROM config WHERE name = 'alert__job_queue_waiting' +ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value; + +DELETE FROM config WHERE name = 'alert__job_queue_waiting'; diff --git a/backend/migrations/20260408151657_ws_specific_table.down.sql b/backend/migrations/20260408151657_ws_specific_table.down.sql new file mode 100644 index 0000000000..4b68cf8964 --- /dev/null +++ b/backend/migrations/20260408151657_ws_specific_table.down.sql @@ -0,0 +1,12 @@ +-- Re-add the column to resource +ALTER TABLE resource ADD COLUMN IF NOT EXISTS ws_specific BOOLEAN NOT NULL DEFAULT false; + +-- Migrate data back +UPDATE resource SET ws_specific = true +FROM ws_specific ws +WHERE ws.workspace_id = resource.workspace_id + AND ws.item_kind = 'resource' + AND ws.path = resource.path; + +-- Drop the table +DROP TABLE IF EXISTS ws_specific; diff --git a/backend/migrations/20260408151657_ws_specific_table.up.sql b/backend/migrations/20260408151657_ws_specific_table.up.sql new file mode 100644 index 0000000000..62033f7198 --- /dev/null +++ b/backend/migrations/20260408151657_ws_specific_table.up.sql @@ -0,0 +1,17 @@ +-- Create the ws_specific table +CREATE TABLE IF NOT EXISTS ws_specific ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + item_kind VARCHAR(50) NOT NULL, + path VARCHAR(255) NOT NULL, + PRIMARY KEY (workspace_id, item_kind, path) +); + +-- Migrate existing data from resource.ws_specific +INSERT INTO ws_specific (workspace_id, item_kind, path) +SELECT workspace_id, 'resource', path +FROM resource +WHERE ws_specific = true +ON CONFLICT DO NOTHING; + +-- Drop the column from resource +ALTER TABLE resource DROP COLUMN IF EXISTS ws_specific; diff --git a/backend/src/main.rs b/backend/src/main.rs index af22a203c6..1386a4b9bf 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1600,7 +1600,7 @@ async fn process_notify_event( "restart_worker_group" => { if worker_mode && payload == *WORKER_GROUP { tracing::info!("Restart requested for worker group '{payload}'"); - spawn_graceful_killpill(tx, db, 10, "worker group restart requested").await; + spawn_graceful_killpill(tx, db, 30, "worker group restart requested").await; } } "notify_webhook_change" => { @@ -1804,7 +1804,7 @@ async fn process_notify_event( reload_otel_tracing_proxy_setting(conn).await; if worker_mode { tracing::info!("OTEL tracing proxy setting changed, restarting worker"); - spawn_graceful_killpill(tx, db, 10, "OTEL tracing proxy setting change") + spawn_graceful_killpill(tx, db, 30, "OTEL tracing proxy setting change") .await; } } @@ -1813,12 +1813,12 @@ async fn process_notify_event( } EXPOSE_METRICS_SETTING => { tracing::info!("Metrics setting changed, restarting"); - spawn_graceful_killpill(tx, db, 10, "metrics setting change").await; + spawn_graceful_killpill(tx, db, 30, "metrics setting change").await; } EMAIL_DOMAIN_SETTING => { tracing::info!("Email domain setting changed"); if server_mode { - spawn_graceful_killpill(tx, db, 10, "email domain setting change").await; + spawn_graceful_killpill(tx, db, 30, "email domain setting change").await; } } EXPOSE_DEBUG_METRICS_SETTING => { @@ -1854,19 +1854,19 @@ async fn process_notify_event( } OTEL_SETTING => { tracing::info!("OTEL setting changed, restarting"); - spawn_graceful_killpill(tx, db, 10, "OTEL setting change").await; + spawn_graceful_killpill(tx, db, 30, "OTEL setting change").await; } REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); - spawn_graceful_killpill(tx, db, 10, "request size limit change").await; + spawn_graceful_killpill(tx, db, 30, "request size limit change").await; } } SAML_METADATA_SETTING => { tracing::info!( "SAML metadata change detected, killing server expecting to be restarted" ); - spawn_graceful_killpill(tx, db, 10, "SAML metadata change").await; + spawn_graceful_killpill(tx, db, 30, "SAML metadata change").await; } HUB_BASE_URL_SETTING => { if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await { @@ -2099,35 +2099,72 @@ async fn spawn_graceful_killpill( // Minimum delay before any restart to let in-flight requests drain const DRAIN_DELAY_SECS: u64 = 3; - let delay = match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await { - Ok(d) => d, - Err(e) => { - tracing::warn!( - "Failed to coordinate restart for {context}: {e:#}, \ - falling back to drain delay of {DRAIN_DELAY_SECS}s" - ); - DRAIN_DELAY_SECS - } - }; + let (delay, is_first) = + match coordinate_restart_delay(db, safety_margin_secs, DRAIN_DELAY_SECS).await { + Ok(r) => r, + Err(e) => { + tracing::warn!( + "Failed to coordinate restart for {context}: {e:#}, \ + falling back to drain delay of {DRAIN_DELAY_SECS}s" + ); + (DRAIN_DELAY_SECS, true) + } + }; - tracing::info!("Scheduling {context} graceful shutdown in {delay}s"); + tracing::info!( + "Scheduling {context} graceful shutdown in {delay}s (first_to_restart={is_first})" + ); let tx = tx.clone(); + let db = db.clone(); + let context = context.to_string(); + // Capture the current time so the health check only considers heartbeats + // written *after* this restart was initiated (not stale pre-restart ones). + let not_before = chrono::Utc::now(); tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(delay)).await; + if !is_first { + // Non-first instance: wait for another server to announce itself as + // started (via the coordination record) before shutting down, ensuring + // zero-downtime. Falls back to the computed delay as a maximum timeout. + let started_waiting = tokio::time::Instant::now(); + let max_wait = Duration::from_secs(delay.max(safety_margin_secs).max(120)); + loop { + if windmill_api::check_any_server_started(&db, not_before).await { + tracing::info!( + "{context}: healthy peer detected after {:.1}s, proceeding with shutdown", + started_waiting.elapsed().as_secs_f64() + ); + break; + } + if started_waiting.elapsed() >= max_wait { + tracing::warn!( + "{context}: no healthy peer detected after {:.1}s, proceeding with shutdown anyway", + started_waiting.elapsed().as_secs_f64() + ); + break; + } + tokio::time::sleep(Duration::from_secs(5)).await; + } + } else { + tokio::time::sleep(Duration::from_secs(delay)).await; + } tx.send(); }); } /// Coordinate a restart delay with other instances via the DB. /// -/// Returns the delay (in seconds from now) at which this instance should restart. -/// The first server gets `drain_delay_secs` (to let in-flight requests complete). -/// Each subsequent server is spaced `safety_margin_secs` after the latest scheduled restart. +/// Returns `(delay_secs, is_first)`: +/// - `delay_secs`: how long to wait before restarting +/// - `is_first`: whether this instance is the first to schedule a restart +/// +/// The first server restarts immediately (0 delay) to maximize its head-start. +/// Subsequent servers use a safety margin after the latest scheduled restart as a +/// fallback timeout, but primarily wait for a health-check (see `spawn_graceful_killpill`). async fn coordinate_restart_delay( db: &Pool, safety_margin_secs: u64, drain_delay_secs: u64, -) -> anyhow::Result { +) -> anyhow::Result<(u64, bool)> { const RESTART_LOCK_ID: i64 = 737_483_920; // Stale threshold: ignore coordination entries older than this const STALE_THRESHOLD_SECS: i64 = 120; @@ -2177,16 +2214,22 @@ async fn coordinate_restart_delay( // Find the latest scheduled restart let latest = scheduled.iter().map(|(_, dt)| *dt).max(); + let is_first = latest.is_none(); let earliest_allowed = now + chrono::Duration::seconds(drain_delay_secs as i64); - // Our restart time: drain_delay from now, or safety_margin after the latest existing restart + // Our restart time: + // - First instance (no prior scheduled restarts): brief 1s delay to let in-flight + // requests (like the settings save) complete, then restart quickly to maximize + // head-start before subsequent instances shut down. + // - Subsequent instances: safety_margin after the latest scheduled restart (used as + // a fallback timeout; the actual shutdown is gated by a health-check in the caller). let our_restart = match latest { Some(last) => { let after_last = last + chrono::Duration::seconds(safety_margin_secs as i64); // Use whichever is later: drain delay or staggered position earliest_allowed.max(after_last) } - None => earliest_allowed, + None => now + chrono::Duration::seconds(1), }; // Record our restart time (deduplicate: remove any prior entry for this instance) @@ -2215,5 +2258,5 @@ async fn coordinate_restart_delay( tx.commit().await.context("commit restart coordination")?; let delay = (our_restart - now).num_seconds().max(0) as u64; - Ok(delay) + Ok((delay, is_first)) } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 1d9544c911..5044d26a80 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1205,6 +1205,92 @@ pub async fn delete_expired_items(db: &DB) -> () { } } +#[cfg(feature = "enterprise")] +async fn cleanup_scheduled_job_deletions(db: &Pool) { + const BATCH_SIZE: i64 = 1000; + const MAX_BATCHES: i32 = 10; + + let mut total_deleted = 0u64; + for batch_num in 0..MAX_BATCHES { + let mut tx = match db.begin().await { + Ok(tx) => tx, + Err(e) => { + tracing::error!("Error starting transaction for scheduled job deletion: {e:?}"); + break; + } + }; + + let rows = match sqlx::query_scalar!( + "DELETE FROM job_delete_schedule + WHERE job_id IN ( + SELECT job_id FROM job_delete_schedule + WHERE delete_at <= now() + ORDER BY delete_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + RETURNING job_id", + BATCH_SIZE, + ) + .fetch_all(&mut *tx) + .await + { + Ok(rows) => rows, + Err(e) => { + tracing::error!("Error in scheduled job deletion batch {batch_num}: {e:?}"); + break; + } + }; + + if rows.is_empty() { + break; + } + + let job_ids = rows; + let count = job_ids.len() as u64; + + let cleanup_result: Result<(), sqlx::Error> = async { + sqlx::query!( + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", + &job_ids, + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = ANY($1)", + &job_ids, + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = ANY($1)", + &job_ids, + ) + .execute(&mut *tx) + .await?; + Ok(()) + } + .await; + + if let Err(e) = cleanup_result { + // Roll back so schedule rows survive for retry on next cycle + tracing::error!("Error cleaning job data in batch {batch_num}, rolling back: {e:?}"); + break; + } + + if let Err(e) = tx.commit().await { + tracing::error!("Error committing scheduled job deletion batch {batch_num}: {e:?}"); + break; + } + + total_deleted += count; + } + + if total_deleted > 0 { + tracing::info!("Scheduled job deletion: cleaned {total_deleted} jobs"); + } +} + pub async fn check_expiring_tokens(db: &DB) { // Find tokens expiring within 7 days that still have a pending notification row. // The notification table stores token_hash (not plaintext) so the join works @@ -2328,6 +2414,15 @@ pub async fn monitor_db( } }; + let cleanup_scheduled_job_deletions_f = async { + #[cfg(feature = "enterprise")] + if server_mode && !initial_load { + if let Some(db) = conn.as_sql() { + cleanup_scheduled_job_deletions(&db).await; + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -2351,6 +2446,7 @@ pub async fn monitor_db( cleanup_notify_events_f, check_expiring_tokens_f, manage_audit_partitions_f, + cleanup_scheduled_job_deletions_f, ); } diff --git a/backend/tests/delete_after_secs.rs b/backend/tests/delete_after_secs.rs new file mode 100644 index 0000000000..4ec0622686 --- /dev/null +++ b/backend/tests/delete_after_secs.rs @@ -0,0 +1,92 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::jobs::{resolve_delete_after_secs, schedule_job_deletion}; +use windmill_test_utils::*; + +// --------------------------------------------------------------------------- +// Unit tests for resolve_delete_after_secs +// --------------------------------------------------------------------------- + +#[test] +fn test_resolve_no_deletion() { + assert_eq!(resolve_delete_after_secs(None, None), None); + assert_eq!(resolve_delete_after_secs(Some(false), None), None); +} + +#[test] +fn test_resolve_immediate_backward_compat() { + // delete_after_use=true with no secs → immediate (0) + assert_eq!(resolve_delete_after_secs(Some(true), None), Some(0)); +} + +#[test] +fn test_resolve_explicit_secs() { + assert_eq!(resolve_delete_after_secs(None, Some(0)), Some(0)); + assert_eq!(resolve_delete_after_secs(None, Some(3600)), Some(3600)); + assert_eq!(resolve_delete_after_secs(Some(true), Some(60)), Some(60)); + assert_eq!(resolve_delete_after_secs(Some(false), Some(120)), Some(120)); +} + +#[test] +fn test_resolve_rejects_negative() { + assert_eq!(resolve_delete_after_secs(None, Some(-1)), None); + assert_eq!(resolve_delete_after_secs(Some(true), Some(-100)), None); +} + +// --------------------------------------------------------------------------- +// Integration: schedule_job_deletion inserts into job_delete_schedule +// --------------------------------------------------------------------------- + +#[sqlx::test(fixtures("base"))] +async fn test_schedule_job_deletion_inserts_row(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let job_id = uuid::Uuid::new_v4(); + + schedule_job_deletion(&db, job_id, "test-workspace", 3600).await?; + + let row = sqlx::query_as::<_, (uuid::Uuid, String)>( + "SELECT job_id, workspace_id FROM job_delete_schedule WHERE job_id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + + assert_eq!(row.0, job_id); + assert_eq!(row.1, "test-workspace"); + + // Verify delete_at is approximately now + 3600s + let delete_at: chrono::DateTime = + sqlx::query_scalar("SELECT delete_at FROM job_delete_schedule WHERE job_id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + + let expected_min = chrono::Utc::now() + chrono::Duration::seconds(3500); + let expected_max = chrono::Utc::now() + chrono::Duration::seconds(3700); + assert!( + delete_at > expected_min && delete_at < expected_max, + "delete_at should be ~1 hour from now, got {delete_at}" + ); + + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_schedule_job_deletion_is_idempotent(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let job_id = uuid::Uuid::new_v4(); + + schedule_job_deletion(&db, job_id, "test-workspace", 60).await?; + // Second call should not error (ON CONFLICT DO NOTHING) + schedule_job_deletion(&db, job_id, "test-workspace", 120).await?; + + let count: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM job_delete_schedule WHERE job_id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + assert_eq!(count, 1, "should have exactly one row (idempotent)"); + + Ok(()) +} diff --git a/backend/tests/dependency_map.rs b/backend/tests/dependency_map.rs index 86bca53984..d6d49c8d85 100644 --- a/backend/tests/dependency_map.rs +++ b/backend/tests/dependency_map.rs @@ -35,7 +35,7 @@ mod dependency_map { schema: std::collections::HashMap::new(), ws_error_handler_muted: Some(false), priority: None, - delete_after_use: None, + delete_after_secs: None, timeout: None, restart_unless_cancelled: None, deployment_message: None, diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 442a71405d..363ab4fb16 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -20,7 +20,7 @@ use serde_json::json; use sqlx::{Pool, Postgres}; use windmill_common::{ - flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Branch}, + flows::{Branch, FlowModule, FlowModuleValue, FlowValue, InputTransform}, jobs::JobPayload, scripts::ScriptLang, }; @@ -44,6 +44,7 @@ fn flow_module(id: &str, value: FlowModuleValue) -> FlowModule { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -54,12 +55,18 @@ fn flow_module(id: &str, value: FlowModuleValue) -> FlowModule { /// Helper to create input transforms from JavaScript expressions fn js_input(key: &str, expr: &str) -> (String, InputTransform) { - (key.to_string(), InputTransform::Javascript { expr: expr.to_string() }) + ( + key.to_string(), + InputTransform::Javascript { expr: expr.to_string() }, + ) } /// Helper to create static input transforms fn static_input(key: &str, value: T) -> (String, InputTransform) { - (key.to_string(), InputTransform::Static { value: windmill_common::worker::to_raw_value(&value) }) + ( + key.to_string(), + InputTransform::Static { value: windmill_common::worker::to_raw_value(&value) }, + ) } // ============================================================================= @@ -115,12 +122,13 @@ export function main(total: number, doubled_items: number[], filtered: number[], ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .arg("multiplier", json!(3)) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .arg("multiplier", json!(3)) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // Expected: sum=15, product=50, total=65, doubled=[2,4,6,8,10], filtered=[3,4,5], from_flow_input=45 assert_eq!(result["total"], json!(65)); @@ -144,10 +152,12 @@ async fn test_flow_forloop_complex_expressions(db: Pool) -> anyhow::Re let flow = FlowValue { modules: vec![ // Step a: return data to iterate over - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return { users: [ @@ -157,57 +167,66 @@ export function main() { ] }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - // Step b: for-loop over filtered users - flow_module("b", FlowModuleValue::ForloopFlow { - iterator: InputTransform::Javascript { - expr: "results.a.users.filter(u => u.score >= 80)".to_string() +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, }, - skip_failures: false, - parallel: false, - squash: None, - parallelism: None, - modules: vec![ - flow_module("c", FlowModuleValue::RawScript { - input_transforms: [ - js_input("user_name", "flow_input.iter.value.name"), - js_input("user_score", "flow_input.iter.value.score"), - js_input("bonus", "flow_input.iter.value.score >= 90 ? 10 : 5"), - js_input("index", "flow_input.iter.index"), - ].into(), - language: ScriptLang::Deno, - content: r#" + ), + // Step b: for-loop over filtered users + flow_module( + "b", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { + expr: "results.a.users.filter(u => u.score >= 80)".to_string(), + }, + skip_failures: false, + parallel: false, + squash: None, + parallelism: None, + modules: vec![flow_module( + "c", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("user_name", "flow_input.iter.value.name"), + js_input("user_score", "flow_input.iter.value.score"), + js_input("bonus", "flow_input.iter.value.score >= 90 ? 10 : 5"), + js_input("index", "flow_input.iter.index"), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(user_name: string, user_score: number, bonus: number, index: number) { return {name: user_name, final_score: user_score + bonus, position: index}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - modules_node: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + )], + modules_node: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // Only Alice (85) and Bob (92) pass the filter (score >= 80) // Alice gets bonus=5, Bob gets bonus=10 @@ -215,10 +234,10 @@ export function main(user_name: string, user_score: number, bonus: number, index let arr = result.as_array().unwrap(); assert_eq!(arr.len(), 2); assert_eq!(arr[0]["name"], "Alice"); - assert_eq!(arr[0]["final_score"], 90); // 85 + 5 + assert_eq!(arr[0]["final_score"], 90); // 85 + 5 assert_eq!(arr[0]["position"], 0); assert_eq!(arr[1]["name"], "Bob"); - assert_eq!(arr[1]["final_score"], 102); // 92 + 10 + assert_eq!(arr[1]["final_score"], 102); // 92 + 10 assert_eq!(arr[1]["position"], 1); Ok(()) @@ -237,105 +256,122 @@ async fn test_flow_branchone_conditions(db: Pool) -> anyhow::Result<() let flow = FlowValue { modules: vec![ // Step a: return data for branching - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {status: "premium", score: 95, items: [1, 2, 3]}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), // Step b: branch based on status and score - flow_module("b", FlowModuleValue::BranchOne { - branches: vec![ - Branch { - summary: Some("Premium with high score".to_string()), - expr: "results.a.status === 'premium' && results.a.score >= 90".to_string(), - modules: vec![ - flow_module("premium_high", FlowModuleValue::RawScript { - input_transforms: [ - js_input("discount", "results.a.score >= 95 ? 30 : 20"), - js_input("score_from_a", "results.a.score"), - ].into(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "b", + FlowModuleValue::BranchOne { + branches: vec![ + Branch { + summary: Some("Premium with high score".to_string()), + expr: "results.a.status === 'premium' && results.a.score >= 90" + .to_string(), + modules: vec![flow_module( + "premium_high", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("discount", "results.a.score >= 95 ? 30 : 20"), + js_input("score_from_a", "results.a.score"), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(discount: number, score_from_a: number) { return {branch: "premium_high", discount, score_from_a}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - modules_node: None, - skip_failure: true, - parallel: true, - }, - Branch { - summary: Some("Premium with low score".to_string()), - expr: "results.a.status === 'premium' && results.a.score < 90".to_string(), - modules: vec![ - flow_module("premium_low", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + )], + modules_node: None, + skip_failure: true, + parallel: true, + }, + Branch { + summary: Some("Premium with low score".to_string()), + expr: "results.a.status === 'premium' && results.a.score < 90" + .to_string(), + modules: vec![flow_module( + "premium_low", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {branch: "premium_low", discount: 10}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - modules_node: None, - skip_failure: true, - parallel: true, - }, - ], - default: vec![ - flow_module("default_branch", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + )], + modules_node: None, + skip_failure: true, + parallel: true, + }, + ], + default: vec![flow_module( + "default_branch", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {branch: "default", discount: 0}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - default_node: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + )], + default_node: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // status=premium, score=95 -> premium_high branch, discount=30 assert_eq!(result["branch"], "premium_high"); @@ -357,69 +393,90 @@ async fn test_flow_previous_result_aggregation(db: Pool) -> anyhow::Re let flow = FlowValue { modules: vec![ - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {value: 10, items: [1, 2, 3]}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - flow_module("b", FlowModuleValue::RawScript { - input_transforms: [ - js_input("prev_value", "previous_result.value"), - js_input("prev_items_sum", "previous_result.items.reduce((a, b) => a + b, 0)"), - ].into(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + flow_module( + "b", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("prev_value", "previous_result.value"), + js_input( + "prev_items_sum", + "previous_result.items.reduce((a, b) => a + b, 0)", + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(prev_value: number, prev_items_sum: number) { return {value: prev_value * 2, sum: prev_items_sum}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - flow_module("c", FlowModuleValue::RawScript { - input_transforms: [ - js_input("a_value", "results.a.value"), - js_input("b_value", "results.b.value"), - js_input("b_sum", "results.b.sum"), - js_input("combined", "results.a.value + results.b.value + results.b.sum"), - ].into(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + flow_module( + "c", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("a_value", "results.a.value"), + js_input("b_value", "results.b.value"), + js_input("b_sum", "results.b.sum"), + js_input( + "combined", + "results.a.value + results.b.value + results.b.sum", + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(a_value: number, b_value: number, b_sum: number, combined: number) { return {a_value, b_value, b_sum, combined}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // a: value=10, items=[1,2,3] // b: prev_value=10, prev_items_sum=6 -> value=20, sum=6 @@ -504,11 +561,12 @@ export function main(cat_name: string, multiplier: number, values: number[], sum ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // Category A (multiplier=2): values=[20,40], total=60 // Category B (multiplier=3): values=[30,60], total=90 @@ -601,11 +659,12 @@ export function main(admins: string[], active_count: number, admin_bonus: number ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // Admins: Alice, Charlie // Active count: 1 (only Alice) @@ -638,72 +697,84 @@ async fn test_flow_skip_if_expressions(db: Pool) -> anyhow::Result<()> let flow = FlowValue { modules: vec![ - flow_module("check", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "check", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {should_skip: true, value: 100}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - { - let mut module = flow_module("maybe_skipped", FlowModuleValue::RawScript { - input_transforms: [ - js_input("input_val", "results.check.value * 2"), - ].into(), - language: ScriptLang::Deno, - content: r#" -export function main(input_val: number) { - return {processed: input_val, was_run: true}; -} -"#.to_string(), +"# + .to_string(), path: None, lock: None, tag: None, concurrency_settings: Default::default(), is_trigger: None, assets: None, - }); + }, + ), + { + let mut module = flow_module( + "maybe_skipped", + FlowModuleValue::RawScript { + input_transforms: [js_input("input_val", "results.check.value * 2")].into(), + language: ScriptLang::Deno, + content: r#" +export function main(input_val: number) { + return {processed: input_val, was_run: true}; +} +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ); module.skip_if = Some(windmill_common::flows::SkipIf { expr: "results.check.should_skip === true".to_string(), }); module }, - flow_module("final", FlowModuleValue::RawScript { - input_transforms: [ - js_input("check_val", "results.check.value"), - js_input("prev", "previous_result"), - ].into(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "final", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("check_val", "results.check.value"), + js_input("prev", "previous_result"), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(check_val: number, prev: any) { return {check_val, previous: prev}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // maybe_skipped should be skipped because check.should_skip === true // So previous_result in final should be from check, not maybe_skipped @@ -771,11 +842,12 @@ export function main(full_name: string, greeting: string, items_str: string, upp ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["full_name"], "John Doe"); assert_eq!(result["greeting"], "Hello, John! You have 42 items."); @@ -850,11 +922,12 @@ export function main(city: string, missing_city: any, null_user_name: any, defau ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["city"], "NYC"); assert_eq!(result["missing_city"], serde_json::Value::Null); @@ -877,62 +950,74 @@ async fn test_flow_parallel_forloop(db: Pool) -> anyhow::Result<()> { let flow = FlowValue { modules: vec![ - flow_module("data", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "data", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return {items: [1, 2, 3, 4, 5]}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - flow_module("parallel_loop", FlowModuleValue::ForloopFlow { - iterator: InputTransform::Javascript { - expr: "results.data.items.map(x => ({ value: x, squared: x * x }))".to_string() +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, }, - skip_failures: false, - parallel: true, - squash: None, - parallelism: None, - modules: vec![ - flow_module("process", FlowModuleValue::RawScript { - input_transforms: [ - js_input("original", "flow_input.iter.value.value"), - js_input("squared", "flow_input.iter.value.squared"), - js_input("cubed", "flow_input.iter.value.value ** 3"), - ].into(), - language: ScriptLang::Deno, - content: r#" + ), + flow_module( + "parallel_loop", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { + expr: "results.data.items.map(x => ({ value: x, squared: x * x }))" + .to_string(), + }, + skip_failures: false, + parallel: true, + squash: None, + parallelism: None, + modules: vec![flow_module( + "process", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("original", "flow_input.iter.value.value"), + js_input("squared", "flow_input.iter.value.squared"), + js_input("cubed", "flow_input.iter.value.value ** 3"), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(original: number, squared: number, cubed: number) { return {original, squared, cubed}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - modules_node: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + )], + modules_node: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // Results may be in any order due to parallel execution assert!(result.is_array()); @@ -940,7 +1025,8 @@ export function main(original: number, squared: number, cubed: number) { assert_eq!(arr.len(), 5); // Verify all expected values are present (order may vary) - let mut values: Vec = arr.iter() + let mut values: Vec = arr + .iter() .map(|r| r["original"].as_i64().unwrap()) .collect(); values.sort(); @@ -970,14 +1056,26 @@ async fn test_flow_env_access(db: Pool) -> anyhow::Result<()> { // Create flow_env with various types of values let mut flow_env = std::collections::HashMap::new(); - flow_env.insert("ENV".to_string(), windmill_common::worker::to_raw_value(&json!("production"))); - flow_env.insert("DEBUG".to_string(), windmill_common::worker::to_raw_value(&json!(false))); - flow_env.insert("TIMEOUT".to_string(), windmill_common::worker::to_raw_value(&json!(30))); - flow_env.insert("CONFIG".to_string(), windmill_common::worker::to_raw_value(&json!({ - "apiUrl": "https://api.example.com", - "retries": 3, - "features": ["auth", "logging"] - }))); + flow_env.insert( + "ENV".to_string(), + windmill_common::worker::to_raw_value(&json!("production")), + ); + flow_env.insert( + "DEBUG".to_string(), + windmill_common::worker::to_raw_value(&json!(false)), + ); + flow_env.insert( + "TIMEOUT".to_string(), + windmill_common::worker::to_raw_value(&json!(30)), + ); + flow_env.insert( + "CONFIG".to_string(), + windmill_common::worker::to_raw_value(&json!({ + "apiUrl": "https://api.example.com", + "retries": 3, + "features": ["auth", "logging"] + })), + ); let flow = FlowValue { modules: vec![ @@ -1009,11 +1107,12 @@ export function main(env_name: string, is_debug: boolean, timeout_val: number, a ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["env_name"], "production"); assert_eq!(result["is_debug"], false); @@ -1037,8 +1136,14 @@ async fn test_flow_input_and_env_combined(db: Pool) -> anyhow::Result< // flow_env with environment-specific configuration let mut flow_env = std::collections::HashMap::new(); - flow_env.insert("ENV".to_string(), windmill_common::worker::to_raw_value(&json!("production"))); - flow_env.insert("MAX_ITEMS".to_string(), windmill_common::worker::to_raw_value(&json!(100))); + flow_env.insert( + "ENV".to_string(), + windmill_common::worker::to_raw_value(&json!("production")), + ); + flow_env.insert( + "MAX_ITEMS".to_string(), + windmill_common::worker::to_raw_value(&json!(100)), + ); let flow = FlowValue { modules: vec![ @@ -1071,21 +1176,22 @@ export function main(effective_limit: number, env_prefix: string, is_prod: boole ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .arg("requested_limit", json!(150)) - .arg("value", json!(25)) - .arg("prod_mult", json!(3)) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .arg("requested_limit", json!(150)) + .arg("value", json!(25)) + .arg("prod_mult", json!(3)) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // effective_limit = min(150, 100) = 100 assert_eq!(result["effective_limit"], 100); assert_eq!(result["env_prefix"], "[production]"); assert_eq!(result["is_prod"], true); - assert_eq!(result["doubled_input"], 50); // 25 * 2 - // final_value = 50 * 3 (prod_mult because ENV is production) + assert_eq!(result["doubled_input"], 50); // 25 * 2 + // final_value = 50 * 3 (prod_mult because ENV is production) assert_eq!(result["final_value"], 150); Ok(()) @@ -1104,10 +1210,12 @@ async fn test_flow_results_optional_chaining(db: Pool) -> anyhow::Resu let flow = FlowValue { modules: vec![ // Step a: return nested data with some null values - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return { user: { @@ -1127,39 +1235,59 @@ export function main() { null_field: null }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), // Step b: use optional chaining on results - flow_module("b", FlowModuleValue::RawScript { - input_transforms: [ - // Basic optional chaining - js_input("user_name", "results.a.user?.name"), - js_input("user_email", "results.a.user?.profile?.email"), - // Optional chaining with null value - js_input("user_phone", "results.a.user?.profile?.phone ?? 'no_phone'"), - // Optional chaining on null settings - js_input("user_setting", "results.a.user?.settings?.theme ?? 'default_theme'"), - // Optional chaining with array access - js_input("first_item_value", "results.a.items?.[0]?.value"), - js_input("second_item_value", "results.a.items?.[1]?.value ?? 0"), - // Optional chaining with find - js_input("item_by_id", "results.a.items?.find(i => i.id === 1)?.value"), - js_input("missing_item", "results.a.items?.find(i => i.id === 999)?.value ?? 'not_found'"), - // Optional chaining on empty array - js_input("empty_first", "results.a.empty_array?.[0]?.value ?? 'empty'"), - // Nullish coalescing with null field - js_input("null_with_default", "results.a.null_field ?? 'was_null'"), - // Accessing missing property with ?. - js_input("missing_prop", "results.a.nonexistent?.nested?.deep ?? 'missing'"), - ].into(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "b", + FlowModuleValue::RawScript { + input_transforms: [ + // Basic optional chaining + js_input("user_name", "results.a.user?.name"), + js_input("user_email", "results.a.user?.profile?.email"), + // Optional chaining with null value + js_input("user_phone", "results.a.user?.profile?.phone ?? 'no_phone'"), + // Optional chaining on null settings + js_input( + "user_setting", + "results.a.user?.settings?.theme ?? 'default_theme'", + ), + // Optional chaining with array access + js_input("first_item_value", "results.a.items?.[0]?.value"), + js_input("second_item_value", "results.a.items?.[1]?.value ?? 0"), + // Optional chaining with find + js_input( + "item_by_id", + "results.a.items?.find(i => i.id === 1)?.value", + ), + js_input( + "missing_item", + "results.a.items?.find(i => i.id === 999)?.value ?? 'not_found'", + ), + // Optional chaining on empty array + js_input( + "empty_first", + "results.a.empty_array?.[0]?.value ?? 'empty'", + ), + // Nullish coalescing with null field + js_input("null_with_default", "results.a.null_field ?? 'was_null'"), + // Accessing missing property with ?. + js_input( + "missing_prop", + "results.a.nonexistent?.nested?.deep ?? 'missing'", + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main( user_name: string, user_email: string, @@ -1179,24 +1307,27 @@ export function main( empty_first, null_with_default, missing_prop }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["user_name"], "Alice"); assert_eq!(result["user_email"], "alice@example.com"); @@ -1225,10 +1356,12 @@ async fn test_flow_large_integers(db: Pool) -> anyhow::Result<()> { let flow = FlowValue { modules: vec![ - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return { small_int: 42, @@ -1238,29 +1371,37 @@ export function main() { large_safe: 9007199254740991, // MAX_SAFE_INTEGER }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - flow_module("b", FlowModuleValue::RawScript { - input_transforms: [ - js_input("small", "results.a.small_int"), - js_input("i32_max", "results.a.i32_max"), - js_input("over_i32", "results.a.i32_max_plus_1"), - js_input("timestamp", "results.a.timestamp"), - js_input("ts_plus_day", "results.a.timestamp + 86400000"), - js_input("large", "results.a.large_safe"), - // Arithmetic on large numbers - js_input("large_minus_1", "results.a.large_safe - 1"), - // Comparisons - js_input("is_large_safe", "Number.isSafeInteger(results.a.large_safe)"), - ].into(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + flow_module( + "b", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("small", "results.a.small_int"), + js_input("i32_max", "results.a.i32_max"), + js_input("over_i32", "results.a.i32_max_plus_1"), + js_input("timestamp", "results.a.timestamp"), + js_input("ts_plus_day", "results.a.timestamp + 86400000"), + js_input("large", "results.a.large_safe"), + // Arithmetic on large numbers + js_input("large_minus_1", "results.a.large_safe - 1"), + // Comparisons + js_input( + "is_large_safe", + "Number.isSafeInteger(results.a.large_safe)", + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main( small: number, i32_max: number, over_i32: number, timestamp: number, ts_plus_day: number, @@ -1269,24 +1410,27 @@ export function main( ) { return {small, i32_max, over_i32, timestamp, ts_plus_day, large, large_minus_1, is_large_safe}; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["small"], 42); assert_eq!(result["i32_max"], 2147483647_i64); @@ -1365,11 +1509,12 @@ export function main( ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["greeting"], "Hello World"); assert_eq!(result["greeting_len"], 11); @@ -1466,16 +1611,20 @@ export function main( ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); assert_eq!(result["sorted_asc"], json!([1, 2, 3, 4, 5, 6, 7, 8, 9])); assert_eq!(result["sorted_desc"], json!([9, 8, 7, 6, 5, 4, 3, 2, 1])); assert_eq!(result["active_names"], json!(["Alice", "Charlie", "Diana"])); - assert_eq!(result["high_scorers"], json!([{"name": "Bob", "score": 92}, {"name": "Diana", "score": 95}])); + assert_eq!( + result["high_scorers"], + json!([{"name": "Bob", "score": 92}, {"name": "Diana", "score": 95}]) + ); assert_eq!(result["total_score"], 350); // 85 + 92 + 78 + 95 assert_eq!(result["avg_score"], 87.5); assert_eq!(result["top_scorer"], "Diana"); @@ -1582,17 +1731,21 @@ export function main( ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // subtotal = 10*2 + 20*3 + 30*1 = 20 + 60 + 30 = 110 assert_eq!(result["subtotal"], 110); // discounted_total = 110 * (1 - 0.1) = 99 assert_eq!(result["discounted_total"], 99.0); - assert_eq!(result["item_summary"]["names"], json!(["Item A", "Item B", "Item C"])); + assert_eq!( + result["item_summary"]["names"], + json!(["Item A", "Item B", "Item C"]) + ); assert_eq!(result["item_summary"]["total_qty"], 6); // safe_calculation = 110 * 1.08 = 118.8 assert_eq!(result["safe_calculation"], 118.8); @@ -1670,18 +1823,28 @@ export function main( ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); - assert_eq!(result["merged_config"], json!({"host": "localhost", "port": 3000, "timeout": 5000})); + assert_eq!( + result["merged_config"], + json!({"host": "localhost", "port": 3000, "timeout": 5000}) + ); assert_eq!(result["all_tags"], json!(["api", "v1", "production"])); - assert_eq!(result["full_user"], json!({"name": "Alice", "role": "admin", "permissions": ["read", "write"]})); + assert_eq!( + result["full_user"], + json!({"name": "Alice", "role": "admin", "permissions": ["read", "write"]}) + ); assert_eq!(result["max_port"], 8080); assert_eq!(result["rest_config"], json!({"port": 3000})); - assert_eq!(result["combined"]["config"], json!({"host": "localhost", "port": 3000})); + assert_eq!( + result["combined"]["config"], + json!({"host": "localhost", "port": 3000}) + ); assert_eq!(result["combined"]["tags"], json!(["api", "v1"])); assert_eq!(result["combined"]["source"], "flow"); @@ -1701,42 +1864,51 @@ async fn test_flow_nested_forloop_results_access(db: Pool) -> anyhow:: let flow = FlowValue { modules: vec![ // Step a: outer data - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return { multiplier: 10, categories: ["cat1", "cat2"] }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - // Outer for-loop - flow_module("outer", FlowModuleValue::ForloopFlow { - iterator: InputTransform::Javascript { - expr: "results.a.categories".to_string() +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, }, - skip_failures: false, - parallel: false, - squash: None, - parallelism: None, - modules: vec![ - // Step b: generate inner items based on category - flow_module("b", FlowModuleValue::RawScript { - input_transforms: [ - js_input("category", "flow_input.iter.value"), - js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop - ].into(), - language: ScriptLang::Deno, - content: r#" + ), + // Outer for-loop + flow_module( + "outer", + FlowModuleValue::ForloopFlow { + iterator: InputTransform::Javascript { + expr: "results.a.categories".to_string(), + }, + skip_failures: false, + parallel: false, + squash: None, + parallelism: None, + modules: vec![ + // Step b: generate inner items based on category + flow_module( + "b", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("category", "flow_input.iter.value"), + js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(category: string, multiplier: number) { return { category, @@ -1746,23 +1918,28 @@ export function main(category: string, multiplier: number) { })) }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - // Step c: process inner items and access previous step in loop - flow_module("c", FlowModuleValue::RawScript { - input_transforms: [ - js_input("items", "results.b.items"), // Access sibling step - js_input("category", "results.b.category"), - js_input("original_mult", "results.a.multiplier"), // Access outer step - ].into(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + // Step c: process inner items and access previous step in loop + flow_module( + "c", + FlowModuleValue::RawScript { + input_transforms: [ + js_input("items", "results.b.items"), // Access sibling step + js_input("category", "results.b.category"), + js_input("original_mult", "results.a.multiplier"), // Access outer step + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main(items: any[], category: string, original_mult: number) { return { category, @@ -1771,27 +1948,31 @@ export function main(items: any[], category: string, original_mult: number) { total_value: items.reduce((sum, i) => sum + i.value, 0) }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - ], - modules_node: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + ], + modules_node: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // outer loop produces 2 results (for cat1 and cat2) assert!(result.is_array()); @@ -1827,36 +2008,49 @@ async fn test_flow_results_non_existent_step(db: Pool) -> anyhow::Resu let flow = FlowValue { modules: vec![ - flow_module("a", FlowModuleValue::RawScript { - input_transforms: Default::default(), - language: ScriptLang::Deno, - content: r#" + flow_module( + "a", + FlowModuleValue::RawScript { + input_transforms: Default::default(), + language: ScriptLang::Deno, + content: r#" export function main() { return { value: 42 }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), - flow_module("b", FlowModuleValue::RawScript { - input_transforms: [ - // Access existing step - should work - js_input("existing", "results.a.value"), - // Access non-existent step - should return null, not error - // Note: The expression gets wrapped as (await results.nonexistent) - // The proxy returns a Promise that resolves to null for non-existent steps - js_input("non_existent", "results.nonexistent"), - // Access non-existent step with nullish coalescing - js_input("non_existent_with_default", "results.nonexistent ?? 'default_value'"), - // Nested access on non-existent step (null?.value -> undefined -> ?? kicks in) - js_input("non_existent_nested", "results.nonexistent?.value ?? 'nested_default'"), - ].into(), - language: ScriptLang::Deno, - content: r#" +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), + flow_module( + "b", + FlowModuleValue::RawScript { + input_transforms: [ + // Access existing step - should work + js_input("existing", "results.a.value"), + // Access non-existent step - should return null, not error + // Note: The expression gets wrapped as (await results.nonexistent) + // The proxy returns a Promise that resolves to null for non-existent steps + js_input("non_existent", "results.nonexistent"), + // Access non-existent step with nullish coalescing + js_input( + "non_existent_with_default", + "results.nonexistent ?? 'default_value'", + ), + // Nested access on non-existent step (null?.value -> undefined -> ?? kicks in) + js_input( + "non_existent_nested", + "results.nonexistent?.value ?? 'nested_default'", + ), + ] + .into(), + language: ScriptLang::Deno, + content: r#" export function main( existing: number, non_existent: any, @@ -1870,24 +2064,27 @@ export function main( non_existent_nested }; } -"#.to_string(), - path: None, - lock: None, - tag: None, - concurrency_settings: Default::default(), - is_trigger: None, - assets: None, - }), +"# + .to_string(), + path: None, + lock: None, + tag: None, + concurrency_settings: Default::default(), + is_trigger: None, + assets: None, + }, + ), ], same_worker: false, ..Default::default() }; - let result = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) - .run_until_complete(&db, false, server.addr.port()) - .await - .json_result() - .unwrap(); + let result = + RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); // existing step should work assert_eq!(result["existing"], 42); diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index afd414b0c3..055b633d28 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -1151,3 +1151,125 @@ async fn test_replace_mode_protects_jwt_secret_and_rsa_keys(db: Pool) ); assert!(get_global_setting(&db, "normal_setting").await.is_none()); } + +// ======================================================================== +// Alert config migration tests +// ======================================================================== + +#[sqlx::test(fixtures("base"))] +async fn test_alert_config_in_global_settings_roundtrip(db: Pool) { + clear_settings_and_configs(&db).await; + + let alert_value = serde_json::json!({ + "alerts": [ + { + "name": "Test Alert", + "tags_to_monitor": ["default", "gpu"], + "jobs_num_threshold": 5, + "alert_cooldown_seconds": 300, + "alert_time_threshold_seconds": 60 + } + ] + }); + + // Insert alert_config into global_settings + insert_global_setting(&db, "alert_job_queue_waiting", alert_value.clone()).await; + + // Verify it appears in InstanceConfig global_settings (via extra) + let config = InstanceConfig::from_db(&db).await.unwrap(); + assert_eq!( + config.global_settings.extra["alert_job_queue_waiting"], alert_value, + "alert_config should appear in global_settings extra" + ); + + // Verify it does NOT appear in worker_configs + assert!( + !config + .worker_configs + .contains_key("alert_job_queue_waiting"), + "alert_config should not appear in worker_configs" + ); + + // Modify the alert_config + let updated_value = serde_json::json!({ + "alerts": [ + { + "name": "Updated Alert", + "tags_to_monitor": ["batch"], + "jobs_num_threshold": 10, + "alert_cooldown_seconds": 600, + "alert_time_threshold_seconds": 120 + } + ] + }); + + let current = config.global_settings.to_settings_map(); + let mut desired = current.clone(); + desired.insert("alert_job_queue_waiting".to_string(), updated_value.clone()); + + let diff = diff_global_settings(¤t, &desired, ApplyMode::Merge); + assert!( + diff.upserts.contains_key("alert_job_queue_waiting"), + "alert_config change should be detected in diff" + ); + + apply_settings_diff(&db, &diff).await.unwrap(); + + // Re-read and verify the update + let config2 = InstanceConfig::from_db(&db).await.unwrap(); + assert_eq!( + config2.global_settings.extra["alert_job_queue_waiting"], + updated_value + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_alert_config_not_in_worker_configs(db: Pool) { + clear_settings_and_configs(&db).await; + + // Insert alert_config in global_settings (the correct location) + insert_global_setting( + &db, + "alert_job_queue_waiting", + serde_json::json!({"alerts": []}), + ) + .await; + + // Also insert a real worker config + insert_config( + &db, + "worker__default", + serde_json::json!({"worker_tags": ["default"]}), + ) + .await; + + let config = InstanceConfig::from_db(&db).await.unwrap(); + + // alert_config should be in global_settings, not worker_configs + assert!( + config + .global_settings + .extra + .contains_key("alert_job_queue_waiting"), + "alert_config should be in global_settings.extra" + ); + assert_eq!(config.worker_configs.len(), 1); + assert!( + config.worker_configs.contains_key("default"), + "only the real worker config should be in worker_configs" + ); +} + +#[sqlx::test(fixtures("base"))] +async fn test_no_alert_in_config_table_after_migration(db: Pool) { + // After the migration runs, no alert__* entries should remain in the config table + let rows: Vec<(String,)> = sqlx::query_as("SELECT name FROM config WHERE name LIKE 'alert__%'") + .fetch_all(&db) + .await + .unwrap(); + assert!( + rows.is_empty(), + "No alert entries should remain in config table after migration, found: {:?}", + rows.iter().map(|(n,)| n.as_str()).collect::>() + ); +} diff --git a/backend/tests/relock_skip.rs b/backend/tests/relock_skip.rs index 44e8de3db9..eb8dfb261d 100644 --- a/backend/tests/relock_skip.rs +++ b/backend/tests/relock_skip.rs @@ -33,7 +33,7 @@ mod relock_skip { schema: std::collections::HashMap::new(), ws_error_handler_muted: Some(false), priority: None, - delete_after_use: None, + delete_after_secs: None, timeout: None, restart_unless_cancelled: None, deployment_message: None, diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index dc24915486..42ff589bcd 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -206,6 +206,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -253,6 +254,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -274,6 +276,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -389,6 +392,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -445,6 +449,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -487,6 +492,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -508,6 +514,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -556,6 +563,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index 06ddbafcca..6b18b07c2b 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -608,11 +608,12 @@ fn extract_domain_from_route( ))) } -const RUN_WHITELISTED_GET_PATHS: [&'static str; 19] = [ +const RUN_WHITELISTED_GET_PATHS: [&'static str; 20] = [ "jobs_u/get_flow/", "jobs_u/get_root_job_id/", "jobs_u/get/", "jobs_u/get_logs/", + "jobs_u/get_flow_all_logs/", "jobs_u/get_args/", "jobs_u/get_flow_debug_info/", "jobs_u/completed/get/", diff --git a/backend/windmill-api-client/src/lib.rs b/backend/windmill-api-client/src/lib.rs index f2170b3c2f..d4bcb21b67 100644 --- a/backend/windmill-api-client/src/lib.rs +++ b/backend/windmill-api-client/src/lib.rs @@ -364,7 +364,7 @@ pub mod types { #[serde(default, skip_serializing_if = "Option::is_none")] pub dedicated_worker: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, + pub delete_after_secs: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub deployment_message: Option, pub description: String, @@ -540,7 +540,7 @@ pub mod types { #[serde(default, skip_serializing_if = "Option::is_none")] pub continue_on_error: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, + pub delete_after_secs: Option, pub id: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub mock: Option, diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index daa3e216e3..e78331a57d 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -1856,6 +1856,7 @@ mod tests { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -1890,6 +1891,7 @@ mod tests { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -1924,6 +1926,7 @@ mod tests { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -1957,6 +1960,7 @@ mod tests { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, @@ -1972,6 +1976,8 @@ mod tests { early_return: None, chat_input_enabled: None, flow_env: None, + delete_after_use: None, + delete_after_secs: None, concurrency_settings: ConcurrencySettings::default(), debouncing_settings: DebouncingSettings::default(), }; diff --git a/backend/windmill-api-integration-tests/tests/configs.rs b/backend/windmill-api-integration-tests/tests/configs.rs new file mode 100644 index 0000000000..a9559ab456 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/configs.rs @@ -0,0 +1,139 @@ +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +fn client() -> reqwest::Client { + reqwest::Client::new() +} + +fn authed(builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + builder.header("Authorization", "Bearer SECRET_TOKEN") +} + +fn assert_2xx(status: u16, body: &str, endpoint: &str) { + assert!( + (200..300).contains(&status), + "{endpoint} returned {status}: {body}", + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_worker_group_crud(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/configs"); + + // Create a worker group + let resp = authed(client().post(format!("{base}/update/worker__test_group"))) + .json(&json!({"worker_tags": ["test_tag"]})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /configs/update/worker__test_group", + ); + + // Verify it exists via get + let resp = authed(client().get(format!("{base}/get/worker__test_group"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx(status, &body, "GET /configs/get/worker__test_group"); + let config: serde_json::Value = serde_json::from_str(&body)?; + assert_eq!(config["worker_tags"], json!(["test_tag"])); + + // Verify it appears in list_worker_groups with prefix stripped + let resp = authed(client().get(format!("{base}/list_worker_groups"))) + .send() + .await?; + let body = resp.text().await?; + let groups: Vec = serde_json::from_str(&body)?; + let test_group = groups.iter().find(|g| g["name"] == "test_group"); + assert!( + test_group.is_some(), + "test_group should appear in list_worker_groups with prefix stripped" + ); + + // Delete the worker group + let resp = authed(client().delete(format!("{base}/update/worker__test_group"))) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "DELETE /configs/update/worker__test_group", + ); + + // Verify it's gone from list_worker_groups + let resp = authed(client().get(format!("{base}/list_worker_groups"))) + .send() + .await?; + let body = resp.text().await?; + let groups: Vec = serde_json::from_str(&body)?; + let test_group = groups.iter().find(|g| g["name"] == "test_group"); + assert!( + test_group.is_none(), + "test_group should be gone after deletion" + ); + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_list_worker_groups_excludes_non_worker_entries( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/configs"); + + // Insert a non-worker entry directly into the config table + sqlx::query("INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config") + .bind("test__other_entry") + .bind(json!({"some_key": "some_value"})) + .execute(&db) + .await?; + + // list_worker_groups should NOT return it + let resp = authed(client().get(format!("{base}/list_worker_groups"))) + .send() + .await?; + let body = resp.text().await?; + let groups: Vec = serde_json::from_str(&body)?; + let other_entry = groups + .iter() + .find(|g| g["name"] == "test__other_entry" || g["name"] == "other_entry"); + assert!( + other_entry.is_none(), + "non-worker entries should not appear in list_worker_groups" + ); + + // Clean up + sqlx::query("DELETE FROM config WHERE name = $1") + .bind("test__other_entry") + .execute(&db) + .await?; + + Ok(()) +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_no_alert_entries_in_config_table_after_migration( + db: Pool, +) -> anyhow::Result<()> { + // After migration, no alert__* entries should remain in the config table + let rows: Vec<(String,)> = sqlx::query_as("SELECT name FROM config WHERE name LIKE 'alert__%'") + .fetch_all(&db) + .await?; + assert!( + rows.is_empty(), + "No alert entries should remain in config table after migration, found: {:?}", + rows.iter().map(|(n,)| n.as_str()).collect::>() + ); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/settings.rs b/backend/windmill-api-integration-tests/tests/settings.rs index 8f21ffb483..348d945a86 100644 --- a/backend/windmill-api-integration-tests/tests/settings.rs +++ b/backend/windmill-api-integration-tests/tests/settings.rs @@ -114,3 +114,89 @@ async fn test_settings_2xx(db: Pool) -> anyhow::Result<()> { Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_alert_job_queue_waiting_in_global_settings(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let settings_base = format!("http://localhost:{port}/api/settings"); + let configs_base = format!("http://localhost:{port}/api/configs"); + + let alert_payload = json!({ + "alerts": [ + { + "name": "Test Alert", + "tags_to_monitor": ["default", "gpu"], + "jobs_num_threshold": 5, + "alert_cooldown_seconds": 300, + "alert_time_threshold_seconds": 60 + } + ] + }); + + // Set alert_job_queue_waiting in global_settings + let resp = authed(client().post(format!("{settings_base}/global/alert_job_queue_waiting"))) + .json(&json!({"value": alert_payload})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /settings/global/alert_job_queue_waiting", + ); + + // Read it back + let resp = authed(client().get(format!("{settings_base}/global/alert_job_queue_waiting"))) + .send() + .await?; + let status = resp.status().as_u16(); + let body = resp.text().await?; + assert_2xx( + status, + &body, + "GET /settings/global/alert_job_queue_waiting", + ); + let value: serde_json::Value = serde_json::from_str(&body)?; + assert_eq!(value["alerts"][0]["name"], "Test Alert"); + assert_eq!(value["alerts"][0]["jobs_num_threshold"], 5); + + // Verify alert_job_queue_waiting does NOT appear in list_worker_groups + let resp = authed(client().get(format!("{configs_base}/list_worker_groups"))) + .send() + .await?; + let body = resp.text().await?; + let groups: Vec = serde_json::from_str(&body)?; + let alert_in_groups = groups.iter().find(|g| { + let name = g["name"].as_str().unwrap_or(""); + name.contains("alert") + }); + assert!( + alert_in_groups.is_none(), + "alert_job_queue_waiting should not appear in worker groups" + ); + + // Delete by setting to null + let resp = authed(client().post(format!("{settings_base}/global/alert_job_queue_waiting"))) + .json(&json!({"value": null})) + .send() + .await?; + assert_2xx( + resp.status().as_u16(), + &resp.text().await?, + "POST /settings/global/alert_job_queue_waiting (null)", + ); + + // Verify it reads back as null + let resp = authed(client().get(format!("{settings_base}/global/alert_job_queue_waiting"))) + .send() + .await?; + let body = resp.text().await?; + let value: serde_json::Value = serde_json::from_str(&body)?; + assert!( + value.is_null(), + "alert_job_queue_waiting should be null after deletion" + ); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 6bc8c05abd..f98bf89c36 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -476,6 +476,25 @@ pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<() Ok(()) } +pub use windmill_common::jobs::resolve_delete_after_secs; +pub use windmill_common::jobs::schedule_job_deletion; + +/// Handle deletion or scheduling for a completed job. +pub async fn handle_delete_after_completion( + db: &DB, + job_uuid: Uuid, + w_id: &str, + delete_after_use: Option, + delete_after_secs: Option, +) -> Result<(), Error> { + match resolve_delete_after_secs(delete_after_use, delete_after_secs) { + Some(0) => delete_job_metadata_after_use(db, job_uuid).await?, + Some(secs) => schedule_job_deletion(db, job_uuid, w_id, secs).await?, + None => {} + } + Ok(()) +} + // --------------------------------------------------------------------------- // Queue limit check // --------------------------------------------------------------------------- @@ -799,7 +818,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( trigger: Option, ) -> error::Result<( Uuid, - Option, + Option, Option>, )> { #[cfg(feature = "enterprise")] @@ -809,14 +828,16 @@ pub async fn push_script_job_by_path_into_queue<'c>( check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( - script_path, - Some(userdb_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await?; + let (job_payload, tag, delete_after_use, delete_after_secs, timeout, on_behalf_of) = + script_path_to_payload( + script_path, + Some(userdb_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await?; + let resolved_delete_secs = resolve_delete_after_secs(delete_after_use, delete_after_secs); let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); @@ -886,9 +907,9 @@ pub async fn push_script_job_by_path_into_queue<'c>( // If we were given a transaction, return it; otherwise commit it if return_tx { - Ok((uuid, delete_after_use, Some(tx))) + Ok((uuid, resolved_delete_secs, Some(tx))) } else { tx.commit().await?; - Ok((uuid, delete_after_use, None)) + Ok((uuid, resolved_delete_secs, None)) } } diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index cf386a7ebf..8c54378ea5 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -113,6 +113,8 @@ pub struct ScriptWDraft { #[serde(skip_serializing_if = "Option::is_none")] pub delete_after_use: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] pub visible_to_runner_only: Option, @@ -178,6 +180,7 @@ impl ScriptWDraft { priority: self.priority, restart_unless_cancelled: self.restart_unless_cancelled, delete_after_use: self.delete_after_use, + delete_after_secs: self.delete_after_secs, timeout: self.timeout, visible_to_runner_only: self.visible_to_runner_only, auto_kind: self.auto_kind, @@ -943,8 +946,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40)", + delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, cache_ignore_s3_path, runnable_settings_handle, modules, labels) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, $41)", &w_id, &hash.0, ns.path, @@ -970,12 +973,13 @@ async fn create_script_internal<'c>( ns.priority, ns.restart_unless_cancelled, ns.delete_after_use, + ns.delete_after_secs, ns.timeout, guarded_concurrency_key, ns.visible_to_runner_only, auto_kind.as_deref(), codebase, - has_preprocessor.filter(|x: &bool| *x), // should be Some(true) or None + has_preprocessor.filter(|x: &bool| *x), windmill_common::resolve_on_behalf_of_email( ns.on_behalf_of_email.as_deref(), ns.preserve_on_behalf_of.unwrap_or(false), @@ -1438,7 +1442,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, runnable_settings_handle, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, has_preprocessor, on_behalf_of_email, assets, modules, debounce_key, debounce_delay_s, labels FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 ORDER BY script.created_at DESC LIMIT 1", diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 1ca22b98b7..0e5defe07a 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -101,6 +101,10 @@ pub fn global_service() -> Router { "/setup_custom_instance_pg_database/{name}", post(setup_custom_instance_pg_database), ) + .route( + "/drop_custom_instance_pg_database/{name}", + post(drop_custom_instance_pg_database), + ) .route( "/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts), @@ -1179,6 +1183,18 @@ async fn setup_custom_instance_pg_database_inner( Ok(()) } +async fn drop_custom_instance_pg_database( + authed: ApiAuthed, + Extension(db): Extension, + Path(dbname): Path, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + windmill_common::drop_custom_instance_database(&db, &dbname).await?; + + Ok(format!("Database '{}' dropped successfully", dbname)) +} + // ============================================================================ // Secret Backend Settings (HashiCorp Vault Integration) - Enterprise Edition // ============================================================================ diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 86f0649c73..4a9e04c417 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -27,6 +27,7 @@ windmill-git-sync.workspace = true windmill-queue.workspace = true windmill-api-jobs.workspace = true windmill-dep-map.workspace = true +windmill-store.workspace = true axum.workspace = true chrono.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 64c07754e9..15399fbe74 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -40,8 +40,8 @@ use windmill_common::workspaces::GitRepositorySettings; use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ check_user_against_rule, get_datatable_resource_from_db_unchecked, DataTable, - DataTableCatalogResourceType, ProtectionRuleKind, ProtectionRules, ProtectionRuleset, - RuleCheckResult, WorkspaceGitSyncSettings, + DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, + ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -157,6 +157,17 @@ pub fn workspaced_service() -> Router { post(reset_workspace_diffs), ) .route("/compare/{target_workspace_id}", get(compare_workspaces)) + .route("/create_pg_database", post(create_pg_database)) + .route("/import_pg_database", post(import_pg_database)) + .route("/export_pg_schema", post(export_pg_schema)) + .route( + "/drop_forked_datatable_databases", + post(crate::workspaces_extra::drop_forked_datatable_databases), + ) + .route( + "/get_datatable_full_schema", + post(get_datatable_full_schema), + ) .route("/protection_rules", get(list_protection_rules)) .route("/protection_rules", post(create_protection_rule)) .route( @@ -381,6 +392,16 @@ struct CreateWorkspaceFork { id: String, name: String, color: Option, + /// Datatable names that were forked. For each, the backend will update the + /// forked workspace's datatable config to point to the new database. + #[serde(default)] + forked_datatables: Vec, +} + +#[derive(Deserialize)] +struct ForkedDatatableInfo { + name: String, + new_dbname: String, } #[derive(Deserialize)] @@ -1273,26 +1294,40 @@ async fn list_ducklakes( Ok(Json(ducklakes)) } +#[derive(Serialize)] +struct DataTableListItem { + name: String, + resource_type: String, + resource_path: String, +} + async fn list_datatables( _authed: ApiAuthed, Extension(db): Extension, Path(w_id): Path, -) -> JsonResult> { - let datatables = sqlx::query_scalar!( - r#" - SELECT jsonb_object_keys(ws.datatable->'datatables') AS datatable_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, +) -> JsonResult> { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", &w_id ) - .fetch_all(&db) - .await? - .into_iter() - .filter_map(|s| s) - .collect(); + .fetch_one(&db) + .await?; - Ok(Json(datatables)) + let items: Vec = match config { + Some(val) => { + let map: HashMap = serde_json::from_value(val).unwrap_or_default(); + map.into_iter() + .map(|(name, dt)| DataTableListItem { + name, + resource_type: dt.database.resource_type.as_ref().to_string(), + resource_path: dt.database.resource_path, + }) + .collect() + } + None => vec![], + }; + + Ok(Json(items)) } /// Compact column representation: "type" or "type?" for nullable, with "=default" suffix if has default @@ -1455,6 +1490,397 @@ async fn get_datatable_schema(db: &DB, w_id: &str, datatable_name: &str) -> Resu Ok(schema_map) } +/// Resolve a source string to PgDatabase credentials with user-scoped permission checks. +/// For `datatable://name`: accessible to everyone (variables are resolved internally). +/// For `$res:path`: uses UserDB (row-level security) to verify the user can see the resource, +/// then interpolates `$var:` references in the resource value. +pub(crate) async fn resolve_pg_source_checked( + db: &DB, + user_db: &UserDB, + authed: &ApiAuthed, + w_id: &str, + source: &str, +) -> Result { + let db_resource = if let Some(name) = source.strip_prefix("datatable://") { + get_datatable_resource_from_db_unchecked(db, w_id, name).await? + } else if let Some(path) = source.strip_prefix("$res:") { + let db_with_authed = windmill_common::db::DbWithOptAuthed::from_authed( + authed, + db.clone(), + Some(user_db.clone()), + ); + let value = windmill_store::resources::get_resource_value_interpolated_internal( + &db_with_authed, + w_id, + path, + None, + None, + false, + ) + .await?; + + match value { + Some(v) => v, + None => { + return Err(Error::NotAuthorized(format!( + "Resource '{}' not found or you do not have access to it", + path + ))); + } + } + } else { + return Err(Error::BadRequest(format!( + "Invalid source format: '{}'. Expected 'datatable://name' or '$res:path'", + source + ))); + }; + + serde_json::from_value(db_resource) + .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e))) +} + +/// A temporary file for pg_dump output that is automatically deleted when dropped. +struct DumpFile { + path: std::path::PathBuf, +} + +impl DumpFile { + fn new() -> Result { + let dir = std::path::Path::new("/tmp/windmill"); + std::fs::create_dir_all(dir) + .map_err(|e| Error::internal_err(format!("Failed to create /tmp/windmill: {}", e)))?; + // Set directory permissions to owner-only + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)); + } + let path = dir.join(format!("datatable_dump_{}", uuid::Uuid::new_v4())); + // Create the file with restrictive permissions before pg_dump writes to it + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + std::fs::OpenOptions::new() + .write(true) + .create(true) + .mode(0o600) + .open(&path) + .map_err(|e| Error::internal_err(format!("Failed to create dump file: {}", e)))?; + } + #[cfg(not(unix))] + { + std::fs::File::create(&path) + .map_err(|e| Error::internal_err(format!("Failed to create dump file: {}", e)))?; + } + Ok(Self { path }) + } +} + +impl Drop for DumpFile { + fn drop(&mut self) { + if self.path.exists() { + if let Err(e) = std::fs::remove_file(&self.path) { + tracing::warn!("Failed to remove dump file {:?}: {}", self.path, e); + } + } + } +} + +/// Run pg_dump against a PgDatabase, writing output to a temp file on disk. +/// Returns a DumpFile handle; the file is deleted when the handle is dropped. +async fn pg_dump_database(pg_db: &PgDatabase, schema_only: bool) -> Result { + let dump_file = DumpFile::new()?; + + let host = &pg_db.host; + let port = pg_db.port.unwrap_or(5432).to_string(); + let user = pg_db.user.as_deref().unwrap_or("postgres"); + let dbname = &pg_db.dbname; + + let mut cmd = tokio::process::Command::new("pg_dump"); + cmd.arg("--format=plain").arg("--file").arg(&dump_file.path); + if schema_only { + cmd.arg("--schema-only"); + } + cmd.arg("--host") + .arg(host) + .arg("--port") + .arg(&port) + .arg("--username") + .arg(user) + .arg(dbname); + + if let Some(ref password) = pg_db.password { + cmd.env("PGPASSWORD", password); + } + + if let Some(ref sslmode) = pg_db.sslmode { + cmd.env("PGSSLMODE", sslmode); + } + + let output = cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute pg_dump: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!("pg_dump failed: {}", stderr))); + } + + Ok(dump_file) +} + +/// Import a pg_dump file into a target database using psql. +async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<()> { + let host = &target_db.host; + let port = target_db.port.unwrap_or(5432).to_string(); + let user = target_db.user.as_deref().unwrap_or("postgres"); + let dbname = &target_db.dbname; + + let mut cmd = tokio::process::Command::new("psql"); + cmd.arg("--host") + .arg(host) + .arg("--port") + .arg(&port) + .arg("--username") + .arg(user) + .arg("--dbname") + .arg(dbname) + .arg("--no-psqlrc") + .arg("--file") + .arg(&dump_file.path) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + + if let Some(ref password) = target_db.password { + cmd.env("PGPASSWORD", password); + } + + if let Some(ref sslmode) = target_db.sslmode { + cmd.env("PGSSLMODE", sslmode); + } + + let output = cmd + .output() + .await + .map_err(|e| Error::internal_err(format!("Failed to execute psql: {}", e)))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(Error::internal_err(format!( + "psql import failed: {}", + stderr + ))); + } + + Ok(()) +} + +#[derive(Deserialize)] +struct CreatePgDatabaseRequest { + /// The datatable source to determine connection info: 'datatable://name' or '$res:path' + source: String, + /// Name for the new database + target_dbname: String, +} + +/// Create a new PostgreSQL database. For instance datatables, creates on the Windmill PG instance. +/// For resource datatables, creates on the same server as the source. +async fn create_pg_database( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + windmill_common::validate_dbname(&req.target_dbname)?; + + // Non-superadmin: restrict dbname to wm_fork_ prefix + if !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + if !req.target_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest( + "Non-superadmin users can only create databases with names starting with 'wm_fork_'" + .to_string(), + )); + } + } + + // Determine if this is an instance or resource-backed datatable + let is_instance_datatable = if let Some(dt_name) = req.source.strip_prefix("datatable://") { + let config = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + &w_id, + dt_name + ) + .fetch_optional(&db) + .await? + .flatten(); + config + .and_then(|v| { + v.get("database") + .and_then(|d| d.get("resource_type")) + .and_then(|r| r.as_str()) + .map(|s| s == "instance") + }) + .unwrap_or(false) + } else { + false + }; + + if is_instance_datatable { + windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable") + .await?; + } else { + let source_pg = + resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let (client, connection) = source_pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let row = client + .query_one( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + &[&req.target_dbname], + ) + .await + .map_err(|e| { + Error::internal_err(format!("Failed to check database existence: {}", e)) + })?; + let db_exists: bool = row.get(0); + + if db_exists { + drop(client); + let _ = join_handle.await; + return Err(Error::BadRequest(format!( + "Database '{}' already exists on the resource server", + req.target_dbname + ))); + } + + client + .execute(&format!("CREATE DATABASE \"{}\"", &req.target_dbname), &[]) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to create database '{}': {}", + req.target_dbname, e + )) + })?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + } + + Ok(format!("Created database '{}'", req.target_dbname)) +} + +#[derive(Deserialize)] +struct ImportPgDatabaseRequest { + source: String, + target: String, + #[serde(default)] + target_dbname_override: Option, + fork_behavior: DataTableForkBehavior, +} + +/// Import (pg_dump/pg_import) from source to target +async fn import_pg_database( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + if req.fork_behavior == DataTableForkBehavior::KeepOriginal { + return Ok("No action needed for KeepOriginal behavior".to_string()); + } + + if req.fork_behavior == DataTableForkBehavior::SchemaAndData { + require_admin(authed.is_admin, &authed.username)?; + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Importing schema and data is not available on cloud".to_string(), + )); + } + } + + let schema_only = req.fork_behavior == DataTableForkBehavior::SchemaOnly; + let source_pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let mut target_pg = + resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.target).await?; + + if let Some(ref override_dbname) = req.target_dbname_override { + if !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + if !override_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest( + "Non-superadmin users can only override target dbname with names starting with 'wm_fork_'" + .to_string(), + )); + } + } + target_pg.dbname = override_dbname.clone(); + } + windmill_common::validate_dbname(&target_pg.dbname)?; + + let dump_file = pg_dump_database(&source_pg, schema_only).await?; + pg_import_dump(&target_pg, &dump_file).await?; + + Ok(format!( + "Imported from '{}' into '{}'", + req.source, target_pg.dbname + )) +} + +#[derive(Deserialize)] +struct ExportPgSchemaRequest { + source: String, +} + +async fn export_pg_schema( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result { + let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let dump_file = pg_dump_database(&pg, true).await?; + tokio::fs::read_to_string(&dump_file.path) + .await + .map_err(|e| Error::internal_err(format!("Failed to read dump file: {}", e))) +} + +#[derive(Deserialize)] +struct GetDatatableFullSchemaRequest { + source: String, +} + +async fn get_datatable_full_schema( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + let pg = resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?; + let (client, connection) = pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let result = windmill_common::query_builders::pg_get_full_schema(&client) + .await + .map_err(Error::internal_err)?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + Ok(Json(result)) +} + async fn edit_ducklake_config( authed: ApiAuthed, Extension(db): Extension, @@ -3297,7 +3723,7 @@ async fn clone_scripts( extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, timeout, - delete_after_use, restart_unless_cancelled, concurrency_key, + delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, assets, modules ) @@ -3307,7 +3733,7 @@ async fn clone_scripts( extra_perms, lock, lock_error_logs, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, timeout, - delete_after_use, restart_unless_cancelled, concurrency_key, + delete_after_use, delete_after_secs, restart_unless_cancelled, concurrency_key, visible_to_runner_only, auto_kind, codebase, has_preprocessor, on_behalf_of_email, assets, modules FROM script @@ -3732,6 +4158,127 @@ async fn create_workspace_fork_branch( )) } +/// Update a forked workspace's datatable config to point to the new database. +/// For instance datatables: updates resource_path in the datatable config. +/// For resource datatables: updates the resource's dbname and marks it as ws_specific. +/// Snapshot the schema from the source datatable by connecting to its database. +async fn snapshot_datatable_schema( + db: &DB, + parent_w_id: &str, + dt_name: &str, +) -> Result { + let pg = get_datatable_resource_from_db_unchecked(db, parent_w_id, dt_name).await?; + let pg: PgDatabase = serde_json::from_value(pg) + .map_err(|e| Error::internal_err(format!("Failed to parse db credentials: {}", e)))?; + let (client, connection) = pg.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + let schema = windmill_common::query_builders::pg_get_full_schema(&client) + .await + .map_err(Error::internal_err)?; + + drop(client); + join_handle + .await + .map_err(|e| Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + serde_json::to_value(schema) + .map_err(|e| Error::internal_err(format!("Failed to serialize schema: {}", e))) +} + +async fn apply_forked_datatable( + db: &DB, + tx: &mut Transaction<'_, Postgres>, + parent_w_id: &str, + forked_w_id: &str, + fdt: &ForkedDatatableInfo, +) -> Result<()> { + windmill_common::validate_dbname(&fdt.new_dbname)?; + if !fdt.new_dbname.starts_with("wm_fork_") { + return Err(Error::BadRequest(format!( + "Forked datatable database name '{}' must start with 'wm_fork_'", + fdt.new_dbname + ))); + } + + // Snapshot the schema from the source (parent) datatable + let schema = snapshot_datatable_schema(db, parent_w_id, &fdt.name).await?; + let forked_from = serde_json::json!({ "schema": schema }); + + // Read the datatable config from the forked workspace + let config_val = sqlx::query_scalar!( + "SELECT datatable->'datatables'->$2 FROM workspace_settings WHERE workspace_id = $1", + forked_w_id, + &fdt.name + ) + .fetch_optional(&mut **tx) + .await? + .flatten() + .ok_or_else(|| { + Error::NotFound(format!( + "Datatable '{}' not found in workspace '{}'", + fdt.name, forked_w_id + )) + })?; + + let dt: DataTable = serde_json::from_value(config_val) + .map_err(|e| Error::internal_err(format!("Failed to parse datatable config: {}", e)))?; + + if dt.database.resource_type == DataTableCatalogResourceType::Instance { + // Instance: update resource_path to the new dbname + sqlx::query!( + r#"UPDATE workspace_settings + SET datatable = jsonb_set( + jsonb_set(datatable, ARRAY['datatables', $2, 'database', 'resource_path'], to_jsonb($3::text)), + ARRAY['datatables', $2, 'forked_from'], $4::jsonb + ) + WHERE workspace_id = $1"#, + forked_w_id, + &fdt.name, + &fdt.new_dbname, + forked_from, + ) + .execute(&mut **tx) + .await?; + } else { + // Resource: update the resource's dbname and mark as ws_specific + let resource_path = &dt.database.resource_path; + sqlx::query!( + r#"UPDATE resource + SET value = jsonb_set(value, '{dbname}', to_jsonb($3::text)) + WHERE workspace_id = $1 AND path = $2"#, + forked_w_id, + resource_path, + &fdt.new_dbname, + ) + .execute(&mut **tx) + .await?; + + sqlx::query!( + "INSERT INTO ws_specific (workspace_id, item_kind, path) VALUES ($1, 'resource', $2) ON CONFLICT DO NOTHING", + forked_w_id, + resource_path, + ) + .execute(&mut **tx) + .await?; + + // Set forked_from on the datatable config + sqlx::query!( + r#"UPDATE workspace_settings + SET datatable = jsonb_set(datatable, ARRAY['datatables', $2, 'forked_from'], $3::jsonb) + WHERE workspace_id = $1"#, + forked_w_id, + &fdt.name, + forked_from, + ) + .execute(&mut **tx) + .await?; + } + + Ok(()) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, @@ -3813,6 +4360,11 @@ async fn create_workspace_fork( // Clone all data from the parent workspace using Rust implementation clone_workspace_data(&mut tx, &parent_workspace_id, &forked_id).await?; + // Update forked datatable settings to point to new databases + for fdt in &nw.forked_datatables { + apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; + } + audit_log( &mut *tx, &authed, @@ -3824,6 +4376,7 @@ async fn create_workspace_fork( ) .await?; tx.commit().await?; + Ok(format!("Created forked workspace {}", &forked_id)) } @@ -5554,6 +6107,33 @@ async fn compare_two_resources( .fetch_optional(db) .await?; + // If either side is ws_specific, consider unchanged + let source_ws_specific = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)", + source_workspace_id, + path + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + let target_ws_specific = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)", + fork_workspace_id, + path + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if source_ws_specific || target_ws_specific { + return Ok(ItemComparison { + has_changes: false, + exists_in_source: source_resource.is_some(), + exists_in_fork: target_resource.is_some(), + }); + } + let mut has_changes = false; // Check metadata differences diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 12c4fec971..231f7cb429 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -23,8 +23,10 @@ use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ auth::is_super_admin_email, + db::UserDB, error::{Error, Result}, utils::require_admin, + workspaces::DataTable, }; use windmill_queue::schedule::{get_schedule_opt, push_scheduled_job}; @@ -537,6 +539,14 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + info!("Deleting raw_script_temp table"); + sqlx::query!( + "DELETE FROM raw_script_temp WHERE workspace_id = $1", + &old_id + ) + .execute(&mut *tx) + .await?; + info!("Updating resource table"); sqlx::query!( "UPDATE resource SET workspace_id = $1 WHERE workspace_id = $2", @@ -827,6 +837,10 @@ pub(crate) async fn delete_workspace( // NATS triggers have on delete cascade + sqlx::query!("DELETE FROM raw_script_temp WHERE workspace_id = $1", &w_id) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; @@ -846,6 +860,157 @@ pub(crate) async fn delete_workspace( Ok(format!("Deleted workspace {}", &w_id)) } +#[derive(Deserialize)] +pub struct DropForkedDatatableDatabasesRequest { + datatable_names: Vec, +} + +/// Drop forked datatable databases. Returns errors per datatable that failed. +/// Same permission as delete_workspace: fork owner or super admin. +pub async fn drop_forked_datatable_databases( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> Result>> { + // Same permission check as delete_workspace: fork owner or super admin + let mut tx = db.begin().await?; + if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { + require_super_admin(&db, &authed.email).await?; + } + tx.commit().await?; + + let parent_w_id = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .ok_or_else(|| Error::BadRequest("No parent workspace found".to_string()))?; + + let datatable_config = sqlx::query_scalar!( + "SELECT datatable->'datatables' FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten() + .unwrap_or(serde_json::json!({})); + + let datatables: HashMap = + serde_json::from_value(datatable_config).unwrap_or_default(); + + let mut errors: Vec = Vec::new(); + + for dt_name in &req.datatable_names { + let dt = match datatables.get(dt_name) { + Some(dt) if dt.forked_from.is_some() => dt, + _ => continue, + }; + + if dt.database.resource_type + == windmill_common::workspaces::DataTableCatalogResourceType::Instance + { + let db_to_drop = &dt.database.resource_path; + if !db_to_drop.starts_with("wm_fork_") { + errors.push(format!( + "Refusing to drop instance database '{}' for datatable://{}: name does not start with 'wm_fork_'", + db_to_drop, dt_name + )); + continue; + } + if let Err(e) = windmill_common::drop_custom_instance_database(&db, db_to_drop).await { + errors.push(format!( + "Could not drop instance database '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + } + } else { + let fork_pg = match crate::workspaces::resolve_pg_source_checked( + &db, + &user_db, + &authed, + &w_id, + &format!("datatable://{}", dt_name), + ) + .await + { + Ok(pg) => pg, + Err(e) => { + errors.push(format!( + "Could not resolve fork resource for datatable://{}: {}", + dt_name, e + )); + continue; + } + }; + // We cannot drop the current database, so we connect to the parent's version to run DROP DATABASE on + // the forked version + let parent_pg = match crate::workspaces::resolve_pg_source_checked( + &db, + &user_db, + &authed, + &parent_w_id, + &format!("datatable://{}", dt_name), + ) + .await + { + Ok(pg) => pg, + Err(e) => { + errors.push(format!( + "Could not resolve parent resource for datatable://{}: {}", + dt_name, e + )); + continue; + } + }; + + let db_to_drop = &fork_pg.dbname; + if let Err(e) = windmill_common::validate_dbname(db_to_drop) { + errors.push(format!( + "Invalid database name '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + continue; + } + if !db_to_drop.starts_with("wm_fork_") { + errors.push(format!( + "Refusing to drop resource database '{}' for datatable://{}: name does not start with 'wm_fork_'", + db_to_drop, dt_name + )); + continue; + } + + match parent_pg.connect().await { + Ok((client, connection)) => { + let join_handle = tokio::spawn(async move { connection.await }); + if let Err(e) = client + .execute(&format!("DROP DATABASE \"{}\"", db_to_drop), &[]) + .await + { + errors.push(format!( + "Could not drop database '{}' for datatable://{}: {}", + db_to_drop, dt_name, e + )); + } + drop(client); + let _ = join_handle.await; + } + Err(e) => { + errors.push(format!( + "Could not connect to drop database for datatable://{}: {}", + dt_name, e + )); + } + } + } + } + + Ok(Json(errors)) +} + async fn is_workspace_owner( authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 07fdaa99a2..60ee3b8470 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.677.0 + version: 1.680.0 title: Windmill API contact: @@ -1079,6 +1079,27 @@ paths: schema: $ref: "#/components/schemas/CustomInstanceDb" + /settings/drop_custom_instance_pg_database/{name}: + post: + summary: Drops a custom instance database (superadmin only, irreversible) + operationId: dropCustomInstanceDb + tags: + - setting + parameters: + - in: path + name: name + required: true + schema: + type: string + description: The name of the database to drop + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + /settings/global/{key}: get: summary: get global settings @@ -3755,7 +3776,16 @@ paths: schema: type: array items: - type: string + type: object + required: [name, resource_type, resource_path] + properties: + name: + type: string + resource_type: + type: string + enum: [postgres, instance] + resource_path: + type: string /w/{workspace}/workspaces/list_datatable_schemas: get: @@ -3828,6 +3858,211 @@ paths: application/json: schema: {} + /w/{workspace}/workspaces/create_pg_database: + post: + summary: create a new PostgreSQL database for a datatable + operationId: createPgDatabase + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Create pg database request + required: true + content: + application/json: + schema: + type: object + required: [source, target_dbname] + properties: + source: + type: string + description: "Datatable source to determine connection info: 'datatable://name' or '$res:path'" + target_dbname: + type: string + description: "Name for the new database" + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/drop_forked_datatable_databases: + post: + summary: drop forked datatable databases + operationId: dropForkedDatatableDatabases + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [datatable_names] + properties: + datatable_names: + type: array + items: + type: string + responses: + "200": + description: list of errors (empty if all succeeded) + content: + application/json: + schema: + type: array + items: + type: string + + /w/{workspace}/workspaces/import_pg_database: + post: + summary: import a PostgreSQL database from source to target via pg_dump + operationId: importPgDatabase + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Import pg database request + required: true + content: + application/json: + schema: + type: object + required: [source, target, fork_behavior] + properties: + source: + type: string + description: "Source database: 'datatable://name' or '$res:path'" + target: + type: string + description: "Target database: 'datatable://name' or '$res:path'" + target_dbname_override: + type: string + description: "Override the target database name" + fork_behavior: + type: string + enum: + - schema_only + - schema_and_data + - keep_original + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/export_pg_schema: + post: + summary: export the schema of a PostgreSQL database + operationId: exportPgSchema + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: Export pg schema request + required: true + content: + application/json: + schema: + type: object + required: [source] + properties: + source: + type: string + description: "Source database: 'datatable://name' or '$res:path'" + responses: + "200": + description: schema dump + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/get_datatable_full_schema: + post: + summary: get the full schema of a datatable database as TableEditorValues + operationId: getDatatableFullSchema + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [source] + properties: + source: + type: string + description: "Source datatable, e.g. 'datatable://main'" + responses: + "200": + description: "Schema as { schema_name: { table_name: TableEditorValues } }" + content: + application/json: + schema: + type: object + additionalProperties: + type: object + additionalProperties: + type: object + required: [name, columns, foreign_keys] + properties: + name: + type: string + columns: + type: array + items: + type: object + required: [name, datatype] + properties: + name: + type: string + datatype: + type: string + primary_key: + type: boolean + default_value: + type: string + nullable: + type: boolean + foreign_keys: + type: array + items: + type: object + required: [columns, on_delete, on_update] + properties: + target_table: + type: string + columns: + type: array + items: + type: object + properties: + source_column: + type: string + target_column: + type: string + on_delete: + type: string + on_update: + type: string + fk_constraint_name: + type: string + pk_constraint_name: + type: string + /w/{workspace}/workspaces/git_sync_enabled: get: summary: Check if git sync is available for this workspace @@ -11031,6 +11266,23 @@ paths: schema: type: string + /w/{workspace}/jobs_u/get_flow_all_logs/{id}: + get: + summary: get all logs for a flow job + operationId: getFlowAllLogs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: concatenated logs of all flow steps + content: + text/plain: + schema: + type: string + /w/{workspace}/jobs_u/get_completed_logs_tail/{id}: get: summary: get completed job logs tail @@ -18355,7 +18607,7 @@ paths: type: object description: The columns used (for tables) additionalProperties: - $ref: "#/components/schemas/AssetUsageAccessType" + $ref: "#/components/schemas/AssetUsageAccessType" created_at: type: string format: date-time @@ -18459,7 +18711,6 @@ paths: type: string description: The asset path - /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -19914,8 +20165,9 @@ components: type: boolean timeout: type: integer - delete_after_use: - type: boolean + delete_after_secs: + type: integer + description: If set, delete the job's args, result and logs after this many seconds following job completion visible_to_runner_only: type: boolean auto_kind: @@ -20006,8 +20258,9 @@ components: type: boolean timeout: type: integer - delete_after_use: - type: boolean + delete_after_secs: + type: integer + description: If set, delete the job's args, result and logs after this many seconds following job completion deployment_message: type: string concurrency_key: @@ -24203,6 +24456,18 @@ components: type: string color: type: string + forked_datatables: + type: array + items: + type: object + required: [name, new_dbname] + properties: + name: + type: string + description: "Datatable name" + new_dbname: + type: string + description: "New database name for the fork" required: - id - name @@ -24854,7 +25119,14 @@ components: type: string required: - resource_type - + forked_from: + type: object + description: Fork origin info with schema snapshot + properties: + schema: + type: object + description: Schema snapshot at fork time + additionalProperties: true DataTableSchema: type: object required: [datatable_name, schemas] @@ -25436,7 +25708,15 @@ components: kind: type: string enum: - ["script", "flow", "app", "raw_app", "resource", "variable", "resource_type"] + [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "resource_type", + ] description: Type of the item path: type: string diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e9a8fa16b9..4d11b9aa3e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -378,6 +378,7 @@ pub fn workspace_unauthed_service() -> Router { .route("/get_root_job_id/{id}", get(get_root_job)) .route("/get/{id}", get(get_job)) .route("/get_logs/{id}", get(get_job_logs)) + .route("/get_flow_all_logs/{id}", get(get_flow_all_logs)) .route( "/get_completed_logs_tail/{id}", get(get_completed_job_logs_tail), @@ -1686,6 +1687,217 @@ async fn get_job_logs( } } +async fn resolve_logs_to_string( + log_offset: i32, + logs: &str, + log_file_index: &Option>, +) -> String { + use futures::StreamExt; + + #[cfg(all(feature = "enterprise", feature = "parquet"))] + if let Some(stream) = + windmill_object_store::get_logs_from_store(log_offset, logs, log_file_index).await + { + let mut result = String::new(); + futures::pin_mut!(stream); + while let Some(Ok(bytes)) = stream.next().await { + result.push_str(&String::from_utf8_lossy(&bytes)); + } + return result; + } + + if let Some(stream) = + windmill_common::jobs::get_logs_from_disk(log_offset, logs, log_file_index).await + { + let mut result = String::new(); + futures::pin_mut!(stream); + while let Some(Ok(bytes)) = stream.next().await { + result.push_str(&String::from_utf8_lossy(&bytes)); + } + return result; + } + + logs.to_string() +} + +async fn get_flow_all_logs( + OptAuthed(opt_authed): OptAuthed, + opt_tokened: OptTokened, + Extension(db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::Result { + let tags = opt_authed + .as_ref() + .map(|authed| get_scope_tags(authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec())) + .flatten(); + + // Verify the root job exists and check auth + let root_job = sqlx::query!( + "SELECT created_by FROM v2_job WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + w_id, + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_optional(&db) + .await?; + + let root_job = not_found_if_none(root_job, "Job", id.to_string())?; + + if opt_authed.is_none() && root_job.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); + } + + log_job_view( + &db, + opt_authed.as_ref(), + opt_tokened.token.as_deref(), + &w_id, + &id, + ) + .await?; + + // Fetch all jobs in the flow tree using recursive CTE. + // Uses a materialized id_path for depth-first ordering so children + // appear right after their parent (e.g. iteration 1 → its steps → iteration 2 → ...). + // Extracts parent_module_type (branchall, forloopflow, etc.) from parent's flow definition. + let records = sqlx::query!( + "WITH RECURSIVE job_tree AS ( + SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job, + '' as path_label, 0 as depth, + j.id::text as id_path, + ''::text as parent_module_type + FROM v2_job j + WHERE j.id = $2 AND j.workspace_id = $1 + UNION ALL + SELECT j.id, j.kind::text, j.flow_step_id, j.parent_job, + CASE + WHEN jt.path_label = '' THEN COALESCE(j.flow_step_id, '') + ELSE jt.path_label || '/' || COALESCE(j.flow_step_id, '') + END, + jt.depth + 1, + jt.id_path || '/' || j.id::text, + COALESCE(( + SELECT m->'value'->>'type' + FROM v2_job parent_j + LEFT JOIN flow f ON f.path = parent_j.runnable_path + AND f.workspace_id = parent_j.workspace_id + LEFT JOIN flow_node fn ON fn.id = parent_j.runnable_id + CROSS JOIN LATERAL jsonb_array_elements( + COALESCE(parent_j.raw_flow, f.value, fn.flow)->'modules' + ) m + WHERE parent_j.id = jt.id + AND m->>'id' = j.flow_step_id + LIMIT 1 + ), '')::text + FROM v2_job j + JOIN job_tree jt ON j.parent_job = jt.id + WHERE j.workspace_id = $1 + ), + with_sibling_index AS ( + SELECT jt.*, + ROW_NUMBER() OVER ( + PARTITION BY jt.parent_job, jt.flow_step_id + ORDER BY jt.id + ) as sibling_index, + COUNT(*) OVER ( + PARTITION BY jt.parent_job, jt.flow_step_id + ) as sibling_count + FROM job_tree jt + ) + SELECT w.id, w.kind, w.flow_step_id, w.path_label, + w.sibling_index::int as sibling_index, + w.sibling_count::int as sibling_count, + w.depth::int as depth, + w.parent_module_type, + coalesce(job_logs.logs, '') as logs, + COALESCE(job_logs.log_offset, 0) as log_offset, + job_logs.log_file_index + FROM with_sibling_index w + LEFT JOIN job_logs ON job_logs.job_id = w.id + ORDER BY w.id_path ASC", + w_id, + id, + ) + .fetch_all(&db) + .await?; + + let mut all_logs = String::new(); + + for record in &records { + let kind = record.kind.as_deref().unwrap_or(""); + let step_id = record.flow_step_id.as_deref().unwrap_or(""); + let depth = record.depth.unwrap_or(0); + let sibling_count = record.sibling_count.unwrap_or(1); + let sibling_index = record.sibling_index.unwrap_or(0); + let parent_module_type = record.parent_module_type.as_deref().unwrap_or(""); + + // Build a descriptive label + let path = record.path_label.as_deref().unwrap_or(step_id); + + let kind_label = match parent_module_type { + "branchall" => " branchall", + "branchone" => " branchone", + "forloopflow" => " forloop", + "whileloopflow" => " whileloop", + "aiagent" => " ai-agent", + _ => "", + }; + + let label = if depth == 0 { + "Flow".to_string() + } else if matches!( + kind, + "flow" | "flowpreview" | "flownode" | "singlestepflow" | "aiagent" + ) { + // Intermediate flow job (loop iteration or branch) + if parent_module_type == "branchone" { + let branch_label = if sibling_index == 1 { + "default".to_string() + } else { + format!("{}", sibling_index - 1) + }; + format!("Step {}{} (branch {})", path, kind_label, branch_label) + } else if parent_module_type == "branchall" { + format!("Step {}{} (branch {})", path, kind_label, sibling_index) + } else if parent_module_type == "forloopflow" || parent_module_type == "whileloopflow" { + format!( + "Step {}{} (iteration {}/{})", + path, kind_label, sibling_index, sibling_count + ) + } else if sibling_count > 1 { + format!( + "Step {} (iteration {}/{})", + path, sibling_index, sibling_count + ) + } else { + format!("Step {} (subflow)", path) + } + } else if sibling_count > 1 { + // Simple module optimization: forloop/whileloop with single step + // runs iterations as direct script jobs instead of subflows + format!( + "Step {}{} (iteration {}/{})", + path, kind_label, sibling_index, sibling_count + ) + } else { + format!("Step {}", path) + }; + + let job_id = record.id.map(|u| u.to_string()).unwrap_or_default(); + all_logs.push_str(&format!("\n=== {} (Job: {}) ===\n", label, job_id)); + let logs = record.logs.as_deref().unwrap_or(""); + let resolved = + resolve_logs_to_string(record.log_offset.unwrap_or(0), logs, &record.log_file_index) + .await; + all_logs.push_str(&resolved); + all_logs.push('\n'); + } + + Ok(content_plain(Body::from(all_logs))) +} + async fn get_args( OptAuthed(opt_authed): OptAuthed, opt_tokened: OptTokened, @@ -4069,49 +4281,51 @@ pub async fn run_workflow_as_code( ) .await?; - let (job_payload, tag, _delete_after_use, timeout, on_behalf_of) = match job.job_kind { - JobKind::Preview => ( - JobPayload::Code(RawCode { - hash: None, - content: raw_code.unwrap_or_default(), - path: job.script_path, - language: job.language.unwrap_or_else(|| ScriptLang::Deno), - lock: raw_lock, - concurrency_settings: concurrency_settings - .maybe_fallback( - windmill_queue::custom_concurrency_key(&db, &job.id) - .await - .map_err(to_anyhow)?, - job.concurrent_limit, - job.concurrency_time_window_s, - ) - .into(), - cache_ttl: job.cache_ttl, - cache_ignore_s3_path: job.cache_ignore_s3_path, - dedicated_worker: None, - // TODO(debouncing): enable for this mode - debouncing_settings: DebouncingSettings::default(), - modules: None, - }), - Some(job.tag.clone()), - None, - run_query.timeout, - None, - ), - JobKind::Script => { - let userdb_authed = - UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - script_path_to_payload( - job.script_path(), - Some(userdb_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await? - } - _ => return Err(anyhow::anyhow!("Not supported").into()), - }; + let (job_payload, tag, _delete_after_use, _delete_after_secs, timeout, on_behalf_of) = + match job.job_kind { + JobKind::Preview => ( + JobPayload::Code(RawCode { + hash: None, + content: raw_code.unwrap_or_default(), + path: job.script_path, + language: job.language.unwrap_or_else(|| ScriptLang::Deno), + lock: raw_lock, + concurrency_settings: concurrency_settings + .maybe_fallback( + windmill_queue::custom_concurrency_key(&db, &job.id) + .await + .map_err(to_anyhow)?, + job.concurrent_limit, + job.concurrency_time_window_s, + ) + .into(), + cache_ttl: job.cache_ttl, + cache_ignore_s3_path: job.cache_ignore_s3_path, + dedicated_worker: None, + // TODO(debouncing): enable for this mode + debouncing_settings: DebouncingSettings::default(), + modules: None, + }), + Some(job.tag.clone()), + None, + None, + run_query.timeout, + None, + ), + JobKind::Script => { + let userdb_authed = + UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + script_path_to_payload( + job.script_path(), + Some(userdb_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await? + } + _ => return Err(anyhow::anyhow!("Not supported").into()), + }; if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); @@ -4342,14 +4556,15 @@ pub async fn run_wait_result_job_by_path_get( let user_db_with_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let (job_payload, tag, delete_after_use, timeout, on_behalf_authed) = script_path_to_payload( - script_path, - Some(user_db_with_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await?; + let (job_payload, tag, delete_after_use, delete_after_secs, timeout, on_behalf_authed) = + script_path_to_payload( + script_path, + Some(user_db_with_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; @@ -4405,9 +4620,7 @@ pub async fn run_wait_result_job_by_path_get( tx.commit().await?; let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } + handle_delete_after_completion(&db, uuid, &w_id, delete_after_use, delete_after_secs).await?; return wait_result; } @@ -4487,14 +4700,15 @@ pub async fn run_wait_result_script_by_path_internal( check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = script_path_to_payload( - script_path.to_path(), - Some(db_authed), - db.clone(), - &w_id, - run_query.skip_preprocessor, - ) - .await?; + let (job_payload, tag, delete_after_use, delete_after_secs, timeout, on_behalf_of) = + script_path_to_payload( + script_path.to_path(), + Some(db_authed), + db.clone(), + &w_id, + run_query.skip_preprocessor, + ) + .await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; @@ -4550,9 +4764,7 @@ pub async fn run_wait_result_script_by_path_internal( tx.commit().await?; let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } + handle_delete_after_completion(&db, uuid, &w_id, delete_after_use, delete_after_secs).await?; return wait_result; } @@ -4590,6 +4802,7 @@ pub async fn run_wait_result_script_by_hash( dedicated_worker, priority, delete_after_use, + delete_after_secs, timeout, has_preprocessor, on_behalf_of_email, @@ -4676,9 +4889,7 @@ pub async fn run_wait_result_script_by_hash( tx.commit().await?; let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } + handle_delete_after_completion(&db, uuid, &w_id, delete_after_use, delete_after_secs).await?; return wait_result; } @@ -4848,7 +5059,7 @@ pub async fn stream_job( (uuid, None) } RunnableId::ScriptId(ScriptId::ScriptHash(script_hash)) => { - let (uuid, _) = run_job_by_hash_inner( + let (uuid, _, _) = run_job_by_hash_inner( authed.clone(), db.clone(), user_db, @@ -6331,7 +6542,7 @@ pub async fn run_job_by_hash( ) .await?; - let (uuid, _) = run_job_by_hash_inner( + let (uuid, _, _) = run_job_by_hash_inner( authed, db, user_db, @@ -6355,7 +6566,7 @@ pub async fn run_job_by_hash_inner( run_query: RunJobQuery, args: PushArgsOwned, trigger: Option, -) -> error::Result<(Uuid, Option)> { +) -> error::Result<(Uuid, Option, Option)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -6376,6 +6587,7 @@ pub async fn run_job_by_hash_inner( on_behalf_of_email, created_by, delete_after_use, + delete_after_secs, labels, .. } = get_script_info_for_hash(Some(userdb_authed), &db, &w_id, hash) @@ -6456,7 +6668,7 @@ pub async fn run_job_by_hash_inner( .await?; tx.commit().await?; - Ok((uuid, delete_after_use)) + Ok((uuid, delete_after_use, delete_after_secs)) } async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::Result { diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4c40753ae5..255223ad6d 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -1018,6 +1018,11 @@ pub async fn run_server( return Err(anyhow::anyhow!("Failed to send port, exiting early: {e:#}")); } + // Announce this server is ready so coordinated restarts can detect a healthy peer. + if let Err(e) = announce_server_started(&db).await { + tracing::warn!("Failed to announce server started: {e:#}"); + } + let server = server.with_graceful_shutdown(async move { killpill_rx.recv().await.ok(); #[cfg(feature = "agent_worker_server")] @@ -1158,3 +1163,52 @@ pub async fn wait_for_db_migrations( .await .map_err(|e| anyhow::anyhow!("Error waiting for db migrations: {e:#}")) } + +const SERVER_HEARTBEAT_TASK: &str = "server_heartbeat"; + +/// Write a server-started heartbeat to `background_task_state` so that +/// other instances waiting to restart can detect this server is healthy. +async fn announce_server_started(db: &DB) -> anyhow::Result<()> { + use windmill_common::INSTANCE_NAME; + + let instance = INSTANCE_NAME.as_str(); + + sqlx::query( + "INSERT INTO background_task_state (name, value, running, owner, started_at, updated_at) + VALUES ($1, '\"started\"'::jsonb, true, $2, NOW(), NOW()) + ON CONFLICT (name) + DO UPDATE SET updated_at = NOW(), running = true, owner = $2", + ) + .bind(format!("{SERVER_HEARTBEAT_TASK}:{instance}")) + .bind(instance) + .execute(db) + .await?; + + tracing::info!("Announced server started for instance {instance}"); + Ok(()) +} + +/// Check whether any server instance (other than ourselves) has announced +/// itself as started after `not_before` (i.e. after the restart was initiated). +pub async fn check_any_server_started(db: &DB, not_before: chrono::DateTime) -> bool { + use windmill_common::INSTANCE_NAME; + + let my_instance = INSTANCE_NAME.as_str(); + let prefix = format!("{SERVER_HEARTBEAT_TASK}:"); + + sqlx::query_scalar!( + "SELECT EXISTS( + SELECT 1 FROM background_task_state + WHERE name LIKE $1 + AND owner != $2 + AND running = true + AND updated_at > $3 + ) AS \"exists!\"", + format!("{prefix}%"), + my_instance, + not_before + ) + .fetch_one(db) + .await + .unwrap_or(false) +} diff --git a/backend/windmill-api/src/secret_backend_ext.rs b/backend/windmill-api/src/secret_backend_ext.rs index d38adbe14a..f76ba0ba64 100644 --- a/backend/windmill-api/src/secret_backend_ext.rs +++ b/backend/windmill-api/src/secret_backend_ext.rs @@ -217,7 +217,9 @@ async fn is_vault_backend_configured(db: &DB) -> Result { Ok(matches!( config, - SecretBackendConfig::HashiCorpVault(_) | SecretBackendConfig::AzureKeyVault(_) | SecretBackendConfig::AwsSecretsManager(_) + SecretBackendConfig::HashiCorpVault(_) + | SecretBackendConfig::AzureKeyVault(_) + | SecretBackendConfig::AwsSecretsManager(_) )) } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 7ef2f8d5a0..98d46b1421 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -84,7 +84,7 @@ struct ScriptMetadata { #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub delete_after_use: Option, + pub delete_after_secs: Option, #[serde(skip_serializing_if = "Option::is_none")] pub restart_unless_cancelled: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -508,7 +508,7 @@ pub(crate) async fn tarball_workspace( priority: script.priority, tag: script.tag, timeout: script.timeout, - delete_after_use: script.delete_after_use, + delete_after_secs: script.delete_after_secs, restart_unless_cancelled: script.restart_unless_cancelled, visible_to_runner_only: script.visible_to_runner_only, auto_kind: script.auto_kind, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 95e41c3bcd..1abdd2ed7f 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -66,6 +66,7 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; pub const INSTANCE_EVENTS_WEBHOOK_SETTING: &str = "instance_events_webhook"; pub const WORKSPACE_REGISTRIES_SETTING: &str = "workspace_registries"; pub const RESTART_COORDINATION_SETTING: &str = "_restart_coordination"; +pub const ALERT_CONFIG_SETTING: &str = "alert_job_queue_waiting"; use std::sync::Arc; use tokio::sync::RwLock; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index e8ed1e88b1..9972ac9fda 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -43,6 +43,39 @@ pub fn get_has_preprocessor_from_content_and_lang( Ok(has_preprocessor) } +pub async fn schedule_job_deletion( + db: &DB, + job_id: uuid::Uuid, + w_id: &str, + delete_after_secs: i32, +) -> crate::error::Result<()> { + sqlx::query!( + "INSERT INTO job_delete_schedule (job_id, workspace_id, delete_at) \ + VALUES ($1, $2, now() + make_interval(secs => $3::double precision)) \ + ON CONFLICT (job_id) DO NOTHING", + job_id, + w_id, + delete_after_secs as f64, + ) + .execute(db) + .await?; + Ok(()) +} + +/// Resolve effective delete behavior from delete_after_use (bool) and delete_after_secs. +/// Returns Some(secs) if deletion should happen, None otherwise. +pub fn resolve_delete_after_secs( + delete_after_use: Option, + delete_after_secs: Option, +) -> Option { + match (delete_after_use, delete_after_secs) { + (_, Some(secs)) if secs >= 0 => Some(secs), + (_, Some(_)) => None, // reject negative values + (Some(true), None) => Some(0), // backward compat: immediate + _ => None, + } +} + pub async fn script_path_to_payload<'e>( script_path: &str, db_authed: Option>>, @@ -54,88 +87,94 @@ pub async fn script_path_to_payload<'e>( Option, Option, Option, + Option, Option, )> { - let (job_payload, tag, delete_after_use, script_timeout, on_behalf_of) = if script_path - .starts_with("hub/") - { - let hub_script = - get_full_hub_script_by_path(StripPath(script_path.to_string()), &HTTP_CLIENT, None) - .await?; + let (job_payload, tag, delete_after_use, delete_after_secs, script_timeout, on_behalf_of) = + if script_path.starts_with("hub/") { + let hub_script = + get_full_hub_script_by_path(StripPath(script_path.to_string()), &HTTP_CLIENT, None) + .await?; - let has_preprocessor = - get_has_preprocessor_from_content_and_lang(&hub_script.content, &hub_script.language)?; + let has_preprocessor = get_has_preprocessor_from_content_and_lang( + &hub_script.content, + &hub_script.language, + )?; - ( - JobPayload::ScriptHub { - path: script_path.to_owned(), - apply_preprocessor: has_preprocessor && !skip_preprocessor.unwrap_or(false), - }, - None, - None, - None, - None, - ) - } else { - let ScriptHashInfo { - hash, - tag, - runnable_settings: - super::scripts::ScriptRunnableSettingsInline { - concurrency_settings, - debouncing_settings, + ( + JobPayload::ScriptHub { + path: script_path.to_owned(), + apply_preprocessor: has_preprocessor && !skip_preprocessor.unwrap_or(false), }, - cache_ttl, - cache_ignore_s3_path, - language, - dedicated_worker, - priority, - delete_after_use, - timeout, - has_preprocessor, - on_behalf_of_email, - created_by, - labels, - .. - } = get_latest_deployed_hash_for_path(db_authed, db.clone(), w_id, script_path) - .await? - .prefetch_cached(&db) - .await?; - - let on_behalf_of = if let Some(email) = on_behalf_of_email { - Some(OnBehalfOf { - email, - permissioned_as: username_to_permissioned_as(created_by.as_str()), - }) + None, + None, + None, + None, + None, + ) } else { - None - }; - - ( - JobPayload::ScriptHash { - hash: ScriptHash(hash), - path: script_path.to_owned(), + let ScriptHashInfo { + hash, + tag, + runnable_settings: + super::scripts::ScriptRunnableSettingsInline { + concurrency_settings, + debouncing_settings, + }, cache_ttl, cache_ignore_s3_path, language, dedicated_worker, priority, - apply_preprocessor: !skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - debouncing_settings, - concurrency_settings, + delete_after_use, + delete_after_secs, + timeout, + has_preprocessor, + on_behalf_of_email, + created_by, labels, - }, - tag, - delete_after_use, - timeout, - on_behalf_of, - ) - }; + .. + } = get_latest_deployed_hash_for_path(db_authed, db.clone(), w_id, script_path) + .await? + .prefetch_cached(&db) + .await?; + + let on_behalf_of = if let Some(email) = on_behalf_of_email { + Some(OnBehalfOf { + email, + permissioned_as: username_to_permissioned_as(created_by.as_str()), + }) + } else { + None + }; + + ( + JobPayload::ScriptHash { + hash: ScriptHash(hash), + path: script_path.to_owned(), + cache_ttl, + cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: !skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + debouncing_settings, + concurrency_settings, + labels, + }, + tag, + delete_after_use, + delete_after_secs, + timeout, + on_behalf_of, + ) + }; Ok(( job_payload, tag, delete_after_use, + delete_after_secs, script_timeout, on_behalf_of, )) @@ -146,7 +185,7 @@ pub async fn get_payload_tag_from_prefixed_path( db: &DB, w_id: &str, ) -> Result<(JobPayload, Option, Option), Error> { - let (payload, tag, _, _, on_behalf_of) = if path.starts_with("script/") { + let (payload, tag, _, _, _, on_behalf_of) = if path.starts_with("script/") { script_path_to_payload( path.strip_prefix("script/").unwrap(), None, @@ -170,6 +209,7 @@ pub async fn get_payload_tag_from_prefixed_path( None, None, None, + None, ) } else { let FlowVersionInfo { dedicated_worker, tag, version, labels, .. } = @@ -186,6 +226,7 @@ pub async fn get_payload_tag_from_prefixed_path( None, None, None, + None, ) } } else { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 42151553bb..fa48415f80 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -401,7 +401,7 @@ pub struct PrepareQueryResult { pub error: Option, } -#[derive(Deserialize, Serialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct PgDatabase { pub host: String, pub user: Option, @@ -632,6 +632,180 @@ impl PgDatabase { } } +/// Validate a database name to prevent SQL injection. +/// Must start with a letter, contain only alphanumeric characters or underscores, and be <= 63 chars. +pub fn validate_dbname(dbname: &str) -> error::Result<()> { + let dbname = dbname.trim(); + if dbname.is_empty() { + return Err(error::Error::BadRequest( + "Database name cannot be empty".to_string(), + )); + } + if dbname.len() > 63 { + return Err(error::Error::BadRequest( + "Database name cannot exceed 63 characters".to_string(), + )); + } + if !dbname + .chars() + .next() + .map_or(false, |c| c.is_ascii_alphabetic()) + { + return Err(error::Error::BadRequest( + "Database name must start with a letter".to_string(), + )); + } + if !dbname + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + { + return Err(error::Error::BadRequest( + "Database name must contain only alphanumeric characters or underscores".to_string(), + )); + } + Ok(()) +} + +/// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings. +pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> { + let dbname = dbname.trim(); + validate_dbname(dbname)?; + + let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + if wmill_pg_creds.dbname.trim().eq_ignore_ascii_case(dbname) { + return Err(error::Error::BadRequest( + "Cannot drop the main Windmill database".to_string(), + )); + } + + let db_exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if db_exists { + // Terminate active connections + if let Err(e) = sqlx::query(&format!( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()", + dbname.replace('\'', "''") + )) + .execute(db) + .await + { + tracing::warn!("Failed to terminate connections to '{}': {}", dbname, e); + } + + // Drop the database + sqlx::query(&format!("DROP DATABASE IF EXISTS \"{}\"", dbname)) + .execute(db) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to drop database '{}': {}", dbname, e)) + })?; + + tracing::info!("Dropped instance database '{}'", dbname); + } else { + tracing::info!("Database '{}' does not exist, skipping drop", dbname); + } + + // Always remove from global_settings + sqlx::query!( + r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#, + dbname + ) + .execute(db) + .await?; + + Ok(()) +} + +/// Create a custom instance database: CREATE DATABASE, grant permissions, register in global_settings. +/// The `tag` is stored in global_settings metadata (e.g. "datatable" or "ducklake"). +pub async fn create_custom_instance_database( + db: &DB, + dbname: &str, + tag: &str, +) -> error::Result<()> { + let dbname = dbname.trim(); + validate_dbname(dbname)?; + + let db_exists = sqlx::query_scalar!( + "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_database WHERE datname = $1)", + dbname + ) + .fetch_one(db) + .await? + .unwrap_or(false); + + if db_exists { + return Err(error::Error::BadRequest(format!( + "Database '{}' already exists", + dbname + ))); + } + + sqlx::query(&format!("CREATE DATABASE \"{}\"", dbname)) + .execute(db) + .await + .map_err(|e| { + error::Error::internal_err(format!("Failed to create database '{}': {}", dbname, e)) + })?; + + // Grant permissions to custom_instance_user + let wmill_pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?; + let new_pg_creds = PgDatabase { dbname: dbname.to_string(), ..wmill_pg_creds }; + let (client, connection) = new_pg_creds.connect().await?; + let join_handle = tokio::spawn(async move { connection.await }); + + if let Err(e) = client + .batch_execute(&format!( + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; + GRANT USAGE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON DATABASE \"{dbname}\" TO custom_instance_user; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" + )) + .await + { + tracing::warn!( + "Failed to grant permissions on '{}': {}. Continuing.", + dbname, + e + ); + } + + drop(client); + join_handle + .await + .map_err(|e| error::Error::internal_err(format!("join error: {}", e)))? + .map_err(|e| error::Error::internal_err(format!("tokio_postgres error: {}", e)))?; + + // Register in global_settings + let status_json = serde_json::json!({ + "logs": { + "created_database": "OK", + "db_connect": "OK", + "grant_permissions": "OK" + }, + "success": true, + "error": null, + "tag": tag + }); + sqlx::query!( + r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (COALESCE(value->'databases', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, + serde_json::json!({ (dbname): status_json }) + ) + .execute(db) + .await?; + + tracing::info!("Created custom instance database '{}'", dbname); + Ok(()) +} + #[derive(Clone)] pub enum DatabaseUrl { #[cfg(all(feature = "enterprise", feature = "private"))] @@ -863,6 +1037,7 @@ pub struct ScriptHashInfo { pub dedicated_worker: Option, pub priority: Option, pub delete_after_use: Option, + pub delete_after_secs: Option, pub timeout: Option, pub has_preprocessor: Option, pub on_behalf_of_email: Option, @@ -893,6 +1068,7 @@ impl ScriptHashInfo { dedicated_worker: self.dedicated_worker, priority: self.priority, delete_after_use: self.delete_after_use, + delete_after_secs: self.delete_after_secs, timeout: self.timeout, has_preprocessor: self.has_preprocessor, on_behalf_of_email: self.on_behalf_of_email, @@ -1062,6 +1238,7 @@ async fn get_script_info_for_hash_inner<'e, E: sqlx::PgExecutor<'e>>( dedicated_worker, priority, delete_after_use, + delete_after_secs, timeout, has_preprocessor, on_behalf_of_email, diff --git a/backend/windmill-common/src/query_builders.rs b/backend/windmill-common/src/query_builders.rs index d28afb22ba..1c06217110 100644 --- a/backend/windmill-common/src/query_builders.rs +++ b/backend/windmill-common/src/query_builders.rs @@ -394,12 +394,29 @@ pub struct BreakingFeatures { // Helper functions // --------------------------------------------------------------------------- +fn normalize_pg_type(typ: &str) -> &str { + match typ { + "double precision" => "float8", + "character varying" => "varchar", + "time with time zone" => "timetz", + "time without time zone" => "time", + "timestamp with time zone" => "timestamptz", + "timestamp without time zone" => "timestamp", + other => other, + } +} + pub fn build_parameters(columns: &[SimpleColumn], db_type: DbType) -> String { columns .iter() .enumerate() .map(|(i, col)| { let base_type = col.datatype.split('(').next().unwrap_or(&col.datatype); + let base_type = if db_type == DbType::Postgresql { + normalize_pg_type(base_type) + } else { + base_type + }; match db_type { DbType::Postgresql => format!("-- ${} {} ({})", i + 1, col.field, base_type), DbType::Mysql => format!("-- :{} ({})", col.field, base_type), @@ -4471,3 +4488,224 @@ mod tests { assert_eq!(sql, "SHOW PRIMARY KEYS IN ACCOUNT"); } } + +// ============================================================================ +// Full schema introspection types and logic (used by get_datatable_full_schema) +// ============================================================================ + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaColumn { + pub name: String, + pub datatype: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub primary_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub nullable: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaForeignKeyColumn { + #[serde(skip_serializing_if = "Option::is_none")] + pub source_column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub target_column: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaForeignKey { + #[serde(skip_serializing_if = "Option::is_none")] + pub target_table: Option, + pub columns: Vec, + pub on_delete: String, + pub on_update: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub fk_constraint_name: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct FullSchemaTable { + pub name: String, + pub columns: Vec, + pub foreign_keys: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub pk_constraint_name: Option, +} + +/// Full database schema: { schema_name: { table_name: FullSchemaTable } } +pub type FullDatabaseSchema = + std::collections::HashMap>; + +fn pg_action_to_string(action: &str) -> String { + match action { + "a" => "NO ACTION".to_string(), + "r" => "RESTRICT".to_string(), + "c" => "CASCADE".to_string(), + "n" => "SET NULL".to_string(), + "d" => "SET DEFAULT".to_string(), + _ => "NO ACTION".to_string(), + } +} + +/// Introspect a PostgreSQL database and return the full schema. +/// Takes a connected tokio_postgres Client. +pub async fn pg_get_full_schema( + client: &tokio_postgres::Client, +) -> Result { + let column_rows = client + .query( + "SELECT + ns.nspname AS schema_name, + c.relname AS table_name, + a.attname AS column_name, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS datatype, + (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128) + FROM pg_catalog.pg_attrdef d + WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default_value, + CASE a.attnotnull WHEN false THEN true ELSE false END AS nullable, + EXISTS ( + SELECT 1 FROM pg_catalog.pg_index i + WHERE i.indrelid = c.oid AND i.indisprimary AND a.attnum = ANY(i.indkey) + ) AS is_primary_key, + (SELECT con.conname FROM pg_catalog.pg_constraint con + WHERE con.conrelid = c.oid AND con.contype = 'p' LIMIT 1) AS pk_constraint_name + FROM pg_catalog.pg_attribute a + JOIN pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid + WHERE c.relkind = 'r' + AND a.attnum > 0 + AND NOT a.attisdropped + AND ns.nspname NOT IN ('pg_catalog', 'information_schema') + ORDER BY ns.nspname, c.relname, a.attnum", + &[], + ) + .await + .map_err(|e| format!("Failed to query columns: {}", e))?; + + let fk_rows = client + .query( + "SELECT + ns.nspname AS schema_name, + c.relname AS table_name, + con.conname AS fk_constraint_name, + att_src.attname AS source_column, + ns_ref.nspname AS ref_schema, + c_ref.relname AS ref_table, + att_ref.attname AS ref_column, + con.confdeltype::text AS on_delete, + con.confupdtype::text AS on_update + FROM pg_catalog.pg_constraint con + JOIN pg_catalog.pg_class c ON con.conrelid = c.oid + JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid + JOIN pg_catalog.pg_class c_ref ON con.confrelid = c_ref.oid + JOIN pg_catalog.pg_namespace ns_ref ON c_ref.relnamespace = ns_ref.oid + CROSS JOIN LATERAL unnest(con.conkey, con.confkey) WITH ORDINALITY AS u(src_attnum, ref_attnum, ord) + JOIN pg_catalog.pg_attribute att_src ON att_src.attrelid = c.oid AND att_src.attnum = u.src_attnum + JOIN pg_catalog.pg_attribute att_ref ON att_ref.attrelid = c_ref.oid AND att_ref.attnum = u.ref_attnum + WHERE con.contype = 'f' + AND ns.nspname NOT IN ('pg_catalog', 'information_schema') + ORDER BY ns.nspname, c.relname, con.conname, u.ord", + &[], + ) + .await + .map_err(|e| format!("Failed to query foreign keys: {}", e))?; + + let mut result: FullDatabaseSchema = std::collections::HashMap::new(); + + for row in &column_rows { + let schema_name: &str = row.get("schema_name"); + let table_name: &str = row.get("table_name"); + let column_name: &str = row.get("column_name"); + let datatype: &str = row.get("datatype"); + let default_value: Option<&str> = row.get("default_value"); + let nullable: bool = row.get("nullable"); + let is_primary_key: bool = row.get("is_primary_key"); + let pk_constraint_name: Option<&str> = row.get("pk_constraint_name"); + + let schema_tables = result.entry(schema_name.to_string()).or_default(); + let table = schema_tables + .entry(table_name.to_string()) + .or_insert_with(|| FullSchemaTable { + name: table_name.to_string(), + columns: vec![], + foreign_keys: vec![], + pk_constraint_name: pk_constraint_name.map(|s| s.to_string()), + }); + + table.columns.push(FullSchemaColumn { + name: column_name.to_string(), + datatype: datatype.to_string(), + primary_key: if is_primary_key { Some(true) } else { None }, + default_value: default_value.map(|s| s.to_string()), + nullable: Some(nullable), + }); + } + + let mut fk_map: std::collections::HashMap< + (String, String, String), + ( + Option, + Vec, + String, + String, + ), + > = std::collections::HashMap::new(); + + for row in &fk_rows { + let schema_name: &str = row.get("schema_name"); + let table_name: &str = row.get("table_name"); + let fk_name: &str = row.get("fk_constraint_name"); + let source_column: &str = row.get("source_column"); + let ref_schema: &str = row.get("ref_schema"); + let ref_table: &str = row.get("ref_table"); + let ref_column: &str = row.get("ref_column"); + let on_delete: &str = row.get("on_delete"); + let on_update: &str = row.get("on_update"); + + let target_table = if ref_schema == schema_name { + ref_table.to_string() + } else { + format!("{}.{}", ref_schema, ref_table) + }; + + let key = ( + schema_name.to_string(), + table_name.to_string(), + fk_name.to_string(), + ); + let entry = fk_map.entry(key).or_insert_with(|| { + ( + Some(target_table.clone()), + vec![], + pg_action_to_string(on_delete), + pg_action_to_string(on_update), + ) + }); + entry.1.push(FullSchemaForeignKeyColumn { + source_column: Some(source_column.to_string()), + target_column: Some(ref_column.to_string()), + }); + } + + let mut fk_entries: Vec<_> = fk_map.into_iter().collect(); + fk_entries.sort_by(|a, b| a.0.cmp(&b.0)); + + for ((schema_name, table_name, fk_name), (target_table, columns, on_delete, on_update)) in + fk_entries + { + if let Some(schema_tables) = result.get_mut(&schema_name) { + if let Some(table) = schema_tables.get_mut(&table_name) { + table.foreign_keys.push(FullSchemaForeignKey { + target_table, + columns, + on_delete, + on_update, + fk_constraint_name: Some(fk_name), + }); + } + } + } + + Ok(result) +} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 26b3c24730..6d886b6eca 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -106,6 +106,7 @@ pub async fn prefetch_cached_script( cache_ignore_s3_path: script.cache_ignore_s3_path, timeout: script.timeout, delete_after_use: script.delete_after_use, + delete_after_secs: script.delete_after_secs, restart_unless_cancelled: script.restart_unless_cancelled, visible_to_runner_only: script.visible_to_runner_only, auto_kind: script.auto_kind, @@ -346,6 +347,7 @@ pub async fn fetch_script_for_update<'a>( cache_ignore_s3_path, timeout, delete_after_use, + delete_after_secs, restart_unless_cancelled, visible_to_runner_only, auto_kind, @@ -420,6 +422,7 @@ pub async fn clone_script<'c>( priority: s.priority, timeout: s.timeout, delete_after_use: s.delete_after_use, + delete_after_secs: s.delete_after_secs, restart_unless_cancelled: s.restart_unless_cancelled, deployment_message, visible_to_runner_only: s.visible_to_runner_only, @@ -449,14 +452,14 @@ pub async fn clone_script<'c>( created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ + delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels) SELECT workspace_id, $1, path, array_prepend($2::bigint, COALESCE(parent_hashes, '{}'::bigint[])), summary, description, \ content, created_by, schema, is_template, extra_perms, NULL, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, cache_ignore_s3_path, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ + delete_after_use, delete_after_secs, timeout, concurrency_key, visible_to_runner_only, auto_kind, \ codebase, has_preprocessor, on_behalf_of_email, schema_validation, assets, debounce_key, debounce_delay_s, runnable_settings_handle, modules, labels FROM script WHERE hash = $2 AND workspace_id = $3; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 87d2b8917d..4f1e1fbea2 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -389,9 +389,32 @@ pub async fn check_user_against_rule( Ok(RuleCheckResult::Allowed) } +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DataTableForkBehavior { + SchemaOnly, + SchemaAndData, + KeepOriginal, +} + +impl Default for DataTableForkBehavior { + fn default() -> Self { + DataTableForkBehavior::KeepOriginal + } +} + #[derive(Deserialize, Serialize, Debug)] pub struct DataTable { pub database: DataTableDatabase, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub forked_from: Option, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct DataTableForkedFrom { + /// Schema snapshot at fork time + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schema: Option, } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index dfc254476b..a7cb9e66af 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3638,11 +3638,22 @@ pub fn resolve_debounce_key<'b>( .join(":"), )); - tracing::debug!("Original debounce key: {}", original_debounce_key); + tracing::debug!("Original debounce key (len={}): {}", original_debounce_key.len(), original_debounce_key); - // If debounce_key is not too long (< 255 chars), keep it as is, otherwise hash it + + // If debounce_key is not too long (< 255 chars), keep it as is, otherwise hash it. + // On cloud, we prepend "{workspace_id}:" so we must reserve space for that prefix + // to avoid exceeding the VARCHAR(255) column limit. const MAX_DEBOUNCE_KEY_LENGTH: usize = 255; - let resolved = if original_debounce_key.len() <= MAX_DEBOUNCE_KEY_LENGTH { + + #[cfg(feature = "cloud")] + let prefix = format!("{workspace_id}:"); + #[cfg(not(feature = "cloud"))] + let prefix = String::new(); + + let max_key_length = MAX_DEBOUNCE_KEY_LENGTH - prefix.len(); + + let resolved = if original_debounce_key.len() <= max_key_length { original_debounce_key } else { let hash = calculate_hash(&original_debounce_key); @@ -3651,13 +3662,16 @@ pub fn resolve_debounce_key<'b>( original_debounce_key.len(), hash ); - hash + if hash.len() > max_key_length { + hash[..max_key_length].to_string() + } else { + hash + } }; - #[cfg(feature = "cloud")] - let resolved = format!("{workspace_id}:{resolved}"); + let resolved = format!("{prefix}{resolved}"); - tracing::debug!("Final debounce key: {}", resolved); + tracing::debug!("Final debounce key (len={}): {}", resolved.len(), resolved); resolved } @@ -5203,6 +5217,8 @@ async fn push_inner<'c, 'd>( preprocessor_module: None, chat_input_enabled: None, flow_env: None, + delete_after_use: None, + delete_after_secs: None, }; // this is a new flow being pushed, flow_status is set to flow_value: let flow_status: FlowStatus = FlowStatus::new(&flow_value); diff --git a/backend/windmill-runtime-nativets/src/windmill-client.js b/backend/windmill-runtime-nativets/src/windmill-client.js index b0816dd78c..432a6e4b7f 100644 --- a/backend/windmill-runtime-nativets/src/windmill-client.js +++ b/backend/windmill-runtime-nativets/src/windmill-client.js @@ -253,8 +253,8 @@ var $Script = { timeout: { type: "integer", }, - delete_after_use: { - type: "boolean", + delete_after_secs: { + type: "integer", }, visible_to_runner_only: { type: "boolean", @@ -369,8 +369,8 @@ var $NewScript = { timeout: { type: "integer", }, - delete_after_use: { - type: "boolean", + delete_after_secs: { + type: "integer", }, deployment_message: { type: "string", @@ -2830,8 +2830,8 @@ var $FlowModule = { timeout: { type: "number", }, - delete_after_use: { - type: "boolean", + delete_after_secs: { + type: "integer", }, summary: { type: "string", diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 42649129d0..94617f85ec 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -155,7 +155,6 @@ pub struct CreateResource { pub value: Option>, pub description: Option, pub resource_type: String, - #[serde(default)] pub labels: Option>, } #[derive(Deserialize)] @@ -951,6 +950,14 @@ async fn delete_resource( q.fetch_all(&mut *tx).await? }; + sqlx::query!( + "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2", + w_id, + path + ) + .execute(&mut *tx) + .await?; + let deleted_path = sqlx::query_scalar!( "DELETE FROM resource WHERE path = $1 AND workspace_id = $2 RETURNING path", path, @@ -1126,6 +1133,14 @@ async fn delete_resources_bulk( } } + sqlx::query!( + "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = ANY($2)", + w_id, + &request.paths + ) + .execute(&mut *tx) + .await?; + let deleted_paths = sqlx::query_scalar!( "DELETE FROM resource WHERE path = ANY($1) AND workspace_id = $2 RETURNING path", &request.paths, @@ -1270,6 +1285,15 @@ async fn update_resource( ) .execute(&mut *tx) .await?; + + sqlx::query!( + "UPDATE ws_specific SET path = $1 WHERE workspace_id = $2 AND item_kind = 'resource' AND path = $3", + npath, + w_id, + path + ) + .execute(&mut *tx) + .await?; } } diff --git a/backend/windmill-test-utils/src/lib.rs b/backend/windmill-test-utils/src/lib.rs index 4e1534c777..b2e3be2cef 100644 --- a/backend/windmill-test-utils/src/lib.rs +++ b/backend/windmill-test-utils/src/lib.rs @@ -753,7 +753,7 @@ pub async fn assert_lockfile( schema: std::collections::HashMap::new(), ws_error_handler_muted: Some(false), priority: None, - delete_after_use: None, + delete_after_secs: None, timeout: None, restart_unless_cancelled: None, deployment_message: None, @@ -851,7 +851,7 @@ pub async fn run_deployed_relative_imports( schema: std::collections::HashMap::new(), ws_error_handler_muted: Some(false), priority: None, - delete_after_use: None, + delete_after_secs: None, timeout: None, restart_unless_cancelled: None, deployment_message: None, diff --git a/backend/windmill-trigger/src/trigger_helpers.rs b/backend/windmill-trigger/src/trigger_helpers.rs index b50cdb69a3..b89699a169 100644 --- a/backend/windmill-trigger/src/trigger_helpers.rs +++ b/backend/windmill-trigger/src/trigger_helpers.rs @@ -31,7 +31,7 @@ use windmill_api_jobs::{ execution::{ check_tag_available_for_workspace, delete_job_metadata_after_use, push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response, - run_wait_result_internal, + run_wait_result_internal, schedule_job_deletion, }, types::RunJobQuery, }; @@ -522,7 +522,7 @@ pub async fn trigger_runnable_inner<'c>( suspended_mode: Option, ) -> Result<( Uuid, - Option, + Option, Option, Option>, )> { @@ -536,7 +536,7 @@ pub async fn trigger_runnable_inner<'c>( }); let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); - let (uuid, delete_after_use, early_return, tx_out) = if is_flow { + let (uuid, resolved_delete_secs, early_return, tx_out) = if is_flow { let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; let path = StripPath(runnable_path.to_string()); let (uuid, early_return, tx_out) = push_flow_job_by_path_into_queue( @@ -553,7 +553,7 @@ pub async fn trigger_runnable_inner<'c>( .await?; (uuid, None, early_return, tx_out) } else { - let (uuid, delete_after_use, tx_out) = trigger_script_internal( + let (uuid, resolved_delete_secs, tx_out) = trigger_script_internal( db, tx_o, user_db, @@ -570,10 +570,10 @@ pub async fn trigger_runnable_inner<'c>( suspended_mode, ) .await?; - (uuid, delete_after_use, None, tx_out) + (uuid, resolved_delete_secs, None, tx_out) }; - Ok((uuid, delete_after_use, early_return, tx_out)) + Ok((uuid, resolved_delete_secs, early_return, tx_out)) } #[allow(dead_code)] @@ -631,7 +631,7 @@ pub async fn trigger_runnable_and_wait_for_result( trigger: TriggerMetadata, ) -> Result { let username = authed.username.clone(); - let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( + let (uuid, resolved_delete_secs, early_return, _) = trigger_runnable_inner( db, None, user_db, @@ -652,8 +652,10 @@ pub async fn trigger_runnable_and_wait_for_result( let (result, success) = run_wait_result_internal(db, uuid, &workspace_id, early_return, &username).await?; - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; + match resolved_delete_secs { + Some(0) => delete_job_metadata_after_use(&db, uuid).await?, + Some(secs) => schedule_job_deletion(&db, uuid, &workspace_id, secs).await?, + None => {} } result_to_response(result, success) @@ -675,7 +677,7 @@ pub async fn trigger_runnable_and_wait_for_raw_result( trigger: TriggerMetadata, ) -> Result<(Box, bool)> { let username = authed.username.clone(); - let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( + let (uuid, resolved_delete_secs, early_return, _) = trigger_runnable_inner( db, None, user_db, @@ -705,8 +707,10 @@ pub async fn trigger_runnable_and_wait_for_raw_result( ) })?; - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; + match resolved_delete_secs { + Some(0) => delete_job_metadata_after_use(&db, uuid).await?, + Some(secs) => schedule_job_deletion(&db, uuid, &workspace_id, secs).await?, + None => {} } Ok((result, success)) @@ -770,13 +774,13 @@ async fn trigger_script_internal<'c>( suspended_mode: Option, ) -> Result<( Uuid, - Option, + Option, Option>, )> { if retry.is_none() && error_handler_path.is_none() { let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; let path = StripPath(script_path.to_string()); - let (uuid, delete_after_use, tx_out) = push_script_job_by_path_into_queue( + let (uuid, resolved_delete_secs, tx_out) = push_script_job_by_path_into_queue( authed, db.clone(), tx_o, @@ -788,9 +792,9 @@ async fn trigger_script_internal<'c>( Some(trigger), ) .await?; - Ok((uuid, delete_after_use, tx_out)) + Ok((uuid, resolved_delete_secs, tx_out)) } else { - let (uuid, delete_after_use, tx_out) = trigger_script_with_retry_and_error_handler( + let (uuid, resolved_delete_secs, tx_out) = trigger_script_with_retry_and_error_handler( db, tx_o, user_db, @@ -807,7 +811,7 @@ async fn trigger_script_internal<'c>( suspended_mode, ) .await?; - Ok((uuid, delete_after_use, tx_out)) + Ok((uuid, resolved_delete_secs, tx_out)) } } @@ -828,7 +832,7 @@ async fn trigger_script_with_retry_and_error_handler<'c>( suspended_mode: Option, ) -> Result<( Uuid, - Option, + Option, Option>, )> { #[cfg(feature = "enterprise")] @@ -840,7 +844,7 @@ async fn trigger_script_with_retry_and_error_handler<'c>( let error_handler_path = error_handler_path.map(|p| p.to_string()); let error_handler_args = error_handler_args.map(|args| args.0.clone()); - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { + let (job_payload, tag, delete_after_use, delete_after_secs, timeout, on_behalf_of) = { let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; script_path_to_payload( script_path, @@ -851,6 +855,10 @@ async fn trigger_script_with_retry_and_error_handler<'c>( ) .await? }; + let resolved_delete_secs = windmill_api_jobs::execution::resolve_delete_after_secs( + delete_after_use, + delete_after_secs, + ); check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; @@ -951,9 +959,9 @@ async fn trigger_script_with_retry_and_error_handler<'c>( // If we were given a transaction, return it; otherwise commit it if return_tx { - Ok((uuid, delete_after_use, Some(tx))) + Ok((uuid, resolved_delete_secs, Some(tx))) } else { tx.commit().await?; - Ok((uuid, delete_after_use, None)) + Ok((uuid, resolved_delete_secs, None)) } } diff --git a/backend/windmill-types/src/flow_status.rs b/backend/windmill-types/src/flow_status.rs index 6306141ab9..d4fb637701 100644 --- a/backend/windmill-types/src/flow_status.rs +++ b/backend/windmill-types/src/flow_status.rs @@ -111,6 +111,15 @@ pub struct FlowCleanupModule { #[serde(default)] #[serde(skip_serializing_if = "Vec::is_empty")] pub flow_jobs_to_clean: Vec, + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + pub flow_jobs_to_schedule_clean: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FlowJobScheduledClean { + pub id: Uuid, + pub delete_after_secs: i32, } #[derive(Deserialize, Serialize, Debug, Clone)] @@ -513,7 +522,10 @@ impl FlowStatus { } else { None }, - cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, + cleanup_module: FlowCleanupModule { + flow_jobs_to_clean: vec![], + flow_jobs_to_schedule_clean: vec![], + }, retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, restarted_from: None, user_states: HashMap::new(), diff --git a/backend/windmill-types/src/flows.rs b/backend/windmill-types/src/flows.rs index 40d3a13784..14bcf6d8d5 100644 --- a/backend/windmill-types/src/flows.rs +++ b/backend/windmill-types/src/flows.rs @@ -184,6 +184,10 @@ pub struct FlowValue { pub chat_input_enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] pub flow_env: Option>>, + #[serde(skip_serializing, default)] + pub delete_after_use: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_secs: Option, } impl FlowValue { @@ -441,9 +445,11 @@ pub struct FlowModule { pub timeout: Option, #[serde(skip_serializing_if = "Option::is_none")] pub priority: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing, default)] pub delete_after_use: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub continue_on_error: Option, #[serde(skip_serializing_if = "Option::is_none")] pub skip_if: Option, @@ -1122,6 +1128,7 @@ pub fn add_virtual_items_if_necessary(modules: &mut Vec) { timeout: None, priority: None, delete_after_use: None, + delete_after_secs: None, continue_on_error: None, skip_if: None, apply_preprocessor: None, diff --git a/backend/windmill-types/src/scripts.rs b/backend/windmill-types/src/scripts.rs index c7ffe057e5..7bf19e6147 100644 --- a/backend/windmill-types/src/scripts.rs +++ b/backend/windmill-types/src/scripts.rs @@ -357,9 +357,11 @@ pub struct Script { pub cache_ignore_s3_path: Option, #[serde(skip_serializing_if = "Option::is_none")] pub timeout: Option, - #[serde(skip_serializing_if = "Option::is_none")] + #[serde(skip_serializing, default)] pub delete_after_use: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub delete_after_secs: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub restart_unless_cancelled: Option, #[serde(skip_serializing_if = "Option::is_none")] pub visible_to_runner_only: Option, @@ -505,7 +507,9 @@ pub struct NewScript { pub ws_error_handler_muted: Option, pub priority: Option, pub timeout: Option, + #[serde(skip_serializing, default)] pub delete_after_use: Option, + pub delete_after_secs: Option, pub restart_unless_cancelled: Option, pub deployment_message: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 453d6e0e25..e906cda033 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -391,6 +391,7 @@ async fn execute_windmill_tool( tool_module, tag, tool_module.delete_after_use.unwrap_or(false), + None, ) } FlowModuleValue::FlowScript { id, language, concurrency_settings, tag, .. } => { @@ -407,6 +408,7 @@ async fn execute_windmill_tool( }, tag: tag.clone(), delete_after_use: tool_module.delete_after_use.unwrap_or(false), + delete_after_secs: None, timeout: None, on_behalf_of: None, } @@ -429,6 +431,7 @@ async fn execute_windmill_tool( payload: JobPayload::AIAgent { path }, tag: None, delete_after_use: tool_module.delete_after_use.unwrap_or(false), + delete_after_secs: None, timeout: None, on_behalf_of: None, } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index b3c74c5d63..7e2d85f745 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -2590,7 +2590,7 @@ pub async fn handle_wac_v2_output( } "script" => { // Resolve script path to job payload (handles hash, lang, etc.) - let (payload, _, _, _, _) = script_path_to_payload( + let (payload, _, _, _, _, _) = script_path_to_payload( &step.script, None, // no authed db for background workers db.clone(), diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 5aab50c3f9..2e29e4ef89 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -103,7 +103,7 @@ fn otyp_to_pg_type(otyp: &str) -> error::Result { "int" | "integer" | "int4" | "serial" => (Type::INT4, Type::INT4_ARRAY), "bigint" | "bigserial" | "int8" | "serial8" => (Type::INT8, Type::INT8_ARRAY), "real" | "float4" => (Type::FLOAT4, Type::FLOAT4_ARRAY), - "double" | "float8" => (Type::FLOAT8, Type::FLOAT8_ARRAY), + "double" | "double precision" | "float8" => (Type::FLOAT8, Type::FLOAT8_ARRAY), "numeric" | "decimal" => (Type::NUMERIC, Type::NUMERIC_ARRAY), "text" => (Type::TEXT, Type::TEXT_ARRAY), "varchar" | "character varying" => (Type::VARCHAR, Type::VARCHAR_ARRAY), @@ -633,7 +633,9 @@ fn convert_vec_val( "real" | "float4" => Ok(Box::new(map_as_single_type(vec, |v| { v.as_f64().map(|x| x as f32) })?)), - "double" | "float8" => Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)), + "double" | "double precision" | "float8" => { + Ok(Box::new(map_as_single_type(vec, |v| v.as_f64())?)) + } "uuid" => Ok(Box::new(map_as_single_type(vec, |v| { v.as_str().map(|x| Uuid::parse_str(x).ok()).flatten() })?)), @@ -697,7 +699,7 @@ fn convert_val( "oid" => Ok(Box::new(None::)), "bigint" | "bigserial" | "int8" | "serial8" => Ok(Box::new(None::)), "real" | "float4" => Ok(Box::new(None::)), - "double" | "float8" => Ok(Box::new(None::)), + "double" | "double precision" | "float8" => Ok(Box::new(None::)), "uuid" => Ok(Box::new(None::)), "date" => Ok(Box::new(None::)), "time" | "timetz" => Ok(Box::new(None::)), @@ -731,7 +733,10 @@ fn convert_val( Value::Number(n) if (arg_t == "real" || arg_t == "float4") && n.as_f64().is_some() => { Ok(Box::new(n.as_f64().unwrap() as f32)) } - Value::Number(n) if (arg_t == "double" || arg_t == "float8") && n.as_f64().is_some() => { + Value::Number(n) + if (arg_t == "double" || arg_t == "double precision" || arg_t == "float8") + && n.as_f64().is_some() => + { Ok(Box::new(n.as_f64().unwrap())) } Value::Number(n) if (arg_t == "numeric" || arg_t == "decimal") && n.is_i64() => Ok( diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index baadad65c4..8505cd71d4 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1651,13 +1651,33 @@ pub async fn update_flow_status_after_job_completion_internal( .warn_after_seconds(3) .await?; } + if !_cleanup_module.flow_jobs_to_schedule_clean.is_empty() { + let entries_json = serde_json::to_value( + &_cleanup_module.flow_jobs_to_schedule_clean, + ) + .map_err(|e| { + error::Error::internal_err(format!( + "Unable to serialize scheduled clean entries: {e:#}" + )) + })?; + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_schedule_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_schedule_clean', '[]'::jsonb) || $1) + WHERE id = $2", + entries_json, + parent_job + ) + .execute(db) + .warn_after_seconds(3) + .await?; + } } else { // run the cleanup step only when the root job is complete if !_cleanup_module.flow_jobs_to_clean.is_empty() { tracing::debug!( - "Cleaning up jobs arguments, result and logs as they were marked as delete_after_use {:?}", - _cleanup_module.flow_jobs_to_clean - ); + "Cleaning up jobs arguments, result and logs as they were marked as delete_after_use {:?}", + _cleanup_module.flow_jobs_to_clean + ); sqlx::query!( "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", &_cleanup_module.flow_jobs_to_clean, @@ -1677,6 +1697,92 @@ pub async fn update_flow_status_after_job_completion_internal( Error::internal_err(format!("error while cleaning up completed job: {e:#}")) })?; } + + // Process scheduled deletions — insert into job_delete_schedule + if !_cleanup_module.flow_jobs_to_schedule_clean.is_empty() { + let w_id = &flow_job.workspace_id; + for entry in &_cleanup_module.flow_jobs_to_schedule_clean { + windmill_common::jobs::schedule_job_deletion( + db, + entry.id, + w_id, + entry.delete_after_secs, + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "error scheduling job deletion for {}: {e:#}", + entry.id + )) + })?; + } + } + + // Flow-level delete_after_secs: apply to ALL child jobs + the parent flow job + let flow_value = flow_data.value(); + let flow_delete_secs = windmill_common::jobs::resolve_delete_after_secs( + flow_value.delete_after_use, + flow_value.delete_after_secs, + ); + if let Some(secs) = flow_delete_secs { + let w_id = &flow_job.workspace_id; + let mut all_ids: Vec = + sqlx::query_scalar!("SELECT id FROM v2_job WHERE root_job = $1", flow_job.id) + .fetch_all(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error fetching child jobs for flow-level deletion: {e:#}" + )) + })?; + all_ids.push(flow_job.id); + + if secs == 0 { + sqlx::query!( + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = ANY($1)", + &all_ids, + ) + .execute(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error while cleaning up completed job args: {e:#}" + )) + })?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = ANY($1)", + &all_ids, + ) + .execute(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error while cleaning up completed job results: {e:#}" + )) + })?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = ANY($1)", + &all_ids, + ) + .execute(db) + .await + .map_err(|e| { + Error::internal_err(format!( + "error while cleaning up completed job logs: {e:#}" + )) + })?; + } else { + for id in &all_ids { + windmill_common::jobs::schedule_job_deletion(db, *id, w_id, secs) + .await + .map_err(|e| { + Error::internal_err(format!( + "error scheduling job deletion for {id}: {e:#}" + )) + })?; + } + } + } } if flow_job.is_canceled() { @@ -3753,21 +3859,54 @@ async fn push_next_flow_job( } } - if payload_tag.delete_after_use { - let uuid_singleton_json = serde_json::to_value(&[uuid]).map_err(|e| { - error::Error::internal_err(format!("Unable to serialize uuid: {e:#}")) - })?; - - sqlx::query!( - "UPDATE v2_job_status - SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) - WHERE id = $2", - uuid_singleton_json, - flow_innermost_root_job.unwrap_or(flow_job.id) - ) - .execute(&mut *inner_tx) - .warn_after_seconds(3) - .await?; + { + use windmill_common::jobs::resolve_delete_after_secs; + let resolved = resolve_delete_after_secs( + payload_tag.delete_after_use.then_some(true), + payload_tag.delete_after_secs, + ); + let root_id = flow_innermost_root_job.unwrap_or(flow_job.id); + match resolved { + Some(0) => { + // Immediate deletion: track in flow_jobs_to_clean (existing behavior) + let uuid_singleton_json = serde_json::to_value(&[uuid]).map_err(|e| { + error::Error::internal_err(format!("Unable to serialize uuid: {e:#}")) + })?; + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) + WHERE id = $2", + uuid_singleton_json, + root_id + ) + .execute(&mut *inner_tx) + .warn_after_seconds(3) + .await?; + } + Some(secs) => { + // Scheduled deletion: track in flow_jobs_to_schedule_clean + let entry = windmill_types::flow_status::FlowJobScheduledClean { + id: uuid, + delete_after_secs: secs, + }; + let entry_json = serde_json::to_value(&[entry]).map_err(|e| { + error::Error::internal_err(format!( + "Unable to serialize scheduled clean entry: {e:#}" + )) + })?; + sqlx::query!( + "UPDATE v2_job_status + SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_schedule_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_schedule_clean', '[]'::jsonb) || $1) + WHERE id = $2", + entry_json, + root_id + ) + .execute(&mut *inner_tx) + .warn_after_seconds(3) + .await?; + } + None => {} + } } tx = inner_tx; @@ -4162,6 +4301,7 @@ pub struct JobPayloadWithTag { pub payload: JobPayload, pub tag: Option, pub delete_after_use: bool, + pub delete_after_secs: Option, pub timeout: Option, pub on_behalf_of: Option, } @@ -4263,6 +4403,7 @@ async fn compute_next_flow_transform( payload: JobPayload::Identity, tag: None, delete_after_use: false, + delete_after_secs: None, timeout: None, on_behalf_of: None, }), @@ -4275,6 +4416,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use: false, + delete_after_secs: None, timeout: None, on_behalf_of: None, }), @@ -4282,6 +4424,7 @@ async fn compute_next_flow_transform( )) }; let delete_after_use = module.delete_after_use.unwrap_or(false); + let delete_after_secs = module.delete_after_secs; tracing::debug!(id = %flow_job.id, "computing next flow transform for {:?}", &module.value); if is_skipped { @@ -4291,8 +4434,14 @@ async fn compute_next_flow_transform( match module.get_value()? { FlowModuleValue::Identity => trivial_next_job(JobPayload::Identity), FlowModuleValue::Flow { path, .. } => { - let payload = - flow_to_payload(path, delete_after_use, &flow_job.workspace_id, db).await?; + let payload = flow_to_payload( + path, + delete_after_use, + delete_after_secs, + &flow_job.workspace_id, + db, + ) + .await?; Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob(payload), NextStatus::NextStep, @@ -4306,6 +4455,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }), @@ -4348,6 +4498,7 @@ async fn compute_next_flow_transform( module, tag, delete_after_use, + delete_after_secs, ); Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob(payload), @@ -4375,6 +4526,7 @@ async fn compute_next_flow_transform( }, tag: tag.clone(), delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }; @@ -4518,6 +4670,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }) @@ -4615,6 +4768,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }), @@ -4650,6 +4804,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }) @@ -4728,6 +4883,7 @@ async fn compute_next_flow_transform( payload, tag: None, delete_after_use, + delete_after_secs, timeout: None, on_behalf_of: None, }), @@ -4792,6 +4948,7 @@ async fn next_loop_iteration( payload, tag: None, delete_after_use, + delete_after_secs: None, timeout: None, on_behalf_of: None, }), @@ -4986,9 +5143,17 @@ async fn payload_from_simple_module( inner_path: String, ) -> Result { let delete_after_use = module.delete_after_use.unwrap_or(false); + let delete_after_secs = module.delete_after_secs; Ok(match value { FlowModuleValue::Flow { path, .. } => { - flow_to_payload(path, delete_after_use, &flow_job.workspace_id, db).await? + flow_to_payload( + path, + delete_after_use, + delete_after_secs, + &flow_job.workspace_id, + db, + ) + .await? } FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => { script_to_payload( @@ -5019,6 +5184,7 @@ async fn payload_from_simple_module( module, tag, delete_after_use, + delete_after_secs, ), FlowModuleValue::FlowScript { id, // flow_node(id). @@ -5038,6 +5204,7 @@ async fn payload_from_simple_module( }, tag, delete_after_use, + delete_after_secs, timeout: None, // timeout evaluation handled at higher level on_behalf_of: None, }, @@ -5054,6 +5221,7 @@ pub fn raw_script_to_payload( module: &FlowModule, tag: Option, delete_after_use: bool, + delete_after_secs: Option, ) -> JobPayloadWithTag { JobPayloadWithTag { payload: JobPayload::Code(RawCode { @@ -5072,6 +5240,7 @@ pub fn raw_script_to_payload( }), tag, delete_after_use, + delete_after_secs, timeout: None, // timeout evaluation handled at higher level on_behalf_of: None, } @@ -5080,6 +5249,7 @@ pub fn raw_script_to_payload( async fn flow_to_payload( path: String, delete_after_use: bool, + delete_after_secs: Option, w_id: &str, db: &DB, ) -> Result { @@ -5097,7 +5267,14 @@ async fn flow_to_payload( version, labels: None, }; - Ok(JobPayloadWithTag { payload, tag, delete_after_use, timeout: None, on_behalf_of }) + Ok(JobPayloadWithTag { + payload, + tag, + delete_after_use, + delete_after_secs, + timeout: None, + on_behalf_of, + }) } pub async fn script_to_payload( @@ -5114,75 +5291,80 @@ pub async fn script_to_payload( } else { tag_override }; - let (payload, tag, delete_after_use, script_timeout, on_behalf_of) = if script_hash.is_none() { - let (jp, tag, delete_after_use, script_timeout, on_behalf_of) = script_path_to_payload( - &script_path, - None, - db.clone(), - &flow_job.workspace_id, - Some(true), - ) - .await?; - ( - jp, - tag_override.to_owned().or(tag), - delete_after_use, - script_timeout, - on_behalf_of, - ) - } else { - let hash = script_hash.unwrap(); - - let ScriptHashInfo { - tag, - cache_ttl, - language, - dedicated_worker, - priority, - delete_after_use, - timeout, - on_behalf_of_email, - created_by, - runnable_settings: - ScriptRunnableSettingsInline { concurrency_settings, debouncing_settings }, - .. - } = get_script_info_for_hash(None, db, &flow_job.workspace_id, hash.0) - .await? - .prefetch_cached(&db) - .await?; - - let on_behalf_of = if let Some(email) = on_behalf_of_email { - Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&created_by) }) + let (payload, tag, delete_after_use, delete_after_secs, script_timeout, on_behalf_of) = + if script_hash.is_none() { + let (jp, tag, delete_after_use, delete_after_secs, script_timeout, on_behalf_of) = + script_path_to_payload( + &script_path, + None, + db.clone(), + &flow_job.workspace_id, + Some(true), + ) + .await?; + ( + jp, + tag_override.to_owned().or(tag), + delete_after_use, + delete_after_secs, + script_timeout, + on_behalf_of, + ) } else { - None - }; - ( - // We only apply the preprocessor if it's explicitly set to true in the module, - // which can only happen if the the flow is a SingleStepFlow triggered by a trigger with retries or error handling. - // In that case, apply_preprocessor is still only set to true if the script has a preprocesor. - // We only check for script hash because SingleStepFlow triggers specifies the script hash - JobPayload::ScriptHash { - hash, - path: script_path, - concurrency_settings, - debouncing_settings, - cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), - cache_ignore_s3_path: module.cache_ignore_s3_path, + let hash = script_hash.unwrap(); + + let ScriptHashInfo { + tag, + cache_ttl, language, dedicated_worker, priority, - apply_preprocessor: apply_preprocessor.unwrap_or(false), - labels: None, - }, - tag_override.to_owned().or(tag), - delete_after_use, - timeout, - on_behalf_of, - ) - }; - // the module value overrides the value set at the script level. Defaults to false if both are unset. - let final_delete_after_user = + delete_after_use, + delete_after_secs, + timeout, + on_behalf_of_email, + created_by, + runnable_settings: + ScriptRunnableSettingsInline { concurrency_settings, debouncing_settings }, + .. + } = get_script_info_for_hash(None, db, &flow_job.workspace_id, hash.0) + .await? + .prefetch_cached(&db) + .await?; + + let on_behalf_of = if let Some(email) = on_behalf_of_email { + Some(OnBehalfOf { + email, + permissioned_as: username_to_permissioned_as(&created_by), + }) + } else { + None + }; + ( + JobPayload::ScriptHash { + hash, + path: script_path, + concurrency_settings, + debouncing_settings, + cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), + cache_ignore_s3_path: module.cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: apply_preprocessor.unwrap_or(false), + labels: None, + }, + tag_override.to_owned().or(tag), + delete_after_use, + delete_after_secs, + timeout, + on_behalf_of, + ) + }; + // Module-level delete_after_secs takes precedence over script-level, then fall back to booleans + let final_delete_after_use = module.delete_after_use.unwrap_or(false) || delete_after_use.unwrap_or(false); + let final_delete_after_secs = module.delete_after_secs.or(delete_after_secs); let flow_step_timeout = if module.timeout.is_some() { None @@ -5192,7 +5374,8 @@ pub async fn script_to_payload( Ok(JobPayloadWithTag { payload, tag, - delete_after_use: final_delete_after_user, + delete_after_use: final_delete_after_use, + delete_after_secs: final_delete_after_secs, timeout: flow_step_timeout, on_behalf_of, }) diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index dd9dbf784f..67009ca9df 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.677.0"; +export const VERSION = "v1.680.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 865091104c..b3c5ea1e1a 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -67,7 +67,7 @@ const packageJson = { url: "https://github.com/windmill-labs/windmill/issues", }, dependencies: { - esbuild: "^0.24.2", + esbuild: "0.28.0", ...Object.fromEntries(parserPackages.map(p => [p, "*"])), }, optionalDependencies: { diff --git a/cli/bun.lock b/cli/bun.lock index 6c421141b4..ddcf459725 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -1,9 +1,8 @@ { "lockfileVersion": 1, - "configVersion": 1, "workspaces": { "": { - "name": "windmill-cli-dev", + "name": "wmill-dev", "dependencies": { "@cliffy/ansi": "npm:@jsr/cliffy__ansi@1.0.0", "@cliffy/command": "npm:@jsr/cliffy__command@1.0.0", @@ -11,7 +10,7 @@ "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", "@windmill-labs/shared-utils": "^1.0.12", "diff": "^5.2.0", - "esbuild": "0.24.2", + "esbuild": "0.28.0", "get-port": "7.1.0", "jszip": "3.8.0", "minimatch": "^10.0.0", @@ -53,55 +52,57 @@ "@cliffy/table": ["@jsr/cliffy__table@1.0.0", "https://npm.jsr.io/~/11/@jsr/cliffy__table/1.0.0.tgz", { "dependencies": { "@jsr/std__fmt": "^1.0.9" } }, "sha512-VoLxH0DjofHWPWKUc5N+oCwXB6O6e+carnhp23yJTa7qokBb+SCrTIABEgQdIe/p0bxgmZhz17xt2efaAxXvbQ=="], - "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.24.2", "", { "os": "aix", "cpu": "ppc64" }, "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA=="], + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.28.0", "", { "os": "aix", "cpu": "ppc64" }, "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA=="], - "@esbuild/android-arm": ["@esbuild/android-arm@0.24.2", "", { "os": "android", "cpu": "arm" }, "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q=="], + "@esbuild/android-arm": ["@esbuild/android-arm@0.28.0", "", { "os": "android", "cpu": "arm" }, "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ=="], - "@esbuild/android-arm64": ["@esbuild/android-arm64@0.24.2", "", { "os": "android", "cpu": "arm64" }, "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg=="], + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.28.0", "", { "os": "android", "cpu": "arm64" }, "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw=="], - "@esbuild/android-x64": ["@esbuild/android-x64@0.24.2", "", { "os": "android", "cpu": "x64" }, "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw=="], + "@esbuild/android-x64": ["@esbuild/android-x64@0.28.0", "", { "os": "android", "cpu": "x64" }, "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA=="], - "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.24.2", "", { "os": "darwin", "cpu": "arm64" }, "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA=="], + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.28.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q=="], - "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.24.2", "", { "os": "darwin", "cpu": "x64" }, "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA=="], + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.28.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ=="], - "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.24.2", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg=="], + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.28.0", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q=="], - "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.24.2", "", { "os": "freebsd", "cpu": "x64" }, "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q=="], + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.28.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw=="], - "@esbuild/linux-arm": ["@esbuild/linux-arm@0.24.2", "", { "os": "linux", "cpu": "arm" }, "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA=="], + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.28.0", "", { "os": "linux", "cpu": "arm" }, "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw=="], - "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.24.2", "", { "os": "linux", "cpu": "arm64" }, "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg=="], + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.28.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A=="], - "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.24.2", "", { "os": "linux", "cpu": "ia32" }, "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw=="], + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.28.0", "", { "os": "linux", "cpu": "ia32" }, "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ=="], - "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ=="], + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg=="], - "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw=="], + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w=="], - "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.24.2", "", { "os": "linux", "cpu": "ppc64" }, "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw=="], + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.28.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg=="], - "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.24.2", "", { "os": "linux", "cpu": "none" }, "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q=="], + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.28.0", "", { "os": "linux", "cpu": "none" }, "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ=="], - "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.24.2", "", { "os": "linux", "cpu": "s390x" }, "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw=="], + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.28.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q=="], - "@esbuild/linux-x64": ["@esbuild/linux-x64@0.24.2", "", { "os": "linux", "cpu": "x64" }, "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q=="], + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.28.0", "", { "os": "linux", "cpu": "x64" }, "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ=="], - "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.24.2", "", { "os": "none", "cpu": "arm64" }, "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw=="], + "@esbuild/netbsd-arm64": ["@esbuild/netbsd-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw=="], - "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.24.2", "", { "os": "none", "cpu": "x64" }, "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw=="], + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.28.0", "", { "os": "none", "cpu": "x64" }, "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw=="], - "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.24.2", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A=="], + "@esbuild/openbsd-arm64": ["@esbuild/openbsd-arm64@0.28.0", "", { "os": "openbsd", "cpu": "arm64" }, "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g=="], - "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.24.2", "", { "os": "openbsd", "cpu": "x64" }, "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA=="], + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.28.0", "", { "os": "openbsd", "cpu": "x64" }, "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA=="], - "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.24.2", "", { "os": "sunos", "cpu": "x64" }, "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig=="], + "@esbuild/openharmony-arm64": ["@esbuild/openharmony-arm64@0.28.0", "", { "os": "none", "cpu": "arm64" }, "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w=="], - "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.24.2", "", { "os": "win32", "cpu": "arm64" }, "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ=="], + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.28.0", "", { "os": "sunos", "cpu": "x64" }, "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw=="], - "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.24.2", "", { "os": "win32", "cpu": "ia32" }, "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA=="], + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.28.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA=="], - "@esbuild/win32-x64": ["@esbuild/win32-x64@0.24.2", "", { "os": "win32", "cpu": "x64" }, "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg=="], + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.28.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.28.0", "", { "os": "win32", "cpu": "x64" }, "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw=="], "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], @@ -169,9 +170,9 @@ "@types/ws": ["@types/ws@8.18.1", "", { "dependencies": { "@types/node": "*" } }, "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg=="], - "@windmill-labs/shared-utils": ["@windmill-labs/shared-utils@1.0.12", "", {}, "sha512-n68uEYv2B5q2Pp8J9syMS3qPZbppFEfeM7HIBEUfU5lGqi3hwnv4mPvgRUyb6K9im3frXC4gzdIdZdlrDpudXQ=="], + "@windmill-labs/shared-utils": ["@jsr/windmill-labs__shared-utils@1.0.12", "https://npm.jsr.io/~/11/@jsr/windmill-labs__shared-utils/1.0.12.tgz", {}, "sha512-bJOacyfxxNPwNTzA4AxCB5iGFop0h3mCgs+E9j3ZaJYDo1soblY16CebnQ56EPy/M3V344X/QoOFBORyRo1Mnw=="], - "acorn": ["acorn@8.16.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], + "acorn": ["acorn@8.16.0", "", { "bin": "bin/acorn" }, "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw=="], "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], @@ -205,7 +206,7 @@ "diff": ["diff@5.2.2", "", {}, "sha512-vtcDfH3TOjP8UekytvnHH1o1P4FcUdt4eQ1Y+Abap1tk/OB2MWQvcwS2ClCd1zuIhc3JKOx6p3kod8Vfys3E+A=="], - "esbuild": ["esbuild@0.24.2", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.24.2", "@esbuild/android-arm": "0.24.2", "@esbuild/android-arm64": "0.24.2", "@esbuild/android-x64": "0.24.2", "@esbuild/darwin-arm64": "0.24.2", "@esbuild/darwin-x64": "0.24.2", "@esbuild/freebsd-arm64": "0.24.2", "@esbuild/freebsd-x64": "0.24.2", "@esbuild/linux-arm": "0.24.2", "@esbuild/linux-arm64": "0.24.2", "@esbuild/linux-ia32": "0.24.2", "@esbuild/linux-loong64": "0.24.2", "@esbuild/linux-mips64el": "0.24.2", "@esbuild/linux-ppc64": "0.24.2", "@esbuild/linux-riscv64": "0.24.2", "@esbuild/linux-s390x": "0.24.2", "@esbuild/linux-x64": "0.24.2", "@esbuild/netbsd-arm64": "0.24.2", "@esbuild/netbsd-x64": "0.24.2", "@esbuild/openbsd-arm64": "0.24.2", "@esbuild/openbsd-x64": "0.24.2", "@esbuild/sunos-x64": "0.24.2", "@esbuild/win32-arm64": "0.24.2", "@esbuild/win32-ia32": "0.24.2", "@esbuild/win32-x64": "0.24.2" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA=="], + "esbuild": ["esbuild@0.28.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.28.0", "@esbuild/android-arm": "0.28.0", "@esbuild/android-arm64": "0.28.0", "@esbuild/android-x64": "0.28.0", "@esbuild/darwin-arm64": "0.28.0", "@esbuild/darwin-x64": "0.28.0", "@esbuild/freebsd-arm64": "0.28.0", "@esbuild/freebsd-x64": "0.28.0", "@esbuild/linux-arm": "0.28.0", "@esbuild/linux-arm64": "0.28.0", "@esbuild/linux-ia32": "0.28.0", "@esbuild/linux-loong64": "0.28.0", "@esbuild/linux-mips64el": "0.28.0", "@esbuild/linux-ppc64": "0.28.0", "@esbuild/linux-riscv64": "0.28.0", "@esbuild/linux-s390x": "0.28.0", "@esbuild/linux-x64": "0.28.0", "@esbuild/netbsd-arm64": "0.28.0", "@esbuild/netbsd-x64": "0.28.0", "@esbuild/openbsd-arm64": "0.28.0", "@esbuild/openbsd-x64": "0.28.0", "@esbuild/openharmony-arm64": "0.28.0", "@esbuild/sunos-x64": "0.28.0", "@esbuild/win32-arm64": "0.28.0", "@esbuild/win32-ia32": "0.28.0", "@esbuild/win32-x64": "0.28.0" }, "bin": "bin/esbuild" }, "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw=="], "esm-env": ["esm-env@1.2.2", "", {}, "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA=="], @@ -225,9 +226,9 @@ "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], - "is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], + "is-docker": ["is-docker@3.0.0", "", { "bin": "cli.js" }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], - "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": "cli.js" }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], "is-reference": ["is-reference@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.6" } }, "sha512-ixkJoqQvAP88E6wLydLGGqCJsrFUnqoH6HnaczB8XmDH1oaWU+xxdptvikTgaEhtZ53Ky6YXiBuUI2WXLMCwjw=="], @@ -267,7 +268,7 @@ "string_decoder": ["string_decoder@1.1.1", "", { "dependencies": { "safe-buffer": "~5.1.0" } }, "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg=="], - "svelte": ["svelte@5.53.1", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.3", "esm-env": "^1.2.1", "esrap": "^2.2.2", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-WzxFHZhhD23Qzu7JCYdvm1rxvRSzdt9HtHO8TScMBX51bLRFTcJmATVqjqXG+6Ln6hrViGCo9DzwOhAasxwC/w=="], + "svelte": ["svelte@5.53.2", "", { "dependencies": { "@jridgewell/remapping": "^2.3.4", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "@types/trusted-types": "^2.0.7", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "devalue": "^5.6.3", "esm-env": "^1.2.1", "esrap": "^2.2.2", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-yGONuIrcl/BMmqbm6/52Q/NYzfkta7uVlos5NSzGTfNJTTFtPPzra6rAQoQIwAqupeM3s9uuTf5PvioeiCdg9g=="], "tar-stream": ["tar-stream@3.1.7", "", { "dependencies": { "b4a": "^1.6.4", "fast-fifo": "^1.2.0", "streamx": "^2.15.0" } }, "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ=="], @@ -291,7 +292,7 @@ "windmill-parser-wasm-nu": ["windmill-parser-wasm-nu@1.510.1", "", {}, "sha512-AJLFiUy6af+LpUe7CddDo4+JOmw3c0K/1iOWh8NdTwXcLDj90lL6089mdsVo1apyloLgrTbcuFDzZMXVGBgtCg=="], - "windmill-parser-wasm-php": ["windmill-parser-wasm-php@1.574.1", "", {}, "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA=="], + "windmill-parser-wasm-php": ["windmill-parser-wasm-php@1.647.1", "", {}, "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg=="], "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="], @@ -301,7 +302,7 @@ "windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="], - "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="], + "windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.647.1", "", {}, "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="], "windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="], @@ -313,7 +314,7 @@ "wsl-utils": ["wsl-utils@0.1.0", "", { "dependencies": { "is-wsl": "^3.1.0" } }, "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw=="], - "yaml": ["yaml@2.8.2", "", { "bin": { "yaml": "bin.mjs" } }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], + "yaml": ["yaml@2.8.2", "", { "bin": "bin.mjs" }, "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A=="], "zimmerframe": ["zimmerframe@1.1.4", "", {}, "sha512-B58NGBEoc8Y9MWWCQGl/gq9xBCe4IiKM0a2x7GZdQKOW5Exr8S1W24J6OgM1njK8xCRGvAJIL/MxXHf6SkmQKQ=="], } diff --git a/cli/package-lock.json b/cli/package-lock.json index ae46c240c9..b4b9732059 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -12,7 +12,7 @@ "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", "@windmill-labs/shared-utils": "^1.0.12", "diff": "^5.2.0", - "esbuild": "0.24.2", + "esbuild": "0.28.0", "get-port": "7.1.0", "jszip": "3.8.0", "minimatch": "^10.0.0", @@ -99,9 +99,9 @@ } }, "node_modules/@esbuild/aix-ppc64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", - "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.0.tgz", + "integrity": "sha512-lhRUCeuOyJQURhTxl4WkpFTjIsbDayJHih5kZC1giwE+MhIzAb7mEsQMqMf18rHLsrb5qI1tafG20mLxEWcWlA==", "cpu": [ "ppc64" ], @@ -115,9 +115,9 @@ } }, "node_modules/@esbuild/android-arm": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", - "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.0.tgz", + "integrity": "sha512-wqh0ByljabXLKHeWXYLqoJ5jKC4XBaw6Hk08OfMrCRd2nP2ZQ5eleDZC41XHyCNgktBGYMbqnrJKq/K/lzPMSQ==", "cpu": [ "arm" ], @@ -131,9 +131,9 @@ } }, "node_modules/@esbuild/android-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", - "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.0.tgz", + "integrity": "sha512-+WzIXQOSaGs33tLEgYPYe/yQHf0WTU0X42Jca3y8NWMbUVhp7rUnw+vAsRC/QiDrdD31IszMrZy+qwPOPjd+rw==", "cpu": [ "arm64" ], @@ -147,9 +147,9 @@ } }, "node_modules/@esbuild/android-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", - "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.0.tgz", + "integrity": "sha512-+VJggoaKhk2VNNqVL7f6S189UzShHC/mR9EE8rDdSkdpN0KflSwWY/gWjDrNxxisg8Fp1ZCD9jLMo4m0OUfeUA==", "cpu": [ "x64" ], @@ -163,9 +163,9 @@ } }, "node_modules/@esbuild/darwin-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", - "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.0.tgz", + "integrity": "sha512-0T+A9WZm+bZ84nZBtk1ckYsOvyA3x7e2Acj1KdVfV4/2tdG4fzUp91YHx+GArWLtwqp77pBXVCPn2We7Letr0Q==", "cpu": [ "arm64" ], @@ -179,9 +179,9 @@ } }, "node_modules/@esbuild/darwin-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", - "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.0.tgz", + "integrity": "sha512-fyzLm/DLDl/84OCfp2f/XQ4flmORsjU7VKt8HLjvIXChJoFFOIL6pLJPH4Yhd1n1gGFF9mPwtlN5Wf82DZs+LQ==", "cpu": [ "x64" ], @@ -195,9 +195,9 @@ } }, "node_modules/@esbuild/freebsd-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", - "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.0.tgz", + "integrity": "sha512-l9GeW5UZBT9k9brBYI+0WDffcRxgHQD8ShN2Ur4xWq/NFzUKm3k5lsH4PdaRgb2w7mI9u61nr2gI2mLI27Nh3Q==", "cpu": [ "arm64" ], @@ -211,9 +211,9 @@ } }, "node_modules/@esbuild/freebsd-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", - "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.0.tgz", + "integrity": "sha512-BXoQai/A0wPO6Es3yFJ7APCiKGc1tdAEOgeTNy3SsB491S3aHn4S4r3e976eUnPdU+NbdtmBuLncYir2tMU9Nw==", "cpu": [ "x64" ], @@ -227,9 +227,9 @@ } }, "node_modules/@esbuild/linux-arm": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", - "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.0.tgz", + "integrity": "sha512-CjaaREJagqJp7iTaNQjjidaNbCKYcd4IDkzbwwxtSvjI7NZm79qiHc8HqciMddQ6CKvJT6aBd8lO9kN/ZudLlw==", "cpu": [ "arm" ], @@ -243,9 +243,9 @@ } }, "node_modules/@esbuild/linux-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", - "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.0.tgz", + "integrity": "sha512-RVyzfb3FWsGA55n6WY0MEIEPURL1FcbhFE6BffZEMEekfCzCIMtB5yyDcFnVbTnwk+CLAgTujmV/Lgvih56W+A==", "cpu": [ "arm64" ], @@ -259,9 +259,9 @@ } }, "node_modules/@esbuild/linux-ia32": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", - "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.0.tgz", + "integrity": "sha512-KBnSTt1kxl9x70q+ydterVdl+Cn0H18ngRMRCEQfrbqdUuntQQ0LoMZv47uB97NljZFzY6HcfqEZ2SAyIUTQBQ==", "cpu": [ "ia32" ], @@ -275,9 +275,9 @@ } }, "node_modules/@esbuild/linux-loong64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", - "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.0.tgz", + "integrity": "sha512-zpSlUce1mnxzgBADvxKXX5sl8aYQHo2ezvMNI8I0lbblJtp8V4odlm3Yzlj7gPyt3T8ReksE6bK+pT3WD+aJRg==", "cpu": [ "loong64" ], @@ -291,9 +291,9 @@ } }, "node_modules/@esbuild/linux-mips64el": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", - "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.0.tgz", + "integrity": "sha512-2jIfP6mmjkdmeTlsX/9vmdmhBmKADrWqN7zcdtHIeNSCH1SqIoNI63cYsjQR8J+wGa4Y5izRcSHSm8K3QWmk3w==", "cpu": [ "mips64el" ], @@ -307,9 +307,9 @@ } }, "node_modules/@esbuild/linux-ppc64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", - "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.0.tgz", + "integrity": "sha512-bc0FE9wWeC0WBm49IQMPSPILRocGTQt3j5KPCA8os6VprfuJ7KD+5PzESSrJ6GmPIPJK965ZJHTUlSA6GNYEhg==", "cpu": [ "ppc64" ], @@ -323,9 +323,9 @@ } }, "node_modules/@esbuild/linux-riscv64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", - "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.0.tgz", + "integrity": "sha512-SQPZOwoTTT/HXFXQJG/vBX8sOFagGqvZyXcgLA3NhIqcBv1BJU1d46c0rGcrij2B56Z2rNiSLaZOYW5cUk7yLQ==", "cpu": [ "riscv64" ], @@ -339,9 +339,9 @@ } }, "node_modules/@esbuild/linux-s390x": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", - "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.0.tgz", + "integrity": "sha512-SCfR0HN8CEEjnYnySJTd2cw0k9OHB/YFzt5zgJEwa+wL/T/raGWYMBqwDNAC6dqFKmJYZoQBRfHjgwLHGSrn3Q==", "cpu": [ "s390x" ], @@ -355,9 +355,9 @@ } }, "node_modules/@esbuild/linux-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", - "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.0.tgz", + "integrity": "sha512-us0dSb9iFxIi8srnpl931Nvs65it/Jd2a2K3qs7fz2WfGPHqzfzZTfec7oxZJRNPXPnNYZtanmRc4AL/JwVzHQ==", "cpu": [ "x64" ], @@ -371,9 +371,9 @@ } }, "node_modules/@esbuild/netbsd-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", - "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.0.tgz", + "integrity": "sha512-CR/RYotgtCKwtftMwJlUU7xCVNg3lMYZ0RzTmAHSfLCXw3NtZtNpswLEj/Kkf6kEL3Gw+BpOekRX0BYCtklhUw==", "cpu": [ "arm64" ], @@ -387,9 +387,9 @@ } }, "node_modules/@esbuild/netbsd-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", - "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.0.tgz", + "integrity": "sha512-nU1yhmYutL+fQ71Kxnhg8uEOdC0pwEW9entHykTgEbna2pw2dkbFSMeqjjyHZoCmt8SBkOSvV+yNmm94aUrrqw==", "cpu": [ "x64" ], @@ -403,9 +403,9 @@ } }, "node_modules/@esbuild/openbsd-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", - "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.0.tgz", + "integrity": "sha512-cXb5vApOsRsxsEl4mcZ1XY3D4DzcoMxR/nnc4IyqYs0rTI8ZKmW6kyyg+11Z8yvgMfAEldKzP7AdP64HnSC/6g==", "cpu": [ "arm64" ], @@ -419,9 +419,9 @@ } }, "node_modules/@esbuild/openbsd-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", - "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.0.tgz", + "integrity": "sha512-8wZM2qqtv9UP3mzy7HiGYNH/zjTA355mpeuA+859TyR+e+Tc08IHYpLJuMsfpDJwoLo1ikIJI8jC3GFjnRClzA==", "cpu": [ "x64" ], @@ -434,10 +434,26 @@ "node": ">=18" } }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.0.tgz", + "integrity": "sha512-FLGfyizszcef5C3YtoyQDACyg95+dndv79i2EekILBofh5wpCa1KuBqOWKrEHZg3zrL3t5ouE5jgr94vA+Wb2w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/sunos-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", - "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.0.tgz", + "integrity": "sha512-1ZgjUoEdHZZl/YlV76TSCz9Hqj9h9YmMGAgAPYd+q4SicWNX3G5GCyx9uhQWSLcbvPW8Ni7lj4gDa1T40akdlw==", "cpu": [ "x64" ], @@ -451,9 +467,9 @@ } }, "node_modules/@esbuild/win32-arm64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", - "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.0.tgz", + "integrity": "sha512-Q9StnDmQ/enxnpxCCLSg0oo4+34B9TdXpuyPeTedN/6+iXBJ4J+zwfQI28u/Jl40nOYAxGoNi7mFP40RUtkmUA==", "cpu": [ "arm64" ], @@ -467,9 +483,9 @@ } }, "node_modules/@esbuild/win32-ia32": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", - "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.0.tgz", + "integrity": "sha512-zF3ag/gfiCe6U2iczcRzSYJKH1DCI+ByzSENHlM2FcDbEeo5Zd2C86Aq0tKUYAJJ1obRP84ymxIAksZUcdztHA==", "cpu": [ "ia32" ], @@ -483,9 +499,9 @@ } }, "node_modules/@esbuild/win32-x64": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", - "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.0.tgz", + "integrity": "sha512-pEl1bO9mfAmIC+tW5btTmrKaujg3zGtUmWNdCw/xs70FBjwAL3o9OEKNHvNmnyylD6ubxUERiEhdsL0xBQ9efw==", "cpu": [ "x64" ], @@ -960,9 +976,9 @@ } }, "node_modules/esbuild": { - "version": "0.24.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", - "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", + "version": "0.28.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.0.tgz", + "integrity": "sha512-sNR9MHpXSUV/XB4zmsFKN+QgVG82Cc7+/aaxJ8Adi8hyOac+EXptIp45QBPaVyX3N70664wRbTcLTOemCAnyqw==", "hasInstallScript": true, "license": "MIT", "bin": { @@ -972,31 +988,32 @@ "node": ">=18" }, "optionalDependencies": { - "@esbuild/aix-ppc64": "0.24.2", - "@esbuild/android-arm": "0.24.2", - "@esbuild/android-arm64": "0.24.2", - "@esbuild/android-x64": "0.24.2", - "@esbuild/darwin-arm64": "0.24.2", - "@esbuild/darwin-x64": "0.24.2", - "@esbuild/freebsd-arm64": "0.24.2", - "@esbuild/freebsd-x64": "0.24.2", - "@esbuild/linux-arm": "0.24.2", - "@esbuild/linux-arm64": "0.24.2", - "@esbuild/linux-ia32": "0.24.2", - "@esbuild/linux-loong64": "0.24.2", - "@esbuild/linux-mips64el": "0.24.2", - "@esbuild/linux-ppc64": "0.24.2", - "@esbuild/linux-riscv64": "0.24.2", - "@esbuild/linux-s390x": "0.24.2", - "@esbuild/linux-x64": "0.24.2", - "@esbuild/netbsd-arm64": "0.24.2", - "@esbuild/netbsd-x64": "0.24.2", - "@esbuild/openbsd-arm64": "0.24.2", - "@esbuild/openbsd-x64": "0.24.2", - "@esbuild/sunos-x64": "0.24.2", - "@esbuild/win32-arm64": "0.24.2", - "@esbuild/win32-ia32": "0.24.2", - "@esbuild/win32-x64": "0.24.2" + "@esbuild/aix-ppc64": "0.28.0", + "@esbuild/android-arm": "0.28.0", + "@esbuild/android-arm64": "0.28.0", + "@esbuild/android-x64": "0.28.0", + "@esbuild/darwin-arm64": "0.28.0", + "@esbuild/darwin-x64": "0.28.0", + "@esbuild/freebsd-arm64": "0.28.0", + "@esbuild/freebsd-x64": "0.28.0", + "@esbuild/linux-arm": "0.28.0", + "@esbuild/linux-arm64": "0.28.0", + "@esbuild/linux-ia32": "0.28.0", + "@esbuild/linux-loong64": "0.28.0", + "@esbuild/linux-mips64el": "0.28.0", + "@esbuild/linux-ppc64": "0.28.0", + "@esbuild/linux-riscv64": "0.28.0", + "@esbuild/linux-s390x": "0.28.0", + "@esbuild/linux-x64": "0.28.0", + "@esbuild/netbsd-arm64": "0.28.0", + "@esbuild/netbsd-x64": "0.28.0", + "@esbuild/openbsd-arm64": "0.28.0", + "@esbuild/openbsd-x64": "0.28.0", + "@esbuild/openharmony-arm64": "0.28.0", + "@esbuild/sunos-x64": "0.28.0", + "@esbuild/win32-arm64": "0.28.0", + "@esbuild/win32-ia32": "0.28.0", + "@esbuild/win32-x64": "0.28.0" } }, "node_modules/esm-env": { diff --git a/cli/package.json b/cli/package.json index 105915a720..370d6e7050 100644 --- a/cli/package.json +++ b/cli/package.json @@ -20,7 +20,7 @@ "@cliffy/table": "npm:@jsr/cliffy__table@1.0.0", "@windmill-labs/shared-utils": "^1.0.12", "diff": "^5.2.0", - "esbuild": "0.24.2", + "esbuild": "0.28.0", "get-port": "7.1.0", "jszip": "3.8.0", "minimatch": "^10.0.0", diff --git a/cli/src/commands/instance/instance.ts b/cli/src/commands/instance/instance.ts index d95fe7269b..3f21189074 100644 --- a/cli/src/commands/instance/instance.ts +++ b/cli/src/commands/instance/instance.ts @@ -772,7 +772,7 @@ const command = new Command() .option("--dry-run", "Perform a dry run without making changes") .option("--skip-users", "Skip pulling users") .option("--skip-settings", "Skip pulling settings") - .option("--skip-configs", "Skip pulling configs (worker groups and SMTP)") + .option("--skip-configs", "Skip pulling configs (worker groups)") .option("--skip-groups", "Skip pulling instance groups") .option("--include-workspaces", "Also pull workspaces") .option("--folder-per-instance", "Create a folder per instance") @@ -797,7 +797,7 @@ const command = new Command() .option("--dry-run", "Perform a dry run without making changes") .option("--skip-users", "Skip pushing users") .option("--skip-settings", "Skip pushing settings") - .option("--skip-configs", "Skip pushing configs (worker groups and SMTP)") + .option("--skip-configs", "Skip pushing configs (worker groups)") .option("--skip-groups", "Skip pushing instance groups") .option("--include-workspaces", "Also push workspaces") .option("--folder-per-instance", "Create a folder per instance") diff --git a/cli/src/commands/worker-groups/worker-groups.ts b/cli/src/commands/worker-groups/worker-groups.ts index 30c14249ad..c5139a51ac 100644 --- a/cli/src/commands/worker-groups/worker-groups.ts +++ b/cli/src/commands/worker-groups/worker-groups.ts @@ -122,7 +122,7 @@ const command = new Command() .action(pullWorkerGroups as any) .command("push") .description( - "Push instance settings, users, configs, group and overwrite remote" + "Push worker groups (similar to `wmill instance push --skip-users --skip-settings --skip-groups`)" ) .option( "--instance [instance]", diff --git a/cli/src/commands/workspace/fork.ts b/cli/src/commands/workspace/fork.ts index 92bb38b799..21b5cf5a26 100644 --- a/cli/src/commands/workspace/fork.ts +++ b/cli/src/commands/workspace/fork.ts @@ -3,19 +3,18 @@ import { colors } from "@cliffy/ansi/colors"; import { Input } from "@cliffy/prompt/input"; import * as log from "../../core/log.ts"; import { setClient } from "../../core/client.ts"; -import { allWorkspaces, list, removeWorkspace } from "./workspace.ts"; +import { allWorkspaces, list, removeWorkspace } from "./workspace.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, isGitRepository } from "../../utils/git.ts"; import { WM_FORK_PREFIX } from "../../core/constants.ts"; import { tryResolveBranchWorkspace } from "../../core/context.ts"; -// NOTE: This import will work after regenerating the API client -// Run ./gen_wm_client.sh to regenerate after backend changes -// import * as wmill from "../../../gen/services.gen.ts"; - async function createWorkspaceFork( opts: GlobalOptions & { createWorkspaceName: string | undefined; + color: string | undefined; + datatableBehavior: string | undefined; + yes: boolean | undefined; }, workspaceName: string | undefined, workspaceId: string | undefined = undefined, @@ -104,21 +103,139 @@ async function createWorkspaceFork( throw new Error(`This forked workspace '${workspaceId}' (${workspaceName}) already exists. Choose a different id`); } + // --- Datatable cloning (matches ForkDatatableSection.svelte) --- + interface ForkedDatatableInfo { + name: string; + new_dbname: string; + } + const forkedDatatables: ForkedDatatableInfo[] = []; + + let datatables: Awaited> = []; + try { + datatables = await wmill.listDataTables({ + workspace: workspace.workspaceId, + }); + } catch (e) { + log.info( + colors.yellow( + `Note: Could not list datatables: ${(e as Error).message}` + ) + ); + } + + if (datatables && datatables.length > 0) { + const behavior = opts.datatableBehavior ?? (opts.yes ? "skip" : undefined); + + if (behavior !== "skip") { + log.info(`\nFound ${datatables.length} datatable(s):`); + + for (const dt of datatables) { + let dtBehavior: string; + + if (behavior === "schema_only" || behavior === "schema_and_data") { + dtBehavior = behavior; + } else { + // Interactive prompt + const { Select } = await import("@cliffy/prompt/select"); + dtBehavior = await Select.prompt({ + message: `Datatable "${dt.name}" (${dt.resource_type}):`, + options: [ + { name: "Keep original (no cloning)", value: "keep_original" }, + { name: "Clone schema only", value: "schema_only" }, + { name: "Clone schema and data", value: "schema_and_data" }, + ], + }); + } + + if (dtBehavior === "keep_original") { + continue; + } + + const newDbName = `${trueWorkspaceId.replace(/-/g, "_")}__${dt.name}`; + + try { + log.info( + colors.blue(` Creating database "${newDbName}" for datatable "${dt.name}"...`) + ); + + await wmill.createPgDatabase({ + workspace: workspace.workspaceId, + requestBody: { + source: `datatable://${dt.name}`, + target_dbname: newDbName, + }, + }); + + log.info( + colors.blue( + ` Importing ${dtBehavior === "schema_only" ? "schema" : "schema + data"}...` + ) + ); + + await wmill.importPgDatabase({ + workspace: workspace.workspaceId, + requestBody: { + source: `datatable://${dt.name}`, + target: `datatable://${dt.name}`, + target_dbname_override: newDbName, + fork_behavior: dtBehavior as "schema_only" | "schema_and_data", + }, + }); + + log.info(colors.green(` ✓ Datatable "${dt.name}" cloned.`)); + forkedDatatables.push({ name: dt.name, new_dbname: newDbName }); + } catch (e) { + log.info( + colors.yellow( + ` ✗ Failed to clone datatable "${dt.name}": ${(e as Error).message}` + ) + ); + } + } + } + } + + // --- Create git branch for fork (matches UI: createWorkspaceForkGitBranch) --- + const forkColor = opts.color; + try { + const gitSyncJobIds = await wmill.createWorkspaceForkGitBranch({ + workspace: workspace.workspaceId, + requestBody: { + id: trueWorkspaceId, + name: opts.createWorkspaceName ?? trueWorkspaceId, + color: forkColor, + }, + }); + if (gitSyncJobIds && gitSyncJobIds.length > 0) { + log.info( + colors.blue( + `Git sync branch creation triggered (${gitSyncJobIds.length} job(s)). These will complete asynchronously.` + ) + ); + } + } catch (e) { + log.error( + colors.red( + `Failed to create git branch for fork: ${(e as Error).message}` + ) + ); + throw e; + } + + // --- Create the fork workspace --- try { - // TODO: Update to createWorkspaceFork after regenerating client from new OpenAPI spec const result = await wmill.createWorkspaceFork({ workspace: workspace.workspaceId, requestBody: { id: trueWorkspaceId, name: opts.createWorkspaceName ?? trueWorkspaceId, - color: undefined, + color: forkColor, + forked_datatables: forkedDatatables, }, }); log.info(colors.green(`✅ ${result}`)); - } catch (error) { - // If workspace creation fails, we should clean up the git branch log.error( colors.red(`Failed to create forked workspace: ${(error as Error).message}`), ); @@ -135,8 +252,8 @@ async function createWorkspaceFork( When doing operations on the forked workspace, it will use the remote setup in gitBranches for the branch it was forked from. To merge changes back to the parent workspace, you can: + - Use the CLI: ` + colors.white(`git checkout ${newBranchName} && wmill workspace merge`) + ` - Use the Merge UI from the forked workspace home page - - Deploy individual items via the Deploy to staging/prod UI - Use git: ` + colors.white(`git checkout ${clonedBranchName} && git merge ${newBranchName} && wmill sync push`) + ` See: https://www.windmill.dev/docs/advanced/workspace_forks`); } diff --git a/cli/src/commands/workspace/merge.ts b/cli/src/commands/workspace/merge.ts new file mode 100644 index 0000000000..c2b6532254 --- /dev/null +++ b/cli/src/commands/workspace/merge.ts @@ -0,0 +1,427 @@ +import { GlobalOptions } from "../../types.ts"; +import { colors } from "@cliffy/ansi/colors"; +import { Table } from "@cliffy/table"; +import * as log from "../../core/log.ts"; +import { setClient } from "../../core/client.ts"; +import { tryResolveBranchWorkspace } from "../../core/context.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { + deployItem, + deleteItemInWorkspace, + getOnBehalfOf, + type DeployKind, + type DeployProvider, +} from "../../../windmill-utils-internal/src/deploy.ts"; + +// --------------------------------------------------------------------------- +// Provider adapter — wraps CLI's standalone API functions +// --------------------------------------------------------------------------- + +const provider: DeployProvider = { + existsFlowByPath: wmill.existsFlowByPath, + existsScriptByPath: wmill.existsScriptByPath, + existsApp: wmill.existsApp, + existsVariable: wmill.existsVariable, + existsResource: wmill.existsResource, + existsResourceType: wmill.existsResourceType, + existsFolder: wmill.existsFolder, + getFlowByPath: wmill.getFlowByPath, + createFlow: wmill.createFlow, + updateFlow: wmill.updateFlow, + archiveFlowByPath: wmill.archiveFlowByPath, + getScriptByPath: wmill.getScriptByPath, + createScript: wmill.createScript, + archiveScriptByPath: wmill.archiveScriptByPath, + getAppByPath: wmill.getAppByPath, + createApp: wmill.createApp, + updateApp: wmill.updateApp, + createAppRaw: wmill.createAppRaw, + updateAppRaw: wmill.updateAppRaw, + getPublicSecretOfLatestVersionOfApp: + wmill.getPublicSecretOfLatestVersionOfApp, + getRawAppData: wmill.getRawAppData, + deleteApp: wmill.deleteApp, + getVariable: wmill.getVariable, + createVariable: wmill.createVariable, + updateVariable: wmill.updateVariable, + deleteVariable: wmill.deleteVariable, + getResource: wmill.getResource, + createResource: wmill.createResource, + updateResource: wmill.updateResource, + deleteResource: wmill.deleteResource, + getResourceType: wmill.getResourceType, + createResourceType: wmill.createResourceType, + updateResourceType: wmill.updateResourceType, + deleteResourceType: wmill.deleteResourceType, + getFolder: wmill.getFolder, + createFolder: wmill.createFolder, + updateFolder: wmill.updateFolder, + deleteFolder: wmill.deleteFolder, +}; + +// --------------------------------------------------------------------------- +// Main merge command +// --------------------------------------------------------------------------- + +async function mergeWorkspaces( + opts: GlobalOptions & { + direction?: string; + all?: boolean; + skipConflicts?: boolean; + include?: string; + exclude?: string; + preserveOnBehalfOf?: boolean; + yes?: boolean; + } +): Promise { + // 1. Resolve fork workspace + const workspace = await tryResolveBranchWorkspace(opts); + if (!workspace) { + throw new Error( + "Could not resolve workspace from branch name. Make sure you are in a git repo with gitBranches configured." + ); + } + + const token = workspace.token; + if (!token) { + throw new Error("Not logged in. Please run 'wmill workspace add' first."); + } + + const remote = workspace.remote; + setClient( + token, + remote.endsWith("/") ? remote.substring(0, remote.length - 1) : remote + ); + + const forkWorkspaceId = workspace.workspaceId; + + // 2. Find parent workspace + const userWorkspaces = await wmill.listUserWorkspaces(); + const forkEntry = userWorkspaces.workspaces?.find( + (w) => w.id === forkWorkspaceId + ); + + if (!forkEntry?.parent_workspace_id) { + throw new Error( + `Workspace '${forkWorkspaceId}' is not a fork (no parent_workspace_id). ` + + `You can only merge from a forked workspace.` + ); + } + + const parentWorkspaceId = forkEntry.parent_workspace_id; + log.info( + `Fork: ${colors.bold(forkWorkspaceId)} → Parent: ${colors.bold(parentWorkspaceId)}` + ); + + // 3. Compare workspaces + log.info("Comparing workspaces..."); + const comparison = await wmill.compareWorkspaces({ + workspace: parentWorkspaceId, + targetWorkspaceId: forkWorkspaceId, + }); + + if (comparison.skipped_comparison) { + log.info( + colors.yellow( + "This fork was created before change tracking was available. " + + "Use the UI or git-based merge instead." + ) + ); + return; + } + + const summary = comparison.summary; + if (summary.total_diffs === 0) { + log.info(colors.green("Everything is up to date. No differences found.")); + return; + } + + // 4. Display summary + log.info(""); + log.info(colors.bold("Comparison Summary:")); + + const summaryRows: string[][] = []; + if (summary.scripts_changed > 0) + summaryRows.push(["Scripts", String(summary.scripts_changed)]); + if (summary.flows_changed > 0) + summaryRows.push(["Flows", String(summary.flows_changed)]); + if (summary.apps_changed > 0) + summaryRows.push(["Apps", String(summary.apps_changed)]); + if (summary.resources_changed > 0) + summaryRows.push(["Resources", String(summary.resources_changed)]); + if (summary.variables_changed > 0) + summaryRows.push(["Variables", String(summary.variables_changed)]); + if (summary.resource_types_changed > 0) + summaryRows.push(["Resource Types", String(summary.resource_types_changed)]); + if (summary.folders_changed > 0) + summaryRows.push(["Folders", String(summary.folders_changed)]); + summaryRows.push(["Total", String(summary.total_diffs)]); + if (summary.conflicts > 0) + summaryRows.push([ + colors.red("Conflicts"), + colors.red(String(summary.conflicts)), + ]); + + new Table() + .header(["Type", "Changed"]) + .padding(2) + .border(true) + .body(summaryRows) + .render(); + + // 5. Display diffs table + const diffs = comparison.diffs.filter((d) => d.has_changes !== false); + if (diffs.length === 0) { + log.info(colors.green("No effective changes to deploy.")); + return; + } + + log.info(""); + log.info(colors.bold("Changed items:")); + + new Table() + .header(["#", "Kind", "Path", "Ahead", "Behind", "Conflict"]) + .padding(1) + .border(true) + .body( + diffs.map((d, i) => { + const isConflict = d.ahead > 0 && d.behind > 0; + return [ + String(i + 1), + d.kind, + d.path, + d.ahead > 0 ? colors.green(String(d.ahead)) : "0", + d.behind > 0 ? colors.yellow(String(d.behind)) : "0", + isConflict ? colors.red("YES") : "", + ]; + }) + ) + .render(); + + // 6. Determine direction + let direction: "to-parent" | "to-fork"; + if (opts.direction === "to-parent" || opts.direction === "to-fork") { + direction = opts.direction; + } else if (opts.direction) { + throw new Error( + `Invalid direction '${opts.direction}'. Use 'to-parent' or 'to-fork'.` + ); + } else if (opts.yes) { + direction = "to-parent"; + } else { + const { Select } = await import("@cliffy/prompt/select"); + direction = (await Select.prompt({ + message: "Deploy direction:", + options: [ + { + name: `Deploy to parent (${parentWorkspaceId}) ← fork changes`, + value: "to-parent", + }, + { + name: `Update fork (${forkWorkspaceId}) ← parent changes`, + value: "to-fork", + }, + ], + })) as "to-parent" | "to-fork"; + } + + log.info( + `\nDirection: ${colors.bold(direction === "to-parent" ? `Fork → Parent (${parentWorkspaceId})` : `Parent → Fork (${forkWorkspaceId})`)}` + ); + + // 7. Filter selectable diffs based on direction + const selectableDiffs = diffs.filter((d) => { + if (direction === "to-parent") { + return d.ahead > 0; + } else { + return d.behind > 0; + } + }); + + if (selectableDiffs.length === 0) { + log.info( + colors.yellow(`No items to deploy in the '${direction}' direction.`) + ); + return; + } + + // 8. Select items + let selectedDiffs = selectableDiffs; + + if (opts.all) { + selectedDiffs = selectableDiffs; + } else if (opts.skipConflicts) { + selectedDiffs = selectableDiffs.filter( + (d) => !(d.ahead > 0 && d.behind > 0) + ); + } else if (opts.yes && !opts.include && !opts.exclude) { + if (direction === "to-fork") { + selectedDiffs = selectableDiffs.filter( + (d) => !(d.ahead > 0 && d.behind > 0) + ); + } + } else if (!opts.yes) { + const { Checkbox } = await import("@cliffy/prompt/checkbox"); + const defaultForToFork = direction === "to-fork"; + const selectedValues = await Checkbox.prompt({ + message: `Select items to deploy (${selectableDiffs.length} available):`, + options: selectableDiffs.map((d) => { + const isConflict = d.ahead > 0 && d.behind > 0; + const label = `${d.kind}:${d.path}${isConflict ? colors.red(" [CONFLICT]") : ""}`; + return { + name: label, + value: `${d.kind}:${d.path}`, + checked: defaultForToFork ? !isConflict : true, + }; + }), + }); + selectedDiffs = selectableDiffs.filter((d) => + selectedValues.includes(`${d.kind}:${d.path}`) + ); + } + + // Apply --include filter + if (opts.include) { + const includeSet = new Set(opts.include.split(",").map((s) => s.trim())); + selectedDiffs = selectedDiffs.filter((d) => + includeSet.has(`${d.kind}:${d.path}`) + ); + } + + // Apply --exclude filter + if (opts.exclude) { + const excludeSet = new Set(opts.exclude.split(",").map((s) => s.trim())); + selectedDiffs = selectedDiffs.filter( + (d) => !excludeSet.has(`${d.kind}:${d.path}`) + ); + } + + if (selectedDiffs.length === 0) { + log.info(colors.yellow("No items selected for deployment.")); + return; + } + + // Warn about conflicts + const conflicts = selectedDiffs.filter( + (d) => d.ahead > 0 && d.behind > 0 + ); + if (conflicts.length > 0) { + log.info( + colors.yellow( + `\n⚠ ${conflicts.length} conflicting item(s) will be deployed (source will overwrite target):` + ) + ); + for (const c of conflicts) { + log.info(colors.yellow(` - ${c.kind}:${c.path}`)); + } + if (!opts.yes) { + const { Confirm } = await import("@cliffy/prompt/confirm"); + const proceed = await Confirm.prompt( + "Proceed with deploying conflicting items?" + ); + if (!proceed) { + log.info("Aborted."); + return; + } + } + } + + log.info( + `\nDeploying ${colors.bold(String(selectedDiffs.length))} item(s)...` + ); + + // 9. Sort: folders first + const sorted = [...selectedDiffs].sort((a, b) => { + const aFolder = (a.kind as string) === "folder" ? 0 : 1; + const bFolder = (b.kind as string) === "folder" ? 0 : 1; + return aFolder - bFolder; + }); + + // Determine workspaceFrom and workspaceTo based on direction + const workspaceFrom = + direction === "to-parent" ? forkWorkspaceId : parentWorkspaceId; + const workspaceTo = + direction === "to-parent" ? parentWorkspaceId : forkWorkspaceId; + + // 10. Deploy + let successCount = 0; + let failCount = 0; + + for (const diff of sorted) { + const label = `${diff.kind}:${diff.path}`; + + // Check if the item was deleted in the source workspace + const itemDeletedInSource = + direction === "to-parent" + ? diff.exists_in_fork === false + : diff.exists_in_source === false; + + let result; + if (itemDeletedInSource) { + log.info(colors.yellow(` ⌫ ${label} (removing from target)`)); + result = await deleteItemInWorkspace( + provider, + diff.kind as DeployKind, + diff.path, + workspaceTo + ); + } else { + let onBehalfOf: string | undefined; + if (opts.preserveOnBehalfOf) { + onBehalfOf = await getOnBehalfOf( + provider, + diff.kind as DeployKind, + diff.path, + workspaceFrom + ); + } + + result = await deployItem( + provider, + diff.kind as DeployKind, + diff.path, + workspaceFrom, + workspaceTo, + onBehalfOf + ); + } + + if (result.success) { + log.info(colors.green(` ✓ ${label}`)); + successCount++; + } else { + log.info(colors.red(` ✗ ${label}: ${result.error}`)); + failCount++; + } + } + + // 11. Reset diff tally (only if items were successfully deployed) + if (successCount > 0) { + try { + await wmill.resetDiffTally({ + workspace: parentWorkspaceId, + forkWorkspaceId: forkWorkspaceId, + }); + } catch { + // Non-critical + } + } + + // 12. Summary + log.info(""); + if (failCount === 0) { + log.info( + colors.green( + `✅ Successfully deployed ${successCount} item(s) from ${workspaceFrom} to ${workspaceTo}.` + ) + ); + } else { + log.info( + colors.yellow( + `Deployed ${successCount} item(s), ${colors.red(String(failCount) + " failed")} from ${workspaceFrom} to ${workspaceTo}.` + ) + ); + } +} + +export { mergeWorkspaces }; diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index 1b5d287069..6cd0eeb3fd 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -15,6 +15,7 @@ import * as log from "../../core/log.ts"; import { setClient } from "../../core/client.ts"; import { requireLogin } from "../../core/auth.ts"; import { createWorkspaceFork, deleteWorkspaceFork } from "./fork.ts"; +import { mergeWorkspaces } from "./merge.ts"; import * as wmill from "../../../gen/services.gen.ts"; @@ -651,11 +652,27 @@ const command = new Command() "--create-workspace-name ", "Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id." ) + .option("--color ", "Workspace color (hex code, e.g. #ff0000)") + .option( + "--datatable-behavior ", + "How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt)" + ) + .option("-y --yes", "Skip interactive prompts (defaults datatable behavior to 'skip')") .action(createWorkspaceFork as any) .command("delete-fork") .description("Delete a forked workspace and git branch") .arguments("") .option("-y --yes", "Skip confirmation prompt") - .action(deleteWorkspaceFork as any); + .action(deleteWorkspaceFork as any) + .command("merge") + .description("Compare and deploy changes between a fork and its parent workspace") + .option("--direction ", "Deploy direction: to-parent or to-fork") + .option("--all", "Deploy all changed items including conflicts") + .option("--skip-conflicts", "Skip items modified in both workspaces") + .option("--include ", "Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow)") + .option("--exclude ", "Comma-separated kind:path items to exclude") + .option("--preserve-on-behalf-of", "Preserve original on_behalf_of/permissioned_as values") + .option("-y --yes", "Non-interactive mode (deploy without prompts)") + .action(mergeWorkspaces as any); export default command; diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 21ba5463ad..27414e7adc 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -10,7 +10,6 @@ import { AIConfig, Config, GlobalSetting } from "../../gen/types.gen.ts"; import { compareInstanceObjects, InstanceSyncOptions } from "../commands/instance/instance.ts"; import { isSuperset } from "../types.ts"; import { deepEqual } from "../utils/utils.ts"; -import { removeWorkerPrefix } from "../commands/worker-groups/worker-groups.ts"; import { decrypt, encrypt } from "../utils/local_encryption.ts"; // New grouped config interfaces @@ -625,12 +624,7 @@ export async function pullInstanceConfigs( opts: InstanceSyncOptions, preview = false ) { - const remoteConfigs = (await wmill.listConfigs()).map((x) => { - return { - ...x, - name: removeWorkerPrefix(x.name), - }; - }); + const remoteConfigs = await wmill.listWorkerGroups(); if (preview) { const localConfigs: Config[] = await readLocalConfigs(opts); @@ -658,12 +652,7 @@ export async function pushInstanceConfigs( opts: InstanceSyncOptions, preview: boolean = false ) { - const remoteConfigs = (await wmill.listConfigs()).map((x) => { - return { - ...x, - name: removeWorkerPrefix(x.name), - }; - }); + const remoteConfigs = await wmill.listWorkerGroups(); const localConfigs = await readLocalConfigs(opts); if (preview) { @@ -682,9 +671,7 @@ export async function pushInstanceConfigs( } try { await wmill.updateConfig({ - name: config.name.startsWith("worker__") - ? config.name - : `worker__${config.name}`, + name: `worker__${config.name}`, requestBody: config.config, }); } catch (err) { @@ -698,7 +685,7 @@ export async function pushInstanceConfigs( if (!localMatch) { try { await wmill.deleteConfig({ - name: removeConfig.name, + name: `worker__${removeConfig.name}`, }); } catch (err) { log.error(`Failed to delete config ${removeConfig.name}: ${err}`); diff --git a/cli/src/guidance/skills.ts b/cli/src/guidance/skills.ts index 1f75778bb4..b1a6dcff09 100644 --- a/cli/src/guidance/skills.ts +++ b/cli/src/guidance/skills.ts @@ -4474,7 +4474,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_use":{"type":"boolean","description":"If true, this step's result is deleted after use to save memory"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`, "raw-app": `--- name: raw-app description: MUST use when creating raw apps. @@ -5301,7 +5301,7 @@ sync local with a remote instance or the opposite (push or pull) - \`--dry-run\` - Perform a dry run without making changes - \`--skip-users\` - Skip pulling users - \`--skip-settings\` - Skip pulling settings - - \`--skip-configs\` - Skip pulling configs (worker groups and SMTP) + - \`--skip-configs\` - Skip pulling configs (worker groups) - \`--skip-groups\` - Skip pulling instance groups - \`--include-workspaces\` - Also pull workspaces - \`--folder-per-instance\` - Create a folder per instance @@ -5313,7 +5313,7 @@ sync local with a remote instance or the opposite (push or pull) - \`--dry-run\` - Perform a dry run without making changes - \`--skip-users\` - Skip pushing users - \`--skip-settings\` - Skip pushing settings - - \`--skip-configs\` - Skip pushing configs (worker groups and SMTP) + - \`--skip-configs\` - Skip pushing configs (worker groups) - \`--skip-groups\` - Skip pushing instance groups - \`--include-workspaces\` - Also push workspaces - \`--folder-per-instance\` - Create a folder per instance @@ -5617,7 +5617,7 @@ display worker groups, pull and push worker groups configs - \`--instance\` - Name of the instance to push to, override the active instance - \`--base-url\` - Base url to be passed to the instance settings instead of the local one - \`--yes\` - Pull without needing confirmation -- \`worker-groups push\` - Push instance settings, users, configs, group and overwrite remote +- \`worker-groups push\` - Push worker groups (similar to \`wmill instance push --skip-users --skip-settings --skip-groups\`) - \`--instance [instance]\` - Name of the instance to push to, override the active instance - \`--base-url [baseUrl]\` - If used with --token, will be used as the base url for the instance - \`--yes\` - Push without needing confirmation @@ -5654,8 +5654,19 @@ workspace related commands - \`--branch, --env \` - Specify branch/environment (defaults to current) - \`workspace fork [workspace_name:string] [workspace_id:string]\` - Create a forked workspace - \`--create-workspace-name \` - Specify the workspace name. Ignored if --create is not specified or the workspace already exists. Will default to the workspace id. + - \`--color \` - Workspace color (hex code, e.g. #ff0000) + - \`--datatable-behavior \` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt) + - \`-y --yes\` - Skip interactive prompts (defaults datatable behavior to 'skip') - \`workspace delete-fork \` - Delete a forked workspace and git branch - \`-y --yes\` - Skip confirmation prompt +- \`workspace merge\` - Compare and deploy changes between a fork and its parent workspace + - \`--direction \` - Deploy direction: to-parent or to-fork + - \`--all\` - Deploy all changed items including conflicts + - \`--skip-conflicts\` - Skip items modified in both workspaces + - \`--include \` - Comma-separated kind:path items to include (e.g. script:f/test/main,flow:f/my/flow) + - \`--exclude \` - Comma-separated kind:path items to exclude + - \`--preserve-on-behalf-of\` - Preserve original on_behalf_of/permissioned_as values + - \`-y --yes\` - Non-interactive mode (deploy without prompts) `, }; diff --git a/cli/src/main.ts b/cli/src/main.ts index 00d08570f1..1457756a18 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -78,7 +78,7 @@ export { token, }; -export const VERSION = "1.677.0"; +export const VERSION = "1.680.0"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/cli/test/fork_merge.test.ts b/cli/test/fork_merge.test.ts new file mode 100644 index 0000000000..4ac4cb74f1 --- /dev/null +++ b/cli/test/fork_merge.test.ts @@ -0,0 +1,535 @@ +import { expect, test } from "bun:test"; +import { withTestBackend, type TestBackend } from "./test_backend.ts"; + +// ============================================================================= +// FORK & MERGE INTEGRATION TESTS +// +// Tests the full fork/merge cycle using a single test backend instance. +// All sub-tests share the same backend to avoid workspace-limit issues +// (CE limits to 2 non-admins workspaces). +// +// workspace_diff tracking is an EE feature (populated by git sync). +// We manually insert rows into workspace_diff to simulate what git sync +// does, matching the pattern in backend/.../workspace_comparison.rs. +// ============================================================================= + +const FORK_ID = "wm-fork-merge-test"; + +async function api( + backend: TestBackend, + path: string, + options: RequestInit = {} +): Promise { + return backend.apiRequest!(path, options); +} + +async function runSQL(backend: TestBackend, query: string): Promise { + const dbUrl = + process.env["DATABASE_URL"] || + "postgres://postgres:changeme@localhost:5432"; + const proc = Bun.spawn( + [ + "psql", + `${dbUrl}/postgres?sslmode=disable`, + "-t", + "-c", + `SELECT datname FROM pg_database WHERE datname LIKE 'windmill_test_%' ORDER BY datname DESC LIMIT 1`, + ], + { stdout: "pipe", stderr: "pipe" } + ); + const dbName = (await new Response(proc.stdout).text()).trim(); + await proc.exited; + if (!dbName) throw new Error("Could not find test database"); + + const sqlProc = Bun.spawn( + ["psql", `${dbUrl}/${dbName}?sslmode=disable`, "-c", query], + { stdout: "pipe", stderr: "pipe" } + ); + await sqlProc.exited; +} + +async function populateWorkspaceDiff( + backend: TestBackend, + parentWs: string, + forkWs: string, + diffs: Array<{ path: string; kind: string; ahead: number; behind: number }> +): Promise { + const values = diffs + .map( + (d) => + `('${parentWs}', '${forkWs}', '${d.path}', '${d.kind}', ${d.ahead}, ${d.behind})` + ) + .join(",\n"); + await runSQL( + backend, + `INSERT INTO workspace_diff (source_workspace_id, fork_workspace_id, path, kind, ahead, behind) + VALUES ${values} + ON CONFLICT (source_workspace_id, fork_workspace_id, path, kind) + DO UPDATE SET ahead = EXCLUDED.ahead, behind = EXCLUDED.behind, has_changes = NULL` + ); +} + +async function removeFromSkipTally(backend: TestBackend, workspaceId: string) { + await runSQL(backend, `DELETE FROM skip_workspace_diff_tally WHERE workspace_id = '${workspaceId}'`); +} + +async function deleteFork(backend: TestBackend, forkId: string) { + try { + await api(backend, `/api/w/${forkId}/workspaces/delete`, { method: "POST" }); + } catch {} + // Force-clean via SQL to ensure workspace slot is freed (CE limits to 2) + const esc = forkId.replace(/'/g, "''"); + await runSQL(backend, ` + SET session_replication_role = replica; + DO $$ DECLARE r RECORD; BEGIN + FOR r IN SELECT c.table_name FROM information_schema.columns c + JOIN information_schema.tables t ON c.table_name = t.table_name AND c.table_schema = t.table_schema + WHERE c.column_name = 'workspace_id' AND c.table_schema = 'public' AND t.table_type = 'BASE TABLE' + GROUP BY c.table_name + LOOP EXECUTE format('DELETE FROM %I WHERE workspace_id = ''${esc}''', r.table_name); + END LOOP; + END $$; + DELETE FROM workspace WHERE id = '${esc}'; + DELETE FROM workspace_diff WHERE fork_workspace_id = '${esc}'; + DELETE FROM skip_workspace_diff_tally WHERE workspace_id = '${esc}'; + SET session_replication_role = DEFAULT; + `); +} + +async function createTestItems(backend: TestBackend, workspace: string) { + await api(backend, `/api/w/${workspace}/folders/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "merge_test" }), + }); + await api(backend, `/api/w/${workspace}/scripts/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/merge_test/script_a", + content: "export function main() { return 'parent v1'; }", + language: "bun", + summary: "Script A", + schema: { type: "object", properties: {}, required: [] }, + }), + }); + await api(backend, `/api/w/${workspace}/variables/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/var_a", value: "parent_value", is_secret: false, description: "Variable A" }), + }); + await api(backend, `/api/w/${workspace}/resources/type/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "merge_test_type", schema: { type: "object" }, description: "Test type" }), + }); + await api(backend, `/api/w/${workspace}/resources/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/resource_a", resource_type: "merge_test_type", value: { key: "parent" }, description: "Resource A" }), + }); +} + +async function createFork(backend: TestBackend, parentWs: string, forkId: string, color?: string) { + const r = await api(backend, `/api/w/${parentWs}/workspaces/create_fork`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ id: forkId, name: "Test Fork", color, forked_datatables: [] }), + }); + if (!r.ok) { + const err = await r.text(); + throw new Error(`Fork creation failed: ${r.status} ${err}`); + } + await removeFromSkipTally(backend, forkId); + await new Promise((resolve) => setTimeout(resolve, 500)); +} + +// ===================================================================== +// All fork/merge sub-tests run inside a single withTestBackend to share +// the backend instance and avoid CE workspace limits. +// ===================================================================== +test( + "Fork/Merge: full cycle integration tests", + async () => { + await withTestBackend(async (backend, _tempDir) => { + const parentWs = backend.workspace; + + // --------------------------------------------------------------- + // Sub-test 1: Deploy changes from fork to parent + // --------------------------------------------------------------- + console.log("\n--- Sub-test 1: fork→parent deploy ---"); + await deleteFork(backend, FORK_ID); + await createTestItems(backend, parentWs); + await createFork(backend, parentWs, FORK_ID, "#ff5500"); + + // Make changes in fork + const forkScript = await (await api(backend, `/api/w/${FORK_ID}/scripts/get/p/f/merge_test/script_a`)).json(); + await api(backend, `/api/w/${FORK_ID}/scripts/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...forkScript, content: "export function main() { return 'fork v2 - modified!'; }", summary: "Script A (fork)", parent_hash: forkScript.hash }), + }); + await api(backend, `/api/w/${FORK_ID}/variables/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/new_fork_var", value: "fork_only", is_secret: false, description: "New from fork" }), + }); + await api(backend, `/api/w/${FORK_ID}/resources/update/f/merge_test/resource_a`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/resource_a", value: { key: "fork_modified" }, description: "Resource A (fork)" }), + }); + + // Populate workspace_diff + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/script_a", kind: "script", ahead: 1, behind: 0 }, + { path: "f/merge_test/new_fork_var", kind: "variable", ahead: 1, behind: 0 }, + { path: "f/merge_test/resource_a", kind: "resource", ahead: 1, behind: 0 }, + ]); + + // Compare + const comp1 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + expect(comp1.skipped_comparison).toBe(false); + expect(comp1.summary.total_diffs).toBeGreaterThanOrEqual(3); + expect(comp1.summary.conflicts).toBe(0); + + // Deploy fork→parent + for (const diff of comp1.diffs.filter((d: any) => d.ahead > 0)) { + const { kind, path } = diff; + if (kind === "script") { + const s = await (await api(backend, `/api/w/${FORK_ID}/scripts/get/p/${path}`)).json(); + let parentHash; + try { parentHash = (await (await api(backend, `/api/w/${parentWs}/scripts/get/p/${path}`)).json()).hash; } catch {} + expect((await api(backend, `/api/w/${parentWs}/scripts/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...s, lock: s.lock, parent_hash: parentHash }), + })).ok).toBe(true); + } else if (kind === "variable") { + const v = await (await api(backend, `/api/w/${FORK_ID}/variables/get/${path}?decrypt_secret=true`)).json(); + const exists = await (await api(backend, `/api/w/${parentWs}/variables/exists/${path}`)).json(); + if (exists) { + await api(backend, `/api/w/${parentWs}/variables/update/${path}?already_encrypted=false`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path, value: v.value ?? "", is_secret: v.is_secret, description: v.description ?? "" }), + }); + } else { + await api(backend, `/api/w/${parentWs}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path, value: v.value ?? "", is_secret: v.is_secret, description: v.description ?? "" }), + }); + } + } else if (kind === "resource") { + const res = await (await api(backend, `/api/w/${FORK_ID}/resources/get/${path}`)).json(); + const exists = await (await api(backend, `/api/w/${parentWs}/resources/exists/${path}`)).json(); + if (exists) { + await api(backend, `/api/w/${parentWs}/resources/update/${path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path, value: res.value, description: res.description ?? "" }), + }); + } else { + await api(backend, `/api/w/${parentWs}/resources/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path, value: res.value, resource_type: res.resource_type, description: res.description ?? "" }), + }); + } + } + } + + // Verify + expect((await (await api(backend, `/api/w/${parentWs}/scripts/get/p/f/merge_test/script_a`)).json()).content).toContain("fork v2"); + expect((await api(backend, `/api/w/${parentWs}/variables/get/f/merge_test/new_fork_var`)).ok).toBe(true); + expect(JSON.stringify((await (await api(backend, `/api/w/${parentWs}/resources/get/f/merge_test/resource_a`)).json()).value)).toContain("fork_modified"); + console.log(" ✓ Sub-test 1 passed: fork→parent deploy"); + + // --------------------------------------------------------------- + // Sub-test 2: Deploy parent→fork direction + // --------------------------------------------------------------- + console.log("\n--- Sub-test 2: parent→fork deploy ---"); + + // Create new script in parent + await api(backend, `/api/w/${parentWs}/scripts/create`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/script_b", content: "export function main() { return 'parent only'; }", language: "bun", summary: "Script B", schema: { type: "object", properties: {}, required: [] } }), + }); + + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/script_b", kind: "script", ahead: 0, behind: 1 }, + ]); + + const comp2 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + const behindDiffs = comp2.diffs.filter((d: any) => d.behind > 0); + expect(behindDiffs.length).toBeGreaterThanOrEqual(1); + + // Deploy parent→fork + for (const diff of behindDiffs) { + if (diff.kind === "script") { + const s = await (await api(backend, `/api/w/${parentWs}/scripts/get/p/${diff.path}`)).json(); + let forkHash; + try { forkHash = (await (await api(backend, `/api/w/${FORK_ID}/scripts/get/p/${diff.path}`)).json()).hash; } catch {} + await api(backend, `/api/w/${FORK_ID}/scripts/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...s, lock: s.lock, parent_hash: forkHash }), + }); + } + } + + expect((await api(backend, `/api/w/${FORK_ID}/scripts/get/p/f/merge_test/script_b`)).ok).toBe(true); + console.log(" ✓ Sub-test 2 passed: parent→fork deploy"); + + // --------------------------------------------------------------- + // Sub-test 3: Conflict detection + // --------------------------------------------------------------- + console.log("\n--- Sub-test 3: conflict detection ---"); + + // Create actual divergence: modify script_a differently in both workspaces + const parentScriptA = await (await api(backend, `/api/w/${parentWs}/scripts/get/p/f/merge_test/script_a`)).json(); + await api(backend, `/api/w/${parentWs}/scripts/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...parentScriptA, content: "export function main() { return 'parent conflict version'; }", parent_hash: parentScriptA.hash }), + }); + + const forkScriptA = await (await api(backend, `/api/w/${FORK_ID}/scripts/get/p/f/merge_test/script_a`)).json(); + await api(backend, `/api/w/${FORK_ID}/scripts/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...forkScriptA, content: "export function main() { return 'fork conflict version'; }", parent_hash: forkScriptA.hash }), + }); + + // Now populate the conflict diff + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/script_a", kind: "script", ahead: 2, behind: 1 }, + ]); + + const comp3 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + expect(comp3.summary.conflicts).toBeGreaterThanOrEqual(1); + const conflict = comp3.diffs.find((d: any) => d.path === "f/merge_test/script_a" && d.ahead > 0 && d.behind > 0); + expect(conflict).toBeDefined(); + console.log(" ✓ Sub-test 3 passed: conflict detected"); + + // --------------------------------------------------------------- + // Sub-test 4: Fork has correct parent_workspace_id + // --------------------------------------------------------------- + console.log("\n--- Sub-test 4: parent_workspace_id ---"); + // Check parent_workspace_id via direct SQL (no REST endpoint exposes this directly) + const dbUrl = process.env["DATABASE_URL"] || "postgres://postgres:changeme@localhost:5432"; + const dbProc = Bun.spawn( + ["psql", `${dbUrl}/postgres?sslmode=disable`, "-t", "-c", + `SELECT datname FROM pg_database WHERE datname LIKE 'windmill_test_%' ORDER BY datname DESC LIMIT 1`], + { stdout: "pipe", stderr: "pipe" } + ); + const testDb = (await new Response(dbProc.stdout).text()).trim(); + await dbProc.exited; + const parentProc = Bun.spawn( + ["psql", `${dbUrl}/${testDb}?sslmode=disable`, "-t", "-c", + `SELECT parent_workspace_id FROM workspace WHERE id = '${FORK_ID}'`], + { stdout: "pipe", stderr: "pipe" } + ); + const parentId = (await new Response(parentProc.stdout).text()).trim(); + await parentProc.exited; + expect(parentId).toBe(parentWs); + console.log(" ✓ Sub-test 4 passed: parent_workspace_id correct"); + + // --------------------------------------------------------------- + // Sub-test 5: resetDiffTally cleans unchanged items + // --------------------------------------------------------------- + console.log("\n--- Sub-test 5: resetDiffTally ---"); + + // Add a diff for an item that hasn't actually changed (var_a was deployed already) + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/var_a", kind: "variable", ahead: 1, behind: 0 }, + ]); + + const resetResp = await api(backend, `/api/w/${parentWs}/workspaces/reset_diff_tally/${FORK_ID}`, { method: "POST" }); + expect(resetResp.ok).toBe(true); + + const comp5 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + const varDiff = comp5.diffs.find((d: any) => d.path === "f/merge_test/var_a" && d.kind === "variable"); + // var_a was already deployed (same value in both), so it should be cleaned up + expect(varDiff).toBeUndefined(); + console.log(" ✓ Sub-test 5 passed: resetDiffTally cleaned unchanged items"); + + // --------------------------------------------------------------- + // Sub-test 6: All item types in one merge + // --------------------------------------------------------------- + console.log("\n--- Sub-test 6: all item types (script, variable, resource, resource_type, flow, app) ---"); + await deleteFork(backend, FORK_ID); + + // Create resource type + resource in parent + await api(backend, `/api/w/${parentWs}/resources/type/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ name: "merge_test_type", schema: { type: "object" }, description: "Type" }), + }); + await api(backend, `/api/w/${parentWs}/resources/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/res_t6", resource_type: "merge_test_type", value: { key: "parent" }, description: "R" }), + }); + // Create flow in parent + await api(backend, `/api/w/${parentWs}/flows/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/merge_test/flow_t6", summary: "Flow", schema: { type: "object", properties: {}, required: [] }, + value: { modules: [{ id: "a", value: { type: "rawscript", content: "export function main() { return 1; }", language: "bun", input_transforms: {} } }], failure_module: null, same_worker: false }, + }), + }); + // Create app in parent + await api(backend, `/api/w/${parentWs}/apps/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + path: "f/merge_test/app_t6", summary: "App", + value: { type: "rawscript", content: { v: 1 } }, + policy: { on_behalf_of: "", on_behalf_of_email: "", extra_perms: {}, execution_mode: "publisher" }, + }), + }); + + await createFork(backend, parentWs, FORK_ID); + + // Modify all in fork + const forkRes = await (await api(backend, `/api/w/${FORK_ID}/resources/get/f/merge_test/res_t6`)).json(); + await api(backend, `/api/w/${FORK_ID}/resources/update/f/merge_test/res_t6`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: forkRes.path, value: { key: "fork" }, description: "Fork" }), + }); + + const forkFlow = await (await api(backend, `/api/w/${FORK_ID}/flows/get/f/merge_test/flow_t6`)).json(); + forkFlow.value.modules[0].value.content = "export function main() { return 2; }"; + await api(backend, `/api/w/${FORK_ID}/flows/update/f/merge_test/flow_t6`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify(forkFlow), + }); + + const forkApp = await (await api(backend, `/api/w/${FORK_ID}/apps/get/p/f/merge_test/app_t6`)).json(); + await api(backend, `/api/w/${FORK_ID}/apps/update/f/merge_test/app_t6`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ ...forkApp, summary: "Fork App" }), + }); + + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/res_t6", kind: "resource", ahead: 1, behind: 0 }, + { path: "f/merge_test/flow_t6", kind: "flow", ahead: 1, behind: 0 }, + { path: "f/merge_test/app_t6", kind: "app", ahead: 1, behind: 0 }, + ]); + + const comp6 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + const ahead6 = comp6.diffs.filter((d: any) => d.ahead > 0); + expect(ahead6.length).toBeGreaterThanOrEqual(3); + + // Deploy all + for (const d of ahead6) { + if (d.kind === "resource") { + const r = await (await api(backend, `/api/w/${FORK_ID}/resources/get/${d.path}`)).json(); + await api(backend, `/api/w/${parentWs}/resources/update/${d.path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: d.path, value: r.value, description: r.description ?? "" }), + }); + } else if (d.kind === "flow") { + const f = await (await api(backend, `/api/w/${FORK_ID}/flows/get/${d.path}`)).json(); + for (const m of f.value?.modules ?? []) { if (m.value?.hash) m.value.hash = undefined; } + await api(backend, `/api/w/${parentWs}/flows/update/${d.path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify(f), + }); + } else if (d.kind === "app") { + const a = await (await api(backend, `/api/w/${FORK_ID}/apps/get/p/${d.path}`)).json(); + await api(backend, `/api/w/${parentWs}/apps/update/${d.path}`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify(a), + }); + } + } + + // Verify + const pRes = await (await api(backend, `/api/w/${parentWs}/resources/get/f/merge_test/res_t6`)).json(); + expect(JSON.stringify(pRes.value)).toContain("fork"); + const pFlow = await (await api(backend, `/api/w/${parentWs}/flows/get/f/merge_test/flow_t6`)).json(); + expect(pFlow.value.modules[0].value.content).toContain("return 2"); + const pApp = await (await api(backend, `/api/w/${parentWs}/apps/get/p/f/merge_test/app_t6`)).json(); + expect(pApp.summary).toBe("Fork App"); + console.log(" ✓ Sub-test 6 passed: all item types merged"); + + // --------------------------------------------------------------- + // Sub-test 7: Secret variables preserved across fork/merge + // --------------------------------------------------------------- + console.log("\n--- Sub-test 7: secret variable ---"); + await api(backend, `/api/w/${FORK_ID}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/secret_fork", value: "s3cret!", is_secret: true, description: "Secret" }), + }); + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/secret_fork", kind: "variable", ahead: 1, behind: 0 }, + ]); + const comp7 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + const secDiff = comp7.diffs.find((d: any) => d.path === "f/merge_test/secret_fork"); + expect(secDiff).toBeDefined(); + + const secVar = await (await api(backend, `/api/w/${FORK_ID}/variables/get/f/merge_test/secret_fork?decrypt_secret=true`)).json(); + await api(backend, `/api/w/${parentWs}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: secVar.path, value: secVar.value ?? "", is_secret: secVar.is_secret, description: secVar.description ?? "" }), + }); + const pSec = await (await api(backend, `/api/w/${parentWs}/variables/get/f/merge_test/secret_fork?decrypt_secret=true`)).json(); + expect(pSec.value).toBe("s3cret!"); + expect(pSec.is_secret).toBe(true); + console.log(" ✓ Sub-test 7 passed: secret variable preserved"); + + // --------------------------------------------------------------- + // Sub-test 8: Special characters in variable values + // --------------------------------------------------------------- + console.log("\n--- Sub-test 8: special characters ---"); + await api(backend, `/api/w/${FORK_ID}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/special", value: "hello\nworld\t\"quotes\" 'single' \\back 日本語 🎉", is_secret: false, description: "" }), + }); + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/special", kind: "variable", ahead: 1, behind: 0 }, + ]); + const forkSpecial = await (await api(backend, `/api/w/${FORK_ID}/variables/get/f/merge_test/special?decrypt_secret=true`)).json(); + await api(backend, `/api/w/${parentWs}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: forkSpecial.path, value: forkSpecial.value ?? "", is_secret: false, description: "" }), + }); + const pSpecial = await (await api(backend, `/api/w/${parentWs}/variables/get/f/merge_test/special?decrypt_secret=true`)).json(); + expect(pSpecial.value).toBe(forkSpecial.value); + console.log(" ✓ Sub-test 8 passed: special characters preserved"); + + // --------------------------------------------------------------- + // Sub-test 9: Partial deploy — only deployed items cleaned by resetDiffTally + // --------------------------------------------------------------- + console.log("\n--- Sub-test 9: partial deploy + resetDiffTally ---"); + await api(backend, `/api/w/${FORK_ID}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/partial_a", value: "a", is_secret: false, description: "A" }), + }); + await api(backend, `/api/w/${FORK_ID}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: "f/merge_test/partial_b", value: "b", is_secret: false, description: "B" }), + }); + await populateWorkspaceDiff(backend, parentWs, FORK_ID, [ + { path: "f/merge_test/partial_a", kind: "variable", ahead: 1, behind: 0 }, + { path: "f/merge_test/partial_b", kind: "variable", ahead: 1, behind: 0 }, + ]); + + // Deploy only partial_a + const va = await (await api(backend, `/api/w/${FORK_ID}/variables/get/f/merge_test/partial_a?decrypt_secret=true`)).json(); + await api(backend, `/api/w/${parentWs}/variables/create`, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ path: va.path, value: va.value ?? "", is_secret: false, description: "" }), + }); + + await api(backend, `/api/w/${parentWs}/workspaces/reset_diff_tally/${FORK_ID}`, { method: "POST" }); + await new Promise(r => setTimeout(r, 500)); + const comp9 = await (await api(backend, `/api/w/${parentWs}/workspaces/compare/${FORK_ID}`)).json(); + const bAfter = comp9.diffs.find((d: any) => d.path === "f/merge_test/partial_b"); + // partial_b was NOT deployed, so it must still appear in diffs + expect(bAfter).toBeDefined(); + expect(bAfter?.ahead).toBeGreaterThan(0); + console.log(" ✓ Sub-test 9 passed: partial deploy + resetDiffTally"); + + // --------------------------------------------------------------- + // Cleanup + // --------------------------------------------------------------- + await deleteFork(backend, FORK_ID); + console.log("\n✅ All sub-tests passed!"); + }); + }, + 300_000 +); diff --git a/cli/test/instance_configs_unit.test.ts b/cli/test/instance_configs_unit.test.ts new file mode 100644 index 0000000000..c8ba15c5dc --- /dev/null +++ b/cli/test/instance_configs_unit.test.ts @@ -0,0 +1,297 @@ +/** + * Unit tests for pullInstanceConfigs / pushInstanceConfigs in settings.ts. + * + * Verifies that: + * - pullInstanceConfigs writes only worker group configs (no alerts) + * - pushInstanceConfigs calls updateConfig with worker__ prefix + * - pushInstanceConfigs calls deleteConfig with worker__ prefix for removed configs + * - pushInstanceConfigs skips unchanged configs + */ + +import { expect, test, describe, beforeEach, afterEach, mock } from "bun:test"; +import { writeFile, readFile, mkdir, rm } from "node:fs/promises"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify as yamlStringify } from "yaml"; + +// Track calls to mocked wmill functions +let listWorkerGroupsResult: any[] = []; +let updateConfigCalls: { name: string; requestBody: any }[] = []; +let deleteConfigCalls: { name: string }[] = []; + +// Mock the wmill module before importing settings.ts +mock.module("../gen/services.gen.ts", () => ({ + listWorkerGroups: async () => listWorkerGroupsResult, + updateConfig: async (args: { name: string; requestBody: any }) => { + updateConfigCalls.push(args); + }, + deleteConfig: async (args: { name: string }) => { + deleteConfigCalls.push(args); + }, + listConfigs: async () => { + throw new Error("listConfigs should not be called"); + }, +})); + +import { + pullInstanceConfigs, + pushInstanceConfigs, + readLocalConfigs, +} from "../src/core/settings.ts"; + +describe("instance configs", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = mkdtempSync(join(tmpdir(), "wm-config-test-")); + listWorkerGroupsResult = []; + updateConfigCalls = []; + deleteConfigCalls = []; + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + // ========================================================================= + // readLocalConfigs + // ========================================================================= + + describe("readLocalConfigs", () => { + test("reads configs from instance_configs.yaml", async () => { + const configs = [ + { name: "default", config: { worker_tags: ["deno", "bun"] } }, + { name: "gpu", config: { dedicated_worker: "ws:f/gpu_script" } }, + ]; + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(configs), + "utf-8" + ); + + const result = await readLocalConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + expect(result).toEqual(configs); + }); + + test("returns empty array when file does not exist", async () => { + const result = await readLocalConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + expect(result).toEqual([]); + }); + }); + + // ========================================================================= + // pullInstanceConfigs + // ========================================================================= + + describe("pullInstanceConfigs", () => { + test("writes worker group configs to instance_configs.yaml", async () => { + listWorkerGroupsResult = [ + { name: "default", config: { worker_tags: ["deno", "bun"] } }, + { name: "native", config: { worker_tags: ["nativets"] } }, + ]; + + // readLocalConfigs sets instanceConfigsPath when prefix is used + const opts = { + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }; + await readLocalConfigs(opts); + await pullInstanceConfigs(opts); + + const content = await readFile( + join(tempDir, "instance_configs.yaml"), + "utf-8" + ); + expect(content).toContain("default"); + expect(content).toContain("native"); + expect(content).toContain("deno"); + // Should not contain alert entries (listWorkerGroups filters them) + expect(content).not.toContain("alert"); + }); + + test("preview mode returns change count without writing file", async () => { + listWorkerGroupsResult = [ + { name: "default", config: { worker_tags: ["deno"] } }, + ]; + + const changes = await pullInstanceConfigs( + { + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }, + true + ); + + // One remote config not in local = 1 change + expect(changes).toBe(1); + }); + + test("preview mode returns 0 when remote matches local", async () => { + const configs = [ + { name: "default", config: { worker_tags: ["deno"] } }, + ]; + listWorkerGroupsResult = configs; + + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(configs), + "utf-8" + ); + + const changes = await pullInstanceConfigs( + { + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }, + true + ); + + expect(changes).toBe(0); + }); + }); + + // ========================================================================= + // pushInstanceConfigs + // ========================================================================= + + describe("pushInstanceConfigs", () => { + test("calls updateConfig with worker__ prefix for new configs", async () => { + listWorkerGroupsResult = []; + const localConfigs = [ + { name: "mygroup", config: { worker_tags: ["python3"] } }, + ]; + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(localConfigs), + "utf-8" + ); + + await pushInstanceConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + + expect(updateConfigCalls).toHaveLength(1); + expect(updateConfigCalls[0].name).toBe("worker__mygroup"); + expect(updateConfigCalls[0].requestBody).toEqual({ + worker_tags: ["python3"], + }); + }); + + test("calls deleteConfig with worker__ prefix for removed configs", async () => { + listWorkerGroupsResult = [ + { name: "old_group", config: { worker_tags: ["bash"] } }, + ]; + // Empty local configs = old_group should be deleted + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify([]), + "utf-8" + ); + + await pushInstanceConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + + expect(deleteConfigCalls).toHaveLength(1); + expect(deleteConfigCalls[0].name).toBe("worker__old_group"); + }); + + test("skips unchanged configs", async () => { + const configs = [ + { name: "default", config: { worker_tags: ["deno"] } }, + ]; + listWorkerGroupsResult = configs; + + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(configs), + "utf-8" + ); + + await pushInstanceConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + + expect(updateConfigCalls).toHaveLength(0); + expect(deleteConfigCalls).toHaveLength(0); + }); + + test("updates changed configs and deletes removed ones", async () => { + listWorkerGroupsResult = [ + { name: "keep", config: { worker_tags: ["old_tag"] } }, + { name: "remove_me", config: { worker_tags: ["bash"] } }, + ]; + const localConfigs = [ + { name: "keep", config: { worker_tags: ["new_tag"] } }, + { name: "add_me", config: { worker_tags: ["python3"] } }, + ]; + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(localConfigs), + "utf-8" + ); + + await pushInstanceConfigs({ + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }); + + // "keep" was changed, "add_me" is new + expect(updateConfigCalls).toHaveLength(2); + const updateNames = updateConfigCalls.map((c) => c.name).sort(); + expect(updateNames).toEqual(["worker__add_me", "worker__keep"]); + + // "remove_me" was deleted + expect(deleteConfigCalls).toHaveLength(1); + expect(deleteConfigCalls[0].name).toBe("worker__remove_me"); + }); + + test("preview mode returns change count without calling API", async () => { + listWorkerGroupsResult = [ + { name: "default", config: { worker_tags: ["deno"] } }, + ]; + const localConfigs = [ + { name: "default", config: { worker_tags: ["bun"] } }, + { name: "new_group", config: { worker_tags: ["go"] } }, + ]; + await writeFile( + join(tempDir, "instance_configs.yaml"), + yamlStringify(localConfigs), + "utf-8" + ); + + const changes = await pushInstanceConfigs( + { + prefix: tempDir, + folderPerInstance: true, + prefixSettings: true, + }, + true + ); + + // "default" changed + "new_group" added = 2 changes + expect(changes).toBe(2); + expect(updateConfigCalls).toHaveLength(0); + expect(deleteConfigCalls).toHaveLength(0); + }); + }); +}); diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 07cf8b90b0..56dcfd9905 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.7", + "version": "1.3.8", "description": "Internal utility functions for Windmill", "main": "dist/cjs/index.js", "module": "dist/esm/index.js", diff --git a/cli/windmill-utils-internal/src/deploy.ts b/cli/windmill-utils-internal/src/deploy.ts new file mode 100644 index 0000000000..059a4764ea --- /dev/null +++ b/cli/windmill-utils-internal/src/deploy.ts @@ -0,0 +1,624 @@ +/** + * Shared deploy logic for workspace fork/merge operations. + * + * Used by both the CLI (`wmill workspace merge`) and the frontend + * (`CompareWorkspaces.svelte`). The caller provides a {@link DeployProvider} + * that wraps the concrete API client (class-based for the frontend, + * standalone functions for the CLI). + */ + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type DeployKind = + | "script" + | "flow" + | "app" + | "raw_app" + | "resource" + | "variable" + | "resource_type" + | "folder"; + +export interface DeployResult { + success: boolean; + error?: string; +} + +/** + * Abstraction over the generated API client. + * Both the frontend (class-based services) and the CLI (standalone functions) + * can satisfy this interface with a thin adapter. + */ +export interface DeployProvider { + // Existence checks + existsFlowByPath(p: { workspace: string; path: string }): Promise; + existsScriptByPath(p: { workspace: string; path: string }): Promise; + existsApp(p: { workspace: string; path: string }): Promise; + existsVariable(p: { workspace: string; path: string }): Promise; + existsResource(p: { workspace: string; path: string }): Promise; + existsResourceType(p: { workspace: string; path: string }): Promise; + existsFolder(p: { workspace: string; name: string }): Promise; + // Flows + getFlowByPath(p: { workspace: string; path: string }): Promise; + createFlow(p: { workspace: string; requestBody: any }): Promise; + updateFlow(p: { + workspace: string; + path: string; + requestBody: any; + }): Promise; + archiveFlowByPath(p: { + workspace: string; + path: string; + requestBody: any; + }): Promise; + // Scripts + getScriptByPath(p: { workspace: string; path: string }): Promise; + createScript(p: { workspace: string; requestBody: any }): Promise; + archiveScriptByPath(p: { + workspace: string; + path: string; + }): Promise; + // Apps + getAppByPath(p: { workspace: string; path: string }): Promise; + createApp(p: { workspace: string; requestBody: any }): Promise; + updateApp(p: { + workspace: string; + path: string; + requestBody: any; + }): Promise; + createAppRaw(p: { workspace: string; formData: any }): Promise; + updateAppRaw(p: { + workspace: string; + path: string; + formData: any; + }): Promise; + getPublicSecretOfLatestVersionOfApp(p: { + workspace: string; + path: string; + }): Promise; + getRawAppData(p: { + secretWithExtension: string; + workspace: string; + }): Promise; + deleteApp(p: { workspace: string; path: string }): Promise; + // Variables + getVariable(p: { + workspace: string; + path: string; + decryptSecret?: boolean; + }): Promise; + createVariable(p: { workspace: string; requestBody: any }): Promise; + updateVariable(p: { + workspace: string; + path: string; + requestBody: any; + alreadyEncrypted?: boolean; + }): Promise; + deleteVariable(p: { workspace: string; path: string }): Promise; + // Resources + getResource(p: { workspace: string; path: string }): Promise; + createResource(p: { workspace: string; requestBody: any }): Promise; + updateResource(p: { + workspace: string; + path: string; + requestBody: any; + }): Promise; + deleteResource(p: { workspace: string; path: string }): Promise; + // Resource types + getResourceType(p: { workspace: string; path: string }): Promise; + createResourceType(p: { workspace: string; requestBody: any }): Promise; + updateResourceType(p: { + workspace: string; + path: string; + requestBody: any; + }): Promise; + deleteResourceType(p: { workspace: string; path: string }): Promise; + // Folders + getFolder(p: { workspace: string; name: string }): Promise; + createFolder(p: { workspace: string; requestBody: any }): Promise; + updateFolder(p: { + workspace: string; + name: string; + requestBody: any; + }): Promise; + deleteFolder(p: { workspace: string; name: string }): Promise; +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** Folder diff paths carry the `f/` prefix; folder API endpoints expect just the name. */ +export function folderName(path: string): string { + return path.replace(/^f\//, ""); +} + +function getSubModules(flowModule: any): any[][] { + const type = flowModule?.value?.type; + if (type === "forloopflow" || type === "whileloopflow") { + return [flowModule.value.modules ?? []]; + } else if (type === "branchall") { + return (flowModule.value.branches ?? []).map( + (branch: any) => branch.modules ?? [] + ); + } else if (type === "branchone") { + return [ + ...(flowModule.value.branches ?? []).map((b: any) => b.modules ?? []), + flowModule.value.default ?? [], + ]; + } else if (type === "aiagent") { + if (flowModule.value.tools) { + return [ + flowModule.value.tools + .filter( + (t: any) => + t.value?.type === "script" || t.value?.type === "flow" + ) + .map((t: any) => ({ + id: t.id, + value: t.value, + summary: t.summary, + })), + ]; + } + } + return []; +} + +function getAllSubmodules(flowModule: any): any[] { + return getSubModules(flowModule) + .map((modules) => modules.flatMap((m: any) => [m, ...getAllSubmodules(m)])) + .flat(); +} + +/** Recursively collect all modules from a flow definition, including the failure module. */ +export function getAllModules( + flowModules: any[], + failureModule?: any +): any[] { + return [ + ...flowModules, + ...flowModules.flatMap((x) => getAllSubmodules(x)), + ...(failureModule ? [failureModule] : []), + ]; +} + +function toError(e: unknown): string { + const err = e as { body?: string; message?: string }; + return err.body || err.message || String(e); +} + +// --------------------------------------------------------------------------- +// checkItemExists +// --------------------------------------------------------------------------- + +export async function checkItemExists( + provider: DeployProvider, + kind: DeployKind, + path: string, + workspace: string +): Promise { + if (kind === "flow") { + return provider.existsFlowByPath({ workspace, path }); + } else if (kind === "script") { + return provider.existsScriptByPath({ workspace, path }); + } else if (kind === "app" || kind === "raw_app") { + return provider.existsApp({ workspace, path }); + } else if (kind === "variable") { + return provider.existsVariable({ workspace, path }); + } else if (kind === "resource") { + return provider.existsResource({ workspace, path }); + } else if (kind === "resource_type") { + return provider.existsResourceType({ workspace, path }); + } else if (kind === "folder") { + return provider.existsFolder({ workspace, name: folderName(path) }); + } + throw new Error(`Unknown kind: ${kind}`); +} + +// --------------------------------------------------------------------------- +// deployItem +// --------------------------------------------------------------------------- + +export async function deployItem( + provider: DeployProvider, + kind: DeployKind, + path: string, + workspaceFrom: string, + workspaceTo: string, + onBehalfOf?: string +): Promise { + const preserveOnBehalfOf = onBehalfOf !== undefined; + + try { + const alreadyExists = await checkItemExists( + provider, + kind, + path, + workspaceTo + ); + + if (kind === "flow") { + const flow = await provider.getFlowByPath({ + workspace: workspaceFrom, + path, + }); + // Clear inline script hashes so the target workspace resolves by path + getAllModules( + flow.value?.modules ?? [], + flow.value?.failure_module + ).forEach((x: any) => { + if (x.value?.type === "script" && x.value.hash != undefined) { + x.value.hash = undefined; + } + }); + if (alreadyExists) { + await provider.updateFlow({ + workspace: workspaceTo, + path, + requestBody: { + ...flow, + preserve_on_behalf_of: preserveOnBehalfOf, + on_behalf_of_email: onBehalfOf, + }, + }); + } else { + await provider.createFlow({ + workspace: workspaceTo, + requestBody: { + ...flow, + preserve_on_behalf_of: preserveOnBehalfOf, + on_behalf_of_email: onBehalfOf, + }, + }); + } + } else if (kind === "script") { + const script = await provider.getScriptByPath({ + workspace: workspaceFrom, + path, + }); + let parentHash: string | undefined; + if (alreadyExists) { + const existing = await provider.getScriptByPath({ + workspace: workspaceTo, + path, + }); + parentHash = existing.hash; + } + await provider.createScript({ + workspace: workspaceTo, + requestBody: { + ...script, + lock: script.lock, + parent_hash: parentHash, + preserve_on_behalf_of: preserveOnBehalfOf, + on_behalf_of_email: onBehalfOf, + }, + }); + } else if (kind === "app" || kind === "raw_app") { + const app = await provider.getAppByPath({ + workspace: workspaceFrom, + path, + }); + if (alreadyExists) { + if (app.raw_app) { + const secret = await provider.getPublicSecretOfLatestVersionOfApp({ + workspace: workspaceFrom, + path: app.path, + }); + const js = await provider.getRawAppData({ + secretWithExtension: `${secret}.js`, + workspace: workspaceFrom, + }); + const css = await provider.getRawAppData({ + secretWithExtension: `${secret}.css`, + workspace: workspaceFrom, + }); + await provider.updateAppRaw({ + workspace: workspaceTo, + path, + formData: { + app: { ...app, preserve_on_behalf_of: preserveOnBehalfOf }, + css, + js, + }, + }); + } else { + await provider.updateApp({ + workspace: workspaceTo, + path, + requestBody: { + ...app, + preserve_on_behalf_of: preserveOnBehalfOf, + }, + }); + } + } else { + if (app.raw_app) { + const secret = await provider.getPublicSecretOfLatestVersionOfApp({ + workspace: workspaceFrom, + path: app.path, + }); + const js = await provider.getRawAppData({ + secretWithExtension: `${secret}.js`, + workspace: workspaceFrom, + }); + const css = await provider.getRawAppData({ + secretWithExtension: `${secret}.css`, + workspace: workspaceFrom, + }); + await provider.createAppRaw({ + workspace: workspaceTo, + formData: { + app: { ...app, preserve_on_behalf_of: preserveOnBehalfOf }, + css, + js, + }, + }); + } else { + await provider.createApp({ + workspace: workspaceTo, + requestBody: { + ...app, + preserve_on_behalf_of: preserveOnBehalfOf, + }, + }); + } + } + } else if (kind === "variable") { + const variable = await provider.getVariable({ + workspace: workspaceFrom, + path, + decryptSecret: true, + }); + if (alreadyExists) { + await provider.updateVariable({ + workspace: workspaceTo, + path, + requestBody: { + path, + value: variable.value ?? "", + is_secret: variable.is_secret, + description: variable.description ?? "", + }, + alreadyEncrypted: false, + }); + } else { + await provider.createVariable({ + workspace: workspaceTo, + requestBody: { + path, + value: variable.value ?? "", + is_secret: variable.is_secret, + description: variable.description ?? "", + }, + }); + } + } else if (kind === "resource") { + const resource = await provider.getResource({ + workspace: workspaceFrom, + path, + }); + if (alreadyExists) { + await provider.updateResource({ + workspace: workspaceTo, + path, + requestBody: { + path, + value: resource.value ?? "", + description: resource.description ?? "", + }, + }); + } else { + await provider.createResource({ + workspace: workspaceTo, + requestBody: { + path, + value: resource.value ?? "", + resource_type: resource.resource_type, + description: resource.description ?? "", + }, + }); + } + } else if (kind === "resource_type") { + const rt = await provider.getResourceType({ + workspace: workspaceFrom, + path, + }); + if (alreadyExists) { + await provider.updateResourceType({ + workspace: workspaceTo, + path, + requestBody: { + schema: rt.schema, + description: rt.description ?? "", + }, + }); + } else { + await provider.createResourceType({ + workspace: workspaceTo, + requestBody: { + name: rt.name, + schema: rt.schema, + description: rt.description ?? "", + }, + }); + } + } else if (kind === "folder") { + const name = folderName(path); + const folder = await provider.getFolder({ + workspace: workspaceFrom, + name, + }); + if (alreadyExists) { + await provider.updateFolder({ + workspace: workspaceTo, + name, + requestBody: { + owners: folder.owners, + extra_perms: folder.extra_perms, + summary: folder.summary ?? undefined, + }, + }); + } else { + await provider.createFolder({ + workspace: workspaceTo, + requestBody: { + name, + owners: folder.owners, + extra_perms: folder.extra_perms, + summary: folder.summary ?? undefined, + }, + }); + } + } else { + throw new Error(`Unknown kind: ${kind}`); + } + + return { success: true }; + } catch (e: unknown) { + return { success: false, error: toError(e) }; + } +} + +// --------------------------------------------------------------------------- +// deleteItemInWorkspace +// --------------------------------------------------------------------------- + +/** + * Delete/archive an item in a workspace. + * Scripts and flows are archived (reversible). Other types are deleted. + */ +export async function deleteItemInWorkspace( + provider: DeployProvider, + kind: DeployKind, + path: string, + workspace: string +): Promise { + try { + if (kind === "script") { + await provider.archiveScriptByPath({ workspace, path }); + } else if (kind === "flow") { + await provider.archiveFlowByPath({ + workspace, + path, + requestBody: { archived: true }, + }); + } else if (kind === "app" || kind === "raw_app") { + await provider.deleteApp({ workspace, path }); + } else if (kind === "variable") { + await provider.deleteVariable({ workspace, path }); + } else if (kind === "resource") { + await provider.deleteResource({ workspace, path }); + } else if (kind === "resource_type") { + await provider.deleteResourceType({ workspace, path }); + } else if (kind === "folder") { + await provider.deleteFolder({ workspace, name: folderName(path) }); + } else { + throw new Error(`Deletion not supported for kind: ${kind}`); + } + return { success: true }; + } catch (e: unknown) { + return { success: false, error: toError(e) }; + } +} + +// --------------------------------------------------------------------------- +// getOnBehalfOf +// --------------------------------------------------------------------------- + +/** + * Get the value of an item for diff comparison. + * Returns a normalized representation suitable for JSON comparison. + */ +export async function getItemValue( + provider: DeployProvider, + kind: DeployKind, + path: string, + workspace: string +): Promise { + try { + if (kind === "flow") { + const flow = await provider.getFlowByPath({ workspace, path }); + getAllModules(flow.value?.modules ?? [], flow.value?.failure_module).forEach( + (x: any) => { + if (x.value?.type === "script" && x.value.hash != undefined) { + x.value.hash = undefined; + } + } + ); + return { + summary: flow.summary, + description: flow.description, + value: flow.value, + }; + } else if (kind === "script") { + const script = await provider.getScriptByPath({ workspace, path }); + return { + content: script.content, + lock: script.lock, + schema: script.schema, + summary: script.summary, + language: script.language, + }; + } else if (kind === "app" || kind === "raw_app") { + return await provider.getAppByPath({ workspace, path }); + } else if (kind === "variable") { + const variable = await provider.getVariable({ + workspace, + path, + decryptSecret: true, + }); + return variable.value; + } else if (kind === "resource") { + const resource = await provider.getResource({ workspace, path }); + return resource.value; + } else if (kind === "resource_type") { + const rt = await provider.getResourceType({ workspace, path }); + return rt.schema; + } else if (kind === "folder") { + const folder = await provider.getFolder({ + workspace, + name: folderName(path), + }); + return { + name: folder.name, + owners: folder.owners, + extra_perms: folder.extra_perms, + summary: folder.summary, + }; + } + } catch { + // Item may not exist + } + return {}; +} + +/** + * Fetch the on_behalf_of value for a deployable item. + * Returns an email for flows/scripts/apps, or undefined if not applicable. + */ +export async function getOnBehalfOf( + provider: DeployProvider, + kind: DeployKind, + path: string, + workspace: string +): Promise { + try { + if (kind === "flow") { + const flow = await provider.getFlowByPath({ workspace, path }); + return flow.on_behalf_of_email; + } else if (kind === "script") { + const script = await provider.getScriptByPath({ workspace, path }); + return script.on_behalf_of_email; + } else if (kind === "app" || kind === "raw_app") { + const app = await provider.getAppByPath({ workspace, path }); + return app.policy?.on_behalf_of_email; + } + } catch { + // Item may not exist + } + return undefined; +} diff --git a/cli/windmill-utils-internal/src/index.ts b/cli/windmill-utils-internal/src/index.ts index da314a7c5a..52c489da4c 100644 --- a/cli/windmill-utils-internal/src/index.ts +++ b/cli/windmill-utils-internal/src/index.ts @@ -12,4 +12,5 @@ export * from "./inline-scripts"; export * from "./path-utils"; export * from "./parse"; export * from "./config"; +export * from "./deploy"; export { SEP, DELIMITER } from "./constants"; \ No newline at end of file diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 61a365e496..181d87b380 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -36,7 +36,15 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 \ + && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release \ + && rm -rf /var/lib/apt/lists/* + +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* ENV TZ=Etc/UTC diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 7ba85a0358..cc74d14996 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -36,7 +36,15 @@ ENV PATH=/usr/local/bin:/root/.local/bin:/tmp/.local/bin:$PATH # Install system dependencies RUN apt-get update \ - && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 \ + && apt-get install -y ca-certificates wget curl git jq unzip unixodbc xmlsec1 gnupg lsb-release \ + && rm -rf /var/lib/apt/lists/* + +# Install latest PostgreSQL client (pg_dump) from official PostgreSQL apt repository +RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends postgresql-client \ + && apt-get clean \ && rm -rf /var/lib/apt/lists/* ENV TZ=Etc/UTC diff --git a/ee-repo-ref.txt b/ee-repo-ref.txt index 48d6ca2656..6dcdb4be96 100644 --- a/ee-repo-ref.txt +++ b/ee-repo-ref.txt @@ -1 +1 @@ -87fcd9a7e12e9b1ad8e5e27dddca466a8e2b2e5c +bb2c05c399d27b0fadedbf6e1fa5c4f19e71f37f diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c9b1de6017..c38109cad6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.677.0", + "version": "1.680.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.677.0", + "version": "1.680.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -90,7 +90,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.4", + "windmill-utils-internal": "^1.3.8", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -13723,9 +13723,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.4.tgz", - "integrity": "sha512-XVypDKIZ6P4fwIjZwvuvq1m+j0rtAA7BDp1rI2F7hQ+VBKZUHsLskP+jgstXs+kN1LqGGsQJj4ecMYDImpIZ6A==", + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.8.tgz", + "integrity": "sha512-FtVEvAI2PIqPTEpowTjo5c5JkYe09Scu9zcwzJutOWMEh4aDdzOejaG7EZTac0pk+dK4JB46+nbl82hhLsL8Mw==", "license": "Apache 2.0" }, "node_modules/word-wrap": { diff --git a/frontend/package.json b/frontend/package.json index bce684081b..68bb4487c9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.677.0", + "version": "1.680.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -163,7 +163,7 @@ "windmill-parser-wasm-wac": "1.668.6", "windmill-parser-wasm-yaml": "1.593.0", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.4", + "windmill-utils-internal": "^1.3.8", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -592,4 +592,4 @@ "@rollup/rollup-linux-x64-gnu": "^4.35.0", "fsevents": "^2.3.3" } -} \ No newline at end of file +} diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 4f9184c93c..68f1625e17 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -44,7 +44,13 @@ import { userWorkspaces, workspaceStore } from '$lib/stores' import type { Kind } from '$lib/utils_deployable' - import { deployItem, getItemValue, getOnBehalfOf } from '$lib/utils_workspace_deploy' + import { + deployItem, + deleteItemInWorkspace, + getItemValue, + getOnBehalfOf, + type DeployResult + } from '$lib/utils_workspace_deploy' import Tooltip from './Tooltip.svelte' import OnBehalfOfSelector, { needsOnBehalfOfSelection, @@ -60,6 +66,7 @@ import { base } from '$lib/base' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import DatatableSchemaDiff from './DatatableSchemaDiff.svelte' interface Props { currentWorkspaceId: string @@ -306,13 +313,27 @@ ) { deploymentStatus[statusPath] = { status: 'loading' } - const result = await deployItem({ - kind, - path, - workspaceFrom, - workspaceTo: workspaceToDeployTo, - onBehalfOf: getOnBehalfOfForDeploy(statusPath, kind) - }) + // Check if the item was deleted in the source workspace. + // If so, archive/delete it in the target workspace instead of copying. + const diff = comparison?.diffs.find((d) => getItemKey(d) === statusPath) + const itemDeletedInSource = diff + ? mergeIntoParent + ? diff.exists_in_fork === false + : diff.exists_in_source === false + : false + + let result: DeployResult + if (itemDeletedInSource) { + result = await deleteItemInWorkspace(kind, path, workspaceToDeployTo) + } else { + result = await deployItem({ + kind, + path, + workspaceFrom, + workspaceTo: workspaceToDeployTo, + onBehalfOf: getOnBehalfOfForDeploy(statusPath, kind) + }) + } if (result.success) { deploymentStatus[statusPath] = { status: 'deployed' } @@ -716,382 +737,408 @@ {@const selectedConflicts = conflictingDiffs.filter((e) => selectedItems.includes(getItemKey(e)) ).length} - - selectableDiffs.some((d) => getItemKey(d) === item.key)} - {allSelected} - onToggleItem={(item) => { - const diff = comparison?.diffs.find((d) => getItemKey(d) === item.key) - if (diff) toggleItem(diff) - }} - onSelectAll={selectAll} - onDeselectAll={deselectAll} - emptyMessage="No comparison data available" - > - {#snippet header()} -
-
-
- - {#snippet children({ item })} - - - {/snippet} - - {#if currentWorkspaceInfo && parentWorkspaceInfo} - - merge: - {#if mergeIntoParent} - - {currentWorkspaceInfo.id} - {:else} - - {parentWorkspaceInfo.id} +
+
+ selectableDiffs.some((d) => getItemKey(d) === item.key)} + {allSelected} + onToggleItem={(item) => { + const diff = comparison?.diffs.find((d) => getItemKey(d) === item.key) + if (diff) toggleItem(diff) + }} + onSelectAll={selectAll} + onDeselectAll={deselectAll} + emptyMessage="No comparison data available" + > + {#snippet header()} +
+
+
+ + {#snippet children({ item })} + + + {/snippet} + + {#if currentWorkspaceInfo && parentWorkspaceInfo} +
+ + merge: + {#if mergeIntoParent} + + {currentWorkspaceInfo.id} + {:else} + + {parentWorkspaceInfo.id} + {/if} + + + + into: + {#if !mergeIntoParent} + + {currentWorkspaceInfo.id} + {:else} + + {parentWorkspaceInfo.id} + {/if} + +
{/if} - - - - into: - {#if !mergeIntoParent} - - {currentWorkspaceInfo.id} - {:else} - - {parentWorkspaceInfo.id} - {/if} - - {/if} +
+ + {comparison.summary.total_diffs} total items + + + {selectableDiffs.length} + {mergeIntoParent ? 'deployable' : 'updateable'} + + {#if conflictingDiffs.length > 0} + + + {conflictingDiffs.length} conflicts + + {/if} +
+
+
-
- - {comparison.summary.total_diffs} total items - - - {selectableDiffs.length} - {mergeIntoParent ? 'deployable' : 'updateable'} - - {#if conflictingDiffs.length > 0} - - - {conflictingDiffs.length} conflicts - - {/if} -
-
-
- {/snippet} + {/snippet} - {#snippet alerts()} - {#if mergeIntoParent} - { - canDeployToParent = canDeploy - }} - /> - {/if} - {#if conflictingDiffs.length > 0} - - - {conflictingDiffs.length} item{conflictingDiffs.length !== 1 ? 's have' : ' has'} conflicting - changes, it was modified on the original workspace while changes were made on this fork. - Make sure to resolve these before merging. - - - {/if} - {#if hasBehindChanges && hasAheadChanges && !(mergeIntoParent && !canDeployToParent)} - - You have items behind '{parentWorkspaceId}'. You need to update and test your changes - before being able to deploy. - - { + canDeployToParent = canDeploy + }} /> - Override: Deploy despite {itemsWithBehindChanges.length} outdated item{itemsWithBehindChanges.length !== - 1 - ? 's' - : ''} - - - {/if} - {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} - - {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} - This fork is ahead and behind its parent - {:else if !comparison.all_behind_items_visible} - This fork is behind of its parent - {:else if !comparison.all_ahead_items_visible} - This fork is ahead of its parent {/if} - and some of the changes are not visible by you. Only a user with access to the whole context - may deploy or update this fork. You can share the link to this page to someone with proper - permissions to get it deployed. - - {/if} - {/snippet} - - {#snippet itemSummary(item)} - {@const diff = item.diff as WorkspaceItemDiff} - {@const key = item.key} - {@const isSelectable = selectableDiffs.includes(diff)} - {@const oldSummary = mergeIntoParent ? summaryCache[key]?.parent : summaryCache[key]?.current} - {@const newSummary = mergeIntoParent ? summaryCache[key]?.current : summaryCache[key]?.parent} - {@const existsInBothWorkspaces = !( - (diff.exists_in_fork && !diff.exists_in_source) || - (!diff.exists_in_fork && diff.exists_in_source) - )} - {#if oldSummary != newSummary && isSelectable && existsInBothWorkspaces} - {oldSummary || diff.path} - {newSummary || diff.path} - {:else if !existsInBothWorkspaces} - {newSummary || oldSummary || diff.path} - {:else} - {newSummary || diff.path} - {/if} - {/snippet} - - {#snippet itemActions(item)} - {@const diff = item.diff as WorkspaceItemDiff} - {@const key = item.key} - {@const targetOnBehalfOf = getTargetOnBehalfOf(key)} - {@const isConflict = diff.ahead > 0 && diff.behind > 0} - {@const existsInBothWorkspaces = !( - (diff.exists_in_fork && !diff.exists_in_source) || - (!diff.exists_in_fork && diff.exists_in_source) - )} - - {#if itemNeedsOnBehalfOfSelection(key, diff.kind)} - { - onBehalfOfChoice[key] = choice - if (details) customOnBehalfOf[key] = details - }} - kind={diff.kind} - canPreserve={canPreserveOnBehalfOf} - customValue={customOnBehalfOf[key]?.permissionedAs} - /> - {/if} - {#if diff.kind === 'raw_app'} - Raw - {/if} - - {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead == 0 && diff.behind > 0} - New - {/if} - {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead > 0} - Deleted - {/if} - {#if diff.exists_in_fork && !diff.exists_in_source && diff.behind > 0} - Deleted - {/if} - {#if diff.exists_in_fork && !diff.exists_in_source && diff.ahead > 0 && diff.behind == 0} - New - {/if} - {#if !deploymentStatus[key] || deploymentStatus[key].status != 'deployed'} -
- {#if isConflict || existsInBothWorkspaces} - {#if diff.ahead > 0} - - - {diff.ahead} ahead - - {/if} - {#if diff.behind > 0} - - - {diff.behind} behind - - {/if} - {#if isConflict} - - - Conflict - - {/if} - {/if} -
-
- -
- {/if} - {/snippet} - - {#snippet footer()} -
-
- -
- {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} - {#if !(mergeIntoParent && !canDeployToParent)} - - {#if hasUnselectedOnBehalfOf} - - You must set the "on behalf of" user for all items before deploying - - The "run on behalf of" field defines which user's permissions will be applied - during execution. Make sure this is set to an appropriate user before deploying. - - - {/if} - {/if} - {/if} - - {#if deploymentErrorMessage != ''} - + {#if conflictingDiffs.length > 0} + - {deploymentErrorMessage} + {conflictingDiffs.length} item{conflictingDiffs.length !== 1 ? 's have' : ' has'} conflicting + changes, it was modified on the original workspace while changes were made on this fork. + Make sure to resolve these before merging. {/if} -
-
- {/snippet} - - - -
-
-

Triggers created in this fork

- {#if !loadingTriggers} - {forkTriggers.length} trigger{forkTriggers.length !== 1 ? 's' : ''} - {/if} -
- - - When forking a workspace, triggers are not forked to avoid unnecessary executions or - collisions. If you created this triggers with the intention of deploying them to the parent - workspace, you can do so here. Otherwise it is recommended to delete them or disable them. - - - {#if loadingTriggers} -
- - Loading triggers... -
- {:else if forkTriggers.length === 0} -
- No triggers in this fork workspace. -
- {:else} -
- {#each forkTriggers as trigger (trigger.triggerKind + ':' + trigger.path)} - - {#snippet customSummary()} - {getTriggerDisplayName(trigger.triggerKind)} - → - {trigger.scriptPath} - {#if trigger.isFlow} - flow + {#if hasBehindChanges && hasAheadChanges && !(mergeIntoParent && !canDeployToParent)} + + You have items behind '{parentWorkspaceId}'. You need to update and test your changes + before being able to deploy. + + + Override: Deploy despite {itemsWithBehindChanges.length} outdated item{itemsWithBehindChanges.length !== + 1 + ? 's' + : ''} + + + {/if} + {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} + + {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} + This fork is ahead and behind its parent + {:else if !comparison.all_behind_items_visible} + This fork is behind of its parent + {:else if !comparison.all_ahead_items_visible} + This fork is ahead of its parent {/if} - {#if trigger.extraLabel} - ({trigger.extraLabel}) - {/if} - {/snippet} - {#snippet actions()} - {#if trigger.enabled != null} - - {trigger.enabled ? 'Enabled' : 'Disabled'} - + and some of the changes are not visible by you. Only a user with access to the whole context + may deploy or update this fork. You can share the link to this page to someone with proper + permissions to get it deployed. + + {/if} + {/snippet} + + {#snippet itemSummary(item)} + {@const diff = item.diff as WorkspaceItemDiff} + {@const key = item.key} + {@const isSelectable = selectableDiffs.includes(diff)} + {@const oldSummary = mergeIntoParent + ? summaryCache[key]?.parent + : summaryCache[key]?.current} + {@const newSummary = mergeIntoParent + ? summaryCache[key]?.current + : summaryCache[key]?.parent} + {@const existsInBothWorkspaces = !( + (diff.exists_in_fork && !diff.exists_in_source) || + (!diff.exists_in_fork && diff.exists_in_source) + )} + {#if oldSummary != newSummary && isSelectable && existsInBothWorkspaces} + {oldSummary || diff.path} + {newSummary || diff.path} + {:else if !existsInBothWorkspaces} + {newSummary || oldSummary || diff.path} + {:else} + {newSummary || diff.path} + {/if} + {/snippet} + + {#snippet itemActions(item)} + {@const diff = item.diff as WorkspaceItemDiff} + {@const key = item.key} + {@const targetOnBehalfOf = getTargetOnBehalfOf(key)} + {@const isConflict = diff.ahead > 0 && diff.behind > 0} + {@const existsInBothWorkspaces = !( + (diff.exists_in_fork && !diff.exists_in_source) || + (!diff.exists_in_fork && diff.exists_in_source) + )} + + {#if itemNeedsOnBehalfOfSelection(key, diff.kind)} + { + onBehalfOfChoice[key] = choice + if (details) customOnBehalfOf[key] = details + }} + kind={diff.kind} + canPreserve={canPreserveOnBehalfOf} + customValue={customOnBehalfOf[key]?.permissionedAs} + /> + {/if} + {#if diff.kind === 'raw_app'} + Raw + {/if} + + {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead == 0 && diff.behind > 0} + New + {/if} + {#if !diff.exists_in_fork && diff.exists_in_source && diff.ahead > 0} + Deleted + {/if} + {#if diff.exists_in_fork && !diff.exists_in_source && diff.behind > 0} + Deleted + {/if} + {#if diff.exists_in_fork && !diff.exists_in_source && diff.ahead > 0 && diff.behind == 0} + New + {/if} + {#if !deploymentStatus[key] || deploymentStatus[key].status != 'deployed'} +
+ {#if isConflict || existsInBothWorkspaces} + {#if diff.ahead > 0} + + + {diff.ahead} ahead + + {/if} + {#if diff.behind > 0} + + + {diff.behind} behind + + {/if} + {#if isConflict} + + + Conflict + + {/if} {/if} +
+
- - {/snippet} - - {/each} +
+ {/if} + {/snippet} + + {#snippet footer()} +
+
+ +
+ {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} + {#if !(mergeIntoParent && !canDeployToParent)} + + {#if hasUnselectedOnBehalfOf} + + You must set the "on behalf of" user for all items before deploying + + The "run on behalf of" field defines which user's permissions will be + applied during execution. Make sure this is set to an appropriate user + before deploying. + + + {/if} + {/if} + {/if} + + {#if deploymentErrorMessage != ''} + + + {deploymentErrorMessage} + + + {/if} +
+
+ {/snippet} + +
+ + +
+
+

Triggers created in this fork

+ {#if !loadingTriggers} + {forkTriggers.length} trigger{forkTriggers.length !== 1 ? 's' : ''} + {/if}
- {/if} + + + When forking a workspace, triggers are not forked to avoid unnecessary executions or + collisions. If you created this triggers with the intention of deploying them to the parent + workspace, you can do so here. Otherwise it is recommended to delete them or disable them. + + + {#if loadingTriggers} +
+ + Loading triggers... +
+ {:else if forkTriggers.length === 0} +
+ No triggers in this fork workspace. +
+ {:else} +
+ {#each forkTriggers as trigger (trigger.triggerKind + ':' + trigger.path)} + + {#snippet customSummary()} + {getTriggerDisplayName(trigger.triggerKind)} + → + {trigger.scriptPath} + {#if trigger.isFlow} + flow + {/if} + {#if trigger.extraLabel} + ({trigger.extraLabel}) + {/if} + {/snippet} + {#snippet actions()} + {#if trigger.enabled != null} + + {trigger.enabled ? 'Enabled' : 'Disabled'} + + {/if} + + + {/snippet} + + {/each} +
+ {/if} +
+ +
+ +
diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 2b7e643451..46177bc494 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -1,6 +1,14 @@ @@ -96,6 +175,9 @@ bind:hasReplResult bind:selectedSchemaKey={uriState.selectedSchema} bind:selectedTableKey={uriState.selectedTable} + onImport={enableImportExport + ? (mode) => ((importDrawerOpen = true), (importBehavior = mode)) + : undefined} > {#snippet dbSelector()} {#if uriState.isDatatableInput} @@ -119,6 +201,12 @@ {/key} {/if} {#snippet actions()} + {#if enableImportExport} + + + {/if} +
+ {/if} + + + + + (importDrawerOpen = false)}> +
+ + This will import the schemas from the selected source into the current database. Existing + tables with the same names may be affected. + +
+ Source database + +
+
+ Import mode + {:else}
-
{emptyMessage}
+
{emptyMessage}
{/if} {#if footer} -
+
{@render footer()}
{/if} diff --git a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte index 351ee68331..3ed8690079 100644 --- a/frontend/src/lib/components/assets/AssetsDropdownButton.svelte +++ b/frontend/src/lib/components/assets/AssetsDropdownButton.svelte @@ -61,7 +61,9 @@ }) let datatables = resource([], () => - WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }) + WorkspaceService.listDataTables({ workspace: $workspaceStore ?? '' }).then((d) => + d.map((d) => d.name) + ) ) let ducklakes = resource([], () => WorkspaceService.listDucklakes({ workspace: $workspaceStore ?? '' }) diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index f7205b2531..9dee3b6e87 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -86,7 +86,7 @@ {#if open}
diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 86e646b753..60794c6923 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -15,7 +15,10 @@ const s: DataTableSettingsType = { dataTables: [] } if (settings?.datatables) { for (const [name, rest] of Object.entries(settings.datatables)) { - s.dataTables.push({ name, ...rest }) + s.dataTables.push({ + name, + ...rest + }) } } return s @@ -107,7 +110,7 @@ }) } - const customInstanceDbs = resource([], SettingService.listCustomInstanceDbs) + const customInstanceDbs = resource([() => $workspaceStore], SettingService.listCustomInstanceDbs) async function onSave() { try { @@ -252,27 +255,29 @@
+ - {#if dirtyMap[dataTable.name]} - - {#snippet trigger()} - - {/snippet} - {#snippet content()} - Please save settings first - {/snippet} - - {:else} - - {/if} +
+ {#if dirtyMap[dataTable.name]} + + {#snippet trigger()} + + {/snippet} + {#snippet content()} + Please save settings first + {/snippet} + + {:else} + + {/if} +
removeDataTable(dataTableIndex)} /> diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 641b56bb04..0f02b1667f 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -135,7 +135,7 @@ !Object.values(ducklakeIsDirty).every((v) => v === false) ) - const customInstanceDbs = resource([], SettingService.listCustomInstanceDbs) + const customInstanceDbs = resource([() => $workspaceStore], SettingService.listCustomInstanceDbs) async function onSave() { try { diff --git a/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte b/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte new file mode 100644 index 0000000000..87bcce5fbf --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/ForkDatatableSection.svelte @@ -0,0 +1,267 @@ + + + + +{#if allDatatables.current && allDatatables.current.length > 0} +