diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b6fbd91..c1bc3ea10f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +## [1.744.0](https://github.com/windmill-labs/windmill/compare/v1.743.0...v1.744.0) (2026-07-01) + + +### Features + +* add copy-to-clipboard button to rendered Mermaid diagrams in AI chat ([#9838](https://github.com/windmill-labs/windmill/issues/9838)) ([a27e814](https://github.com/windmill-labs/windmill/commit/a27e814a03c615259381eaf684aa90d56569b0af)) +* add dev workspaces paired with a lockable prod workspace ([#9793](https://github.com/windmill-labs/windmill/issues/9793)) ([b4b0c6a](https://github.com/windmill-labs/windmill/commit/b4b0c6a93e52152251fadefe319773faf42549b2)) +* **ansible:** support repo-provided ansible.cfg in delegate_to_git_repo ([#9851](https://github.com/windmill-labs/windmill/issues/9851)) ([68bf0da](https://github.com/windmill-labs/windmill/commit/68bf0daf5815307cda6ce23214dd5159b6aa33b4)) +* **licensing:** enforce offline license seat cap ([#9845](https://github.com/windmill-labs/windmill/issues/9845)) ([83f3d7f](https://github.com/windmill-labs/windmill/commit/83f3d7f910b331c09f60cc9ff556728afa3dec07)) +* **object-store:** make GCS service account key optional for Workload Identity ([#9842](https://github.com/windmill-labs/windmill/issues/9842)) ([83ed011](https://github.com/windmill-labs/windmill/commit/83ed011e264f20ffa66a7bf933f2fe3615cf6b67)) +* **pipeline:** local development for data pipelines (CLI --local + pipeline dev preview) ([#9840](https://github.com/windmill-labs/windmill/issues/9840)) ([74f579e](https://github.com/windmill-labs/windmill/commit/74f579e6d9ef08e74460f904a4c22ed9d6a3b5b0)) +* **pipelines:** add managed SCD2 history materialize strategy ([#9850](https://github.com/windmill-labs/windmill/issues/9850)) ([5a66127](https://github.com/windmill-labs/windmill/commit/5a661279a3690e2393b9b16996f5d1a5a509259c)) + + +### Bug Fixes + +* **ai-chat:** replay anthropic turns verbatim to keep thinking valid ([#9843](https://github.com/windmill-labs/windmill/issues/9843)) ([a37a144](https://github.com/windmill-labs/windmill/commit/a37a144e81cf6b3de935688a617e9d0e1756004a)) +* grant dispatch_event table to windmill roles ([#9852](https://github.com/windmill-labs/windmill/issues/9852)) ([f05b50d](https://github.com/windmill-labs/windmill/commit/f05b50d29ac2fdbb808a97057fb92c8e425b4a2f)) +* grant workspace_diff, materialized_partition, debounce_stale_data to windmill roles ([#9853](https://github.com/windmill-labs/windmill/issues/9853)) ([293647d](https://github.com/windmill-labs/windmill/commit/293647de4c13cb8468cbd81ff1924cba90e164b4)) +* honor verify-ca/verify-full sslmode for postgres connections ([#9835](https://github.com/windmill-labs/windmill/issues/9835)) ([bf6be96](https://github.com/windmill-labs/windmill/commit/bf6be967fa8c74e1299cf63f813c1cfa34b97f3e)) +* **mcp:** stop double-escaping string query params in build_query_string ([#9855](https://github.com/windmill-labs/windmill/issues/9855)) ([1c46f89](https://github.com/windmill-labs/windmill/commit/1c46f899ca03edf62053f4f14d65b4eabff4255d)) +* **s3_proxy:** preserve URL-encoding on Hive-partition proxy writes ([#9848](https://github.com/windmill-labs/windmill/issues/9848)) ([6b79bdd](https://github.com/windmill-labs/windmill/commit/6b79bddd42fe55f891c17cb71a7e36ee31337bac)) +* validate workspace name length (max 50 chars) on create and fork ([#9854](https://github.com/windmill-labs/windmill/issues/9854)) ([b52972d](https://github.com/windmill-labs/windmill/commit/b52972d0de89004e98d18241d238ca028e4eecba)) + +## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29) + + +### Features + +* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005)) +* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0)) + + +### Bug Fixes + +* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b)) + ## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 31b097867b..66e5fd3cb7 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1223,6 +1223,66 @@ - places it in team_a (writable by this non-admin user) and not team_b (read-only) - leaves the result as a draft only +- id: global-test-pipeline-create-node + prompt: |- + Set up the first step of a data pipeline at `f/evals/global/orders_ingest`. + On a schedule, it should pull raw orders and land them in a managed DuckLake + table so later steps can build on it. Keep it as an AI draft only — don't + deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/orders_ingest + valueIncludes: + - pipeline + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - builds a data pipeline node as a script (not a flow) + - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`) + - declares a schedule trigger and writes its output to a managed DuckLake table + - leaves the result as an AI draft and does not deploy or save it + +- id: global-test-pipeline-two-node-chain + prompt: |- + Build a small data pipeline in the `f/evals/global` folder: one step that + ingests orders into a DuckLake table, and a second step that reads that table + and writes a daily order-count rollup table. Wire the second step to run off + the first step's output. Keep everything as drafts — don't deploy. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 14 + validate: + draftCountAtLeast: 2 + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates two data pipeline nodes as scripts (not a flow) in f/evals/global + - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) + - the first ingests orders into a DuckLake table + - the second reads that same table and writes a daily rollup, wired to the first step's output asset + - leaves both as AI drafts without deploying + - id: global-path5-create-folder-then-draft prompt: |- Create a new shared folder called "analytics" for our data work, then draft a diff --git a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json b/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json deleted file mode 100644 index cde17acf76..0000000000 --- a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941" -} diff --git a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json b/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json deleted file mode 100644 index 5ca75fe782..0000000000 --- a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM asset\n WHERE (workspace_id, path, kind) IN (\n SELECT workspace_id, path, kind FROM (\n SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[], \n $2::varchar[], \n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id \n AND a.path = limits.path \n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "VarcharArray", - "VarcharArray", - { - "Custom": { - "name": "asset_kind[]", - "kind": { - "Array": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - } - } - }, - "Int4Array" - ] - }, - "nullable": [] - }, - "hash": "02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0" -} diff --git a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json similarity index 51% rename from backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json rename to backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json index 6fd7d38f69..036e8e4b57 100644 --- a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json +++ b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2", + "query": "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e" + "hash": "0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b" } diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json b/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json deleted file mode 100644 index 911d6c3b07..0000000000 --- a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "x", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346" -} diff --git a/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json new file mode 100644 index 0000000000..f548727abb --- /dev/null +++ b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a" +} diff --git a/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json new file mode 100644 index 0000000000..058018a8ee --- /dev/null +++ b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\", is_dev_workspace\n FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531" +} diff --git a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json b/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json deleted file mode 100644 index 75957a4f2f..0000000000 --- a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - { - "Custom": { - "name": "asset_usage_kind", - "kind": { - "Enum": [ - "script", - "flow", - "job" - ] - } - } - } - ] - }, - "nullable": [] - }, - "hash": "1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6" -} diff --git a/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json new file mode 100644 index 0000000000..b84cd8a14e --- /dev/null +++ b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, name)\n DO UPDATE SET rules = EXCLUDED.rules,\n bypass_groups = EXCLUDED.bypass_groups,\n bypass_users = EXCLUDED.bypass_users\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int4", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index a53d131a3f..f9bc97d29d 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -38,7 +38,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } @@ -75,7 +76,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json b/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json deleted file mode 100644 index 642723decf..0000000000 --- a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) as \"c!\" FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "c!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4" -} diff --git a/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json new file mode 100644 index 0000000000..3ad3666413 --- /dev/null +++ b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5\n FROM workspace WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee" +} diff --git a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json similarity index 71% rename from backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json rename to backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json index 50ed7549e8..6163a0f9fb 100644 --- a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json +++ b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM ws_specific ws\n WHERE ws.path = workspace_diff.path\n AND ws.item_kind = workspace_diff.kind\n AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id)\n )", "describe": { "columns": [ { @@ -55,5 +55,5 @@ true ] }, - "hash": "0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1" + "hash": "2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf" } diff --git a/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json new file mode 100644 index 0000000000..71615c6466 --- /dev/null +++ b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'resource', $2::varchar\n WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1" +} diff --git a/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json new file mode 100644 index 0000000000..1ca539c46d --- /dev/null +++ b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"is_fork!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fork!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd" +} diff --git a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json b/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json deleted file mode 100644 index 3470285737..0000000000 --- a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ORDER BY path, created_at DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "content!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db" -} diff --git a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json b/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json deleted file mode 100644 index 31c0ab4982..0000000000 --- a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca" -} diff --git a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json similarity index 57% rename from backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json rename to backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json index 4fe298370b..c12e0cdac4 100644 --- a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json +++ b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id)\n VALUES ($1, $2, $3, $4)", + "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id, is_dev_workspace)\n VALUES ($1, $2, $3, $4, $5)", "describe": { "columns": [], "parameters": { @@ -8,10 +8,11 @@ "Varchar", "Varchar", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c" + "hash": "41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056" } diff --git a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json similarity index 76% rename from backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json rename to backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json index 0265d7d2b5..76eab1d117 100644 --- a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json +++ b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ", + "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ON CONFLICT DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078" + "hash": "42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7" } diff --git a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json b/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json deleted file mode 100644 index 9e7ffed082..0000000000 --- a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings)\n VALUES ($1, $2, $3)\n ON CONFLICT (hash)\n DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a" -} diff --git a/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json new file mode 100644 index 0000000000..5f114c0156 --- /dev/null +++ b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH potential AS (\n SELECT email, operator FROM usr WHERE is_service_account IS false\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user\n WHERE email NOT IN (SELECT email FROM password WHERE disabled IS true)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44" +} diff --git a/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json new file mode 100644 index 0000000000..8e9e6a8713 --- /dev/null +++ b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a" +} diff --git a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json b/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json deleted file mode 100644 index d165e78ad2..0000000000 --- a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f" -} diff --git a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json similarity index 57% rename from backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json rename to backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json index fce125c6d5..35b70a239c 100644 --- a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json +++ b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", + "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1 AND is_service_account IS false\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9" + "hash": "5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72" } diff --git a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json b/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json deleted file mode 100644 index f634fc2d4b..0000000000 --- a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH mine AS (\n SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1\n ), claim_self AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE id = $1 AND consumed_at IS NULL\n RETURNING debounce_batch\n ), claim_rest AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)\n AND id <> $1 AND consumed_at IS NULL\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM mine) AS \"had_row!\",\n (SELECT debounce_batch FROM claim_self) AS claimed_batch,\n (SELECT consumed_by FROM mine) AS prev_consumed_by,\n ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "had_row!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "claimed_batch", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "prev_consumed_by", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "claimed_ids!", - "type_info": "UuidArray" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89" -} diff --git a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json similarity index 63% rename from backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json rename to backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json index bb80e8d19a..a9cdc35dce 100644 --- a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json +++ b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -30,11 +30,16 @@ }, { "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 6, + "ordinal": 7, "name": "disabled", "type_info": "Bool" } @@ -50,9 +55,10 @@ false, true, true, + false, null, false ] }, - "hash": "c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6" + "hash": "63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19" } diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json deleted file mode 100644 index cb20ec2ffb..0000000000 --- a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" -} diff --git a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json b/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json deleted file mode 100644 index e885c72039..0000000000 --- a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18" -} diff --git a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json deleted file mode 100644 index c5266c66b5..0000000000 --- a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_path!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "kind!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc" -} diff --git a/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json new file mode 100644 index 0000000000..f425d1eebd --- /dev/null +++ b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c" +} diff --git a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json b/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json deleted file mode 100644 index f7d3361d99..0000000000 --- a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523" -} diff --git a/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json new file mode 100644 index 0000000000..e1cf9bc17f --- /dev/null +++ b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376" +} diff --git a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json b/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json deleted file mode 100644 index 7a0c5578b0..0000000000 --- a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', '1970-01-01T00:00:00+00:00')\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23" -} diff --git a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json b/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json deleted file mode 100644 index c0573a8fab..0000000000 --- a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace SELECT $1, $2, owner, false, premium FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745" -} diff --git a/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json new file mode 100644 index 0000000000..673becc2ff --- /dev/null +++ b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "deleted", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38" +} diff --git a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json deleted file mode 100644 index 96587ed1d0..0000000000 --- a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_settings_handle", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2" -} diff --git a/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json new file mode 100644 index 0000000000..0859d3e071 --- /dev/null +++ b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec" +} diff --git a/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json new file mode 100644 index 0000000000..3f159707b6 --- /dev/null +++ b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'variable', $2::varchar\n WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c" +} diff --git a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json b/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json deleted file mode 100644 index 569a5122ba..0000000000 --- a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\"\n FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b" -} diff --git a/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json new file mode 100644 index 0000000000..2b9b0619a1 --- /dev/null +++ b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0" +} diff --git a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json b/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json deleted file mode 100644 index 819928ddc1..0000000000 --- a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END\n FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d" -} diff --git a/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json new file mode 100644 index 0000000000..5d20f83f9d --- /dev/null +++ b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077" +} diff --git a/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json new file mode 100644 index 0000000000..b022524bbb --- /dev/null +++ b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e" +} diff --git a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json deleted file mode 100644 index d12d305b92..0000000000 --- a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b" -} diff --git a/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json new file mode 100644 index 0000000000..94cf77ebc5 --- /dev/null +++ b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via\n FROM usr WHERE workspace_id = $2\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142" +} diff --git a/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json new file mode 100644 index 0000000000..35d3ace40e --- /dev/null +++ b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_diff SET has_changes = NULL\n WHERE path = $2 AND kind = $3\n AND ($1 IN (source_workspace_id, fork_workspace_id))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db" +} diff --git a/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json new file mode 100644 index 0000000000..a1f7783645 --- /dev/null +++ b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM workspace\n WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de" +} diff --git a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json b/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json deleted file mode 100644 index 7361b645b5..0000000000 --- a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch, consumed_at) VALUES\n ($1, nextval('debounce_batch_seq'), now() - interval '20 minutes'),\n ($2, nextval('debounce_batch_seq'), now() - interval '1 minute'),\n ($3, nextval('debounce_batch_seq'), NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed" -} diff --git a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json b/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json deleted file mode 100644 index e900b0f9e0..0000000000 --- a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>$2 FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266" -} diff --git a/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json new file mode 100644 index 0000000000..9f8b86b4e6 --- /dev/null +++ b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2\n AND NOT EXISTS (\n SELECT 1 FROM workspace\n WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5" +} diff --git a/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json new file mode 100644 index 0000000000..4a511c2d6e --- /dev/null +++ b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996" +} diff --git a/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json new file mode 100644 index 0000000000..48a5c693ac --- /dev/null +++ b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8" +} diff --git a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json deleted file mode 100644 index 2ba317edc6..0000000000 --- a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "kind!: AssetKind", - "type_info": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b" -} diff --git a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json b/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json deleted file mode 100644 index 67ce9d8719..0000000000 --- a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Varchar", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef" -} diff --git a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json b/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json deleted file mode 100644 index f38c023cb3..0000000000 --- a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH potential AS (\n SELECT email, operator FROM usr\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "operators!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455" -} diff --git a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json b/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json deleted file mode 100644 index a1b52e81fd..0000000000 --- a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\",\n result AS \"result: Json>\",\n started_at AS \"started_at!\"FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "started_at!", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null, - true, - true - ] - }, - "hash": "e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976" -} diff --git a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json b/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json deleted file mode 100644 index 146c7f05b3..0000000000 --- a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT concurrency_settings, debouncing_settings FROM runnable_settings WHERE hash = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "concurrency_settings", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "debouncing_settings", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01" -} diff --git a/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json new file mode 100644 index 0000000000..844dfb3e47 --- /dev/null +++ b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c" +} diff --git a/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json new file mode 100644 index 0000000000..7f967b60a5 --- /dev/null +++ b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7209b6be37..63081912a3 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -775,9 +775,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -785,14 +785,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -2601,9 +2602,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.89+curl-8.20.0" +version = "0.4.90+curl-8.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" +checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" dependencies = [ "cc", "libc", @@ -6085,9 +6086,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -9998,9 +9999,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -12188,9 +12189,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -12208,9 +12209,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -12922,9 +12923,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -13734,7 +13735,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-nats", @@ -13817,7 +13818,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.742.0" +version = "1.744.0" dependencies = [ "async-stream", "async-trait", @@ -13850,7 +13851,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13863,7 +13864,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "argon2", @@ -14001,7 +14002,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14024,7 +14025,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14039,7 +14040,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14065,7 +14066,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.742.0" +version = "1.744.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14075,7 +14076,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14092,7 +14093,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14114,7 +14115,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14137,7 +14138,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14153,7 +14154,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14174,7 +14175,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14195,7 +14196,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14209,7 +14210,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-nats", @@ -14244,7 +14245,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14269,7 +14270,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14287,7 +14288,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14309,7 +14310,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14329,7 +14330,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14366,7 +14367,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14394,7 +14395,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.742.0" +version = "1.744.0" dependencies = [ "lazy_static", "serde", @@ -14406,7 +14407,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.742.0" +version = "1.744.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14431,7 +14432,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14445,7 +14446,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.742.0" +version = "1.744.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14478,7 +14479,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.742.0" +version = "1.744.0" dependencies = [ "chrono", "lazy_static", @@ -14492,7 +14493,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14511,7 +14512,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.744.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14613,7 +14614,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.742.0" +version = "1.744.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14632,7 +14633,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.742.0" +version = "1.744.0" dependencies = [ "regex", "serde", @@ -14647,7 +14648,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14671,7 +14672,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "futures", @@ -14688,7 +14689,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.744.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14704,7 +14705,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -14725,7 +14726,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -14756,7 +14757,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "arc-swap", @@ -14781,7 +14782,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-stream", @@ -14815,7 +14816,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "futures", @@ -14833,7 +14834,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.744.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14842,7 +14843,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -14854,7 +14855,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -14866,7 +14867,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "gosyn", @@ -14878,7 +14879,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -14890,7 +14891,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -14902,7 +14903,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "nu-parser", @@ -14913,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14924,7 +14925,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14936,7 +14937,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14947,7 +14948,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-recursion", @@ -14969,7 +14970,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -14981,7 +14982,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -14995,7 +14996,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15012,7 +15013,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -15025,7 +15026,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde", @@ -15037,7 +15038,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -15055,7 +15056,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15071,7 +15072,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15087,7 +15088,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde", @@ -15098,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-recursion", @@ -15137,7 +15138,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "const_format", @@ -15176,7 +15177,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.742.0" +version = "1.744.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15187,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-recursion", @@ -15221,7 +15222,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15245,7 +15246,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15278,7 +15279,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15311,7 +15312,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15331,7 +15332,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15365,7 +15366,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15401,7 +15402,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15424,7 +15425,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15448,7 +15449,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-nats", @@ -15472,7 +15473,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15507,7 +15508,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15535,7 +15536,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-trait", @@ -15560,7 +15561,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15579,7 +15580,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-once-cell", @@ -15689,7 +15690,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.742.0" +version = "1.744.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 827bd5d494..7e7b127da2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.742.0" +version = "1.744.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.742.0" +version = "1.744.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/migrations/20260624161218_dev_workspace.down.sql b/backend/migrations/20260624161218_dev_workspace.down.sql new file mode 100644 index 0000000000..45b8e65f67 --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE workspace DROP CONSTRAINT IF EXISTS workspace_dev_requires_parent; +DROP INDEX IF EXISTS workspace_canonical_dev_idx; +ALTER TABLE workspace DROP COLUMN is_dev_workspace; diff --git a/backend/migrations/20260624161218_dev_workspace.up.sql b/backend/migrations/20260624161218_dev_workspace.up.sql new file mode 100644 index 0000000000..a0a252eb7c --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.up.sql @@ -0,0 +1,14 @@ +-- A dev workspace is a fork (parent_workspace_id set) that is the standing editable +-- environment paired with its parent ("prod"), as opposed to a throwaway fork. +ALTER TABLE workspace ADD COLUMN is_dev_workspace BOOLEAN NOT NULL DEFAULT false; + +-- At most one active canonical dev workspace per parent (one editable source per prod). +-- Excludes soft-deleted (archived) workspaces so a new dev can replace an archived one. +CREATE UNIQUE INDEX workspace_canonical_dev_idx ON workspace (parent_workspace_id) + WHERE is_dev_workspace AND deleted = false; + +-- A dev workspace is a fork, so it must have a parent. Enforce the invariant at the schema level so +-- no path (or manual write) can persist a "root dev workspace". No backfill needed: the column is +-- added above with default false, so no existing row can violate this at creation time. +ALTER TABLE workspace ADD CONSTRAINT workspace_dev_requires_parent + CHECK (NOT is_dev_workspace OR parent_workspace_id IS NOT NULL); diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql new file mode 100644 index 0000000000..2278fbfb2e --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql @@ -0,0 +1,4 @@ +REVOKE ALL ON dispatch_event FROM windmill_user; +REVOKE ALL ON dispatch_event FROM windmill_admin; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql new file mode 100644 index 0000000000..5d0b4650ce --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql @@ -0,0 +1,15 @@ +-- The dispatch_event table (migration 20260523055641_dispatch_event) was +-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user +-- and windmill_admin. Those default privileges only apply to objects created by +-- the role that set them (migration 20250205131523), so deployments whose +-- migration runner is a different role leave dispatch_event ungranted. Direct +-- writes run as the invoking role -- the dispatcher insert (asset_dispatch.rs) +-- and the DELETE in delete_jobs (windmill-common/src/jobs.rs), reached whenever +-- a job's side rows are reaped, e.g. on schedule disable -- and fail with +-- "permission denied for table dispatch_event". Grant explicitly to guarantee +-- access regardless of who ran the migrations (same fix as notify_event in +-- 20260619091631 and script_trigger in 20260619112847). +GRANT ALL ON dispatch_event TO windmill_user; +GRANT ALL ON dispatch_event TO windmill_admin; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql new file mode 100644 index 0000000000..b73484e053 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql @@ -0,0 +1,6 @@ +REVOKE ALL ON workspace_diff FROM windmill_user; +REVOKE ALL ON workspace_diff FROM windmill_admin; +REVOKE ALL ON materialized_partition FROM windmill_user; +REVOKE ALL ON materialized_partition FROM windmill_admin; +REVOKE ALL ON debounce_stale_data FROM windmill_user; +REVOKE ALL ON debounce_stale_data FROM windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql new file mode 100644 index 0000000000..9205934600 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql @@ -0,0 +1,22 @@ +-- Same grant gap fixed for notify_event (20260619091631), script_trigger +-- (20260619112847), and dispatch_event: tables created after the one-time +-- GRANT ALL in 20250205131523 rely on ALTER DEFAULT PRIVILEGES, which only +-- applies to objects created by the role that set them. On deployments whose +-- migration runner is a different role, these tables end up ungranted, and +-- writes that run under the RLS role (a transaction opened via +-- user_db.begin(&authed) -> SET LOCAL ROLE windmill_user/windmill_admin) fail +-- with "permission denied for table ". +-- +-- Each table below has a confirmed write on a user_db transaction: +-- * workspace_diff -- UPDATE in set_ws_specific (workspaces.rs) +-- * materialized_partition -- INSERT via record_materialization (assets API) +-- * debounce_stale_data -- DELETE in resume_suspended_trigger_jobs +-- (global_handler.rs), the same tx that reaps a +-- job's side rows +-- None has a sequence, so only table grants are needed. +GRANT ALL ON workspace_diff TO windmill_user; +GRANT ALL ON workspace_diff TO windmill_admin; +GRANT ALL ON materialized_partition TO windmill_user; +GRANT ALL ON materialized_partition TO windmill_admin; +GRANT ALL ON debounce_stale_data TO windmill_user; +GRANT ALL ON debounce_stale_data TO windmill_admin; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 096d93163b..8e059ddf5a 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.744.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.744.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.744.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.744.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2afd2483b..8b88c201bd 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.742.0" +version = "1.744.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 5d9945cadd..f2a1a2b824 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -269,6 +269,12 @@ pub struct DelegateToGitRepoDetails { pub commit: Option, pub inventories_location: Option, pub vars_location: Option, + /// Path (relative to the cloned repo root) of an `ansible.cfg` to use as the + /// effective config for the run. When set, Windmill points `ANSIBLE_CONFIG` at + /// it so the repo's own settings (roles paths, inventory plugins, callbacks…) + /// apply, and only injects the settings that depend on runtime state it alone + /// controls (temp/home dirs, vault password) on top. + pub ansible_cfg: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub install_requirements: bool, } @@ -629,6 +635,10 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option Option, } -// `// materialize [manual] [append] [key=]` — declares that this -// script produces a *managed* materialization of `` (a `ducklake://` -// table). By default the runtime generates the write DDL around the script's -// single trailing `SELECT` and owns idempotency, partition-state and snapshot -// capture. `manual` is the escape hatch: the script writes its own DDL and the -// runtime only records state (track-only). The reconciliation strategy options -// (`append`, `key=`) apply to managed mode: none → DELETE-by-partition + -// INSERT (replace); `key=` → MERGE (dedup within slice); `append` → +// `// materialize [manual] [append] [key=] [history] [track=]` +// — declares that this script produces a *managed* materialization of `` +// (a `ducklake://` table). By default the runtime generates the write DDL around +// the script's single trailing `SELECT` and owns idempotency, partition-state +// and snapshot capture. `manual` is the escape hatch: the script writes its own +// DDL and the runtime only records state (track-only). The reconciliation +// strategy options apply to managed mode: none → DELETE-by-partition + INSERT +// (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). +// `key= history` upgrades the merge to SCD type 2: the SELECT is the current +// snapshot (one row per key), and a change to any tracked column (`track=`, +// default all non-key) closes the prior version and opens a new one, keeping full +// history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a +// recognized alias for `history`. `deletes=close` (scd2 only) also closes a key +// that disappears from the snapshot; default leaves absent keys current. #[derive(Serialize, Debug, PartialEq, Clone)] pub struct MaterializeSpec { pub target_kind: AssetKind, @@ -243,6 +249,20 @@ pub struct MaterializeSpec { pub append: bool, #[serde(skip_serializing_if = "Option::is_none", default)] pub unique_key: Option, + // `scd2` managed history mode: the SELECT is the current snapshot (one row + // per `unique_key`), and the runtime maintains a Slowly-Changing-Dimension + // type-2 history (`valid_from`/`valid_to`/`is_current`). `unique_key` (the + // `key=` opt) is the natural key; `track` lists the columns whose change + // opens a new version (empty ⇒ all non-key columns). Managed mode only. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub scd2: bool, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub track: Vec, + // scd2 only: `deletes=close` opts into hard-delete-close — a key that + // disappears from the snapshot has its current version closed (dbt's + // `hard_deletes=close`). Default (false) leaves absent keys current. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub close_deleted: bool, } // `// data_test …` — a data-quality assertion run against the @@ -800,19 +820,31 @@ fn parse_retry_spec(s: &str) -> Option { Some(RetrySpec { count, delay }) } -// Parse a `// materialize [manual] [append] [key=]` right-hand -// side. An optional leading `manual` token (whitespace-delimited) opts out of -// managed mode (track-only). The next whitespace token is the target asset URI +// Parse a `// materialize [manual] [append] [key=] [history] +// [track=]` right-hand side. An optional leading `manual` token opts out +// of managed mode (track-only); a leading `scd2` token is an alias for the +// `history` flag. The next whitespace token is the target asset URI // (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the -// remainder are strategy options — bare `append` and `key=` (merge key), -// which apply to managed mode only. A missing/empty target yields `None` (the -// annotation is dropped, fail-safe). +// remainder are strategy options — bare `append`, bare `history` (SCD type-2 on +// a keyed merge), `key=` (merge/scd2 key), `track=` (scd2 tracked +// columns), and `deletes=close` (scd2 hard-delete-close) — which apply to managed +// mode only. A missing/empty target yields `None` (the annotation is dropped, +// fail-safe). fn parse_materialize_spec(s: &str) -> Option { - let (manual, rest) = match s.strip_prefix("manual") { - Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => { - (true, after.trim_start()) - } - _ => (false, s), + // One optional leading mode keyword: `manual` (escape hatch, track-only) or + // `scd2` (alias for the `history` flag below). A missing keyword is the + // default managed mode. + fn strip_mode<'a>(s: &'a str, kw: &str) -> Option<&'a str> { + s.strip_prefix(kw) + .filter(|after| after.is_empty() || after.starts_with(char::is_whitespace)) + .map(|after| after.trim_start()) + } + let (manual, scd2_kw, rest) = if let Some(after) = strip_mode(s, "manual") { + (true, false, after) + } else if let Some(after) = strip_mode(s, "scd2") { + (false, true, after) + } else { + (false, false, s) }; let mut it = rest.trim().splitn(2, char::is_whitespace); let asset_tok = it.next()?; @@ -822,11 +854,38 @@ fn parse_materialize_spec(s: &str) -> Option { return None; } let append = opts_str.split_whitespace().any(|t| t == "append"); - let unique_key = parse_kv_opts(opts_str) - .get("key") - .filter(|k| !k.is_empty()) - .cloned(); - Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key }) + // SCD type-2 history mode. The primary spelling is the bare `history` flag on + // a keyed merge (`key= history`) — it reads as "a keyed upsert that keeps + // history"; the leading `scd2` keyword is a recognized alias for the same. + let scd2 = scd2_kw || opts_str.split_whitespace().any(|t| t == "history"); + let opts = parse_kv_opts(opts_str); + let unique_key = opts.get("key").filter(|k| !k.is_empty()).cloned(); + // `track=` (scd2 only): comma-separated columns whose change opens a + // new version. Empty entries dropped; an empty list ⇒ track all non-key cols. + // Like every `=`-option here the value is whitespace-terminated, so the list + // must contain no spaces (`track=a,b`, not `track=a, b` — the rest is dropped). + let track = opts + .get("track") + .map(|v| { + v.split(',') + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty()) + .collect::>() + }) + .unwrap_or_default(); + // `deletes=close` (scd2 only) opts into hard-delete-close; any other value + // (or absence) keeps the soft-delete default. + let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false); + Some(MaterializeSpec { + target_kind, + target_path: path.to_string(), + manual, + append, + unique_key, + scd2, + track, + close_deleted, + }) } // Parse a `// data_test …` right-hand side into one `DataTest`. The @@ -1457,6 +1516,53 @@ mod pipeline_annotation_tests { assert_eq!(m.unique_key, None); } + #[test] + fn materialize_scd2_history_flag_with_key_and_track() { + // Primary spelling: `key= history` on a merge. + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history track=name,tier", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(!m.manual); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert_eq!(m.track, vec!["name".to_string(), "tier".to_string()]); + } + + #[test] + fn materialize_scd2_keyword_is_alias_for_history() { + let out = parse_pipeline_annotations("// materialize scd2 ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert!(m.track.is_empty()); + // soft-delete default + assert!(!m.close_deleted); + } + + #[test] + fn materialize_scd2_deletes_close_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=close", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(m.close_deleted); + // any other value keeps the soft-delete default + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=ignore", + ); + assert!(!out.materialize.expect("materialize").close_deleted); + } + + #[test] + fn materialize_key_without_history_is_plain_merge() { + let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(!m.scd2, "no history flag ⇒ SCD1 merge, not scd2"); + assert_eq!(m.unique_key.as_deref(), Some("id")); + } + #[test] fn materialize_default_syntax_shorthand() { let out = parse_pipeline_annotations("// materialize ducklake"); diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index 7a3c72bdce..2b1523b1ff 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -327,20 +327,45 @@ fn snippet(stmt: &str) -> String { // --------------------------------------------------------------------------- /// How a (partition of a) materialized table is reconciled on each run. -/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else -/// `unique_key` → `Merge`, else `Replace`. +/// Derived at deploy from the annotation: `key= history` (or the `scd2` +/// alias) → `Scd2`, else `append` → `Append`, else `unique_key` → `Merge`, else +/// `Replace`. #[derive(Debug, Clone, PartialEq, Eq)] pub enum MaterializeStrategy { /// DELETE the current partition, then INSERT — partition becomes exactly /// what the SELECT returned. Full-refresh of the slice. Replace, /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows - /// absent from the SELECT are left in place. + /// absent from the SELECT are left in place. This is SCD type 1: a changed + /// row overwrites the prior value, keeping no history. Merge { unique_key: String }, /// INSERT only — immutable event-log semantics. Append, + /// Slowly Changing Dimension type 2: the SELECT is the *current* snapshot + /// (one row per `key`); a change to any tracked column closes the prior + /// version (`valid_to`/`is_current=false`) and opens a new one, so the full + /// history is preserved. `track` empty ⇒ every non-key column is tracked. + /// `close_deleted` (opt-in `deletes=close`) also closes the current version + /// of a key that disappears from the snapshot (dbt's `hard_deletes=close`); + /// default (false) leaves absent keys current (soft delete). + /// Unpartitioned only (the worker rejects `// partitioned` + scd2). + Scd2 { key: String, track: Vec, close_deleted: bool }, } +/// SCD2 metadata columns appended to the managed history table. Fixed names so +/// the generated diff/close/open SQL and any `// data_test` on them agree. +const SCD2_VALID_FROM: &str = "valid_from"; +const SCD2_VALID_TO: &str = "valid_to"; +const SCD2_IS_CURRENT: &str = "is_current"; +/// Connection-local temp table holding the keys whose version must be rotated +/// this run (changed + new). Captured before the write so the close and the +/// open see the same set. `_wm_` prefix so it never collides with user tables. +const SCD2_CHANGED_KEYS: &str = "_wm_scd2_changed"; +/// Connection-local temp table holding the keys that disappeared from the +/// snapshot this run (present-and-current in the table, absent from the SELECT). +/// Only used when `close_deleted` (`deletes=close`) is set. +const SCD2_DELETED_KEYS: &str = "_wm_scd2_deleted"; + /// Inputs to materialization codegen, all resolved at run time by the worker. /// Pure: produces SQL text; executes nothing. #[derive(Debug, Clone)] @@ -377,6 +402,13 @@ impl<'a> MaterializeCodegen<'a> { let pval = self.partition_value_sql; let mut out = Vec::new(); + // SCD2 has a shape unlike the DELETE/INSERT strategies (diff → close old + // → open new) and does not support partitioning (rejected at the worker), + // so it is generated up front by its own helper. + if let MaterializeStrategy::Scd2 { key, track, close_deleted } = &self.strategy { + return self.scd2_statements(key, track, *close_deleted); + } + // Whole-table replace: rebuild the table to match the SELECT's *current* // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake // still snapshots). This is the only path that survives a changed SELECT @@ -441,10 +473,160 @@ impl<'a> MaterializeCodegen<'a> { )); out.push(format!("INSERT INTO {t} {source};")); } + // Handled by the early return above (scd2 has no partitioned form). + MaterializeStrategy::Scd2 { .. } => unreachable!("scd2 handled before this match"), } out.push("COMMIT;".to_string()); out } + + /// SCD2 codegen: the incoming SELECT is the *current desired snapshot* (one + /// row per `key`); we diff it against the live current rows, close the prior + /// version of every changed/new key, and open a fresh one — so history is + /// kept. `track` empty ⇒ every non-key column is tracked for change + /// detection. + /// + /// Shape (all one transaction for the mutation, mirroring the other + /// strategies so a partial failure leaves the prior snapshot intact): + /// 1. bootstrap the table (business columns + `valid_from/valid_to/ + /// is_current`), idempotent; + /// 2. capture changed+new keys into a connection-local temp table *before* + /// the write — the close below flips `is_current`, so recomputing the + /// diff after it would see a different set; + /// 3. close the prior open version of those keys (`UPDATE` — not `MERGE`: + /// DuckLake's MERGE is the unreliable path, plain UPDATE works); + /// 3b. when `close_deleted`, also capture the keys that vanished from the + /// snapshot and close their current version (no reopen) — dbt's + /// `hard_deletes=close`; + /// 4. open a new current version from the snapshot; + /// 5. create the `_current` convenience view once (`IF NOT EXISTS`), + /// inside the same transaction so it doesn't advance the DuckLake snapshot + /// past the data write the summary records (and so an unchanged rerun, + /// whose UPDATE/INSERT touch no rows, stays a true no-op). + /// + /// Close/open match keys with `IS NOT DISTINCT FROM` (via a correlated + /// `EXISTS`), not `key IN (…)`: SQL `IN` never matches `NULL`, so a `NULL` + /// natural key would be flagged as changed yet silently skipped by both the + /// close and the open, dropping the row. Null-safe matching materializes it + /// instead (a `NULL` key is still ill-formed for a dimension — guard it with + /// `// data_test not_null ` — but it must not vanish). + /// + /// Without `close_deleted`, keys present in the table but absent from the + /// SELECT are left current (soft delete — dbt's `hard_deletes=ignore` default; + /// with `close_deleted` they are closed instead — see step 3b). The effective + /// timestamp is `now()`, which DuckDB fixes to + /// the transaction start, so `valid_from`/`valid_to` are consistent within a + /// run without a nondeterministic per-statement clock. + /// + /// Reserved columns: `valid_from`/`valid_to`/`is_current` are appended to the + /// user's SELECT with these fixed names (kept clean so consumers write + /// `WHERE is_current` / `ASOF JOIN … >= valid_from`). A SELECT that already + /// projects one of them is a v1 constraint violation — the bootstrap then + /// produces a duplicate output column and the run fails at execution + /// (documented; not statically checkable here since the SELECT's columns + /// aren't known at codegen time). + fn scd2_statements(&self, key: &str, track: &[String], close_deleted: bool) -> Vec { + let t = self.target_qualified; + let sel = self.select_sql; + let k = quote_ident(key); + let vf = SCD2_VALID_FROM; + let vt = SCD2_VALID_TO; + let ic = SCD2_IS_CURRENT; + let changed = SCD2_CHANGED_KEYS; + let deleted = SCD2_DELETED_KEYS; + // Transaction-stable effective timestamp (see doc above). Cast to plain + // TIMESTAMP so it matches the bootstrapped column type (now() is TZ-aware). + let ts = "CAST(now() AS TIMESTAMP)"; + + // Projection compared to detect change. Empty `track` ⇒ all business + // columns via `* EXCLUDE ()` on the table side (which carries + // the extra metadata columns) and `*` on the snapshot side. An explicit + // `track` ⇒ key + those columns on both sides. `EXCEPT` treats NULLs as + // equal, so an unchanged NULL is not read as a change. + let (src_proj, tgt_proj) = if track.is_empty() { + ( + format!("SELECT * FROM ({sel})"), + format!("SELECT * EXCLUDE ({vf}, {vt}, {ic}) FROM {t} WHERE {ic}"), + ) + } else { + let cols = std::iter::once(key) + .chain(track.iter().map(String::as_str)) + .map(quote_ident) + .collect::>() + .join(", "); + ( + format!("SELECT {cols} FROM ({sel})"), + format!("SELECT {cols} FROM {t} WHERE {ic}"), + ) + }; + + let mut out = vec![ + format!( + "CREATE TABLE IF NOT EXISTS {t} AS SELECT *, \ + CAST(NULL AS TIMESTAMP) AS {vf}, \ + CAST(NULL AS TIMESTAMP) AS {vt}, \ + CAST(NULL AS BOOLEAN) AS {ic} FROM ({sel}) WHERE false;" + ), + format!( + "CREATE OR REPLACE TEMP TABLE {changed} AS \ + SELECT {k} FROM ({src_proj} EXCEPT {tgt_proj});" + ), + ]; + // Hard-delete-close (`deletes=close`): the keys that vanished from the + // snapshot — present-and-current in the table, absent from the SELECT. + // Captured before the transaction (like `changed`) and disjoint from it (a + // key is either in the snapshot or not), so the two closes never overlap. + if close_deleted { + out.push(format!( + "CREATE OR REPLACE TEMP TABLE {deleted} AS \ + SELECT {k} FROM (SELECT {k} FROM {t} WHERE {ic} EXCEPT SELECT {k} FROM ({sel}));" + )); + } + out.push("BEGIN TRANSACTION;".to_string()); + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {changed} \ + WHERE {changed}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + // Close vanished keys — no matching INSERT below, so they close without + // reopening. A key that later reappears isn't in `WHERE is_current`, so the + // `changed` diff treats it as new and opens a fresh version (a validity gap + // between the delete and the reactivation — correct SCD2). + if close_deleted { + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {deleted} \ + WHERE {deleted}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + } + out.push(format!( + "INSERT INTO {t} SELECT s.*, {ts} AS {vf}, CAST(NULL AS TIMESTAMP) AS {vt}, \ + true AS {ic} FROM ({sel}) s WHERE EXISTS (SELECT 1 FROM {changed} c \ + WHERE c.{k} IS NOT DISTINCT FROM s.{k});" + )); + out.push( + // Consumer convenience: a `_current` view (the live slice) so the + // common "just the latest version" read needs no `WHERE is_current`, + // and downstream scripts can `// on` / read it directly. For the + // effective-dated payoff, consumers `ASOF JOIN ON fact.key = + // dim. AND fact.ts >= dim.valid_from`. + // + // `CREATE VIEW IF NOT EXISTS` (not `OR REPLACE`), created inside the + // write transaction, on purpose: the view definition never changes + // (`SELECT * WHERE is_current` always reflects live data), and a + // catalog write advances the DuckLake snapshot — so `OR REPLACE` on + // every run would (a) advance the snapshot on an otherwise no-op + // unchanged run and (b) make the summary's `max(snapshot_id)` record + // the view DDL instead of the data write. `IF NOT EXISTS` creates it + // once (folded into the first data-write snapshot) and is a true no-op + // afterwards. The `_current` name is reserved: if a real table by + // that name already exists, `IF NOT EXISTS` skips silently (no view, + // no error) — documented as a reserved suffix. + format!("CREATE VIEW IF NOT EXISTS {t}_current AS SELECT * FROM {t} WHERE {ic};"), + ); + out.push("COMMIT;".to_string()); + out + } } /// The read that captures the DuckLake snapshot id produced by the write, for @@ -1111,6 +1293,131 @@ mod tests { ); } + #[test] + fn codegen_scd2_default_track_closes_old_opens_new() { + // Empty `track` ⇒ diff on all business columns via `* EXCLUDE (scd cols)`. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim_scd2", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: false, + }, + }; + let st = cg.statements(); + // bootstrap adds the three SCD metadata columns + assert!(st[0].starts_with("CREATE TABLE IF NOT EXISTS _wm_target.dim_scd2 AS SELECT *,")); + assert!(st[0].contains("AS valid_from")); + assert!(st[0].contains("AS valid_to")); + assert!(st[0].contains("AS is_current")); + // changed-key set captured before the transaction, all cols compared + assert!( + st[1].contains("CREATE OR REPLACE TEMP TABLE _wm_scd2_changed AS SELECT \"id\" FROM") + ); + assert!(st[1].contains("SELECT * FROM (SELECT id, name FROM dl.src) EXCEPT")); + assert!(st[1].contains("SELECT * EXCLUDE (valid_from, valid_to, is_current) FROM _wm_target.dim_scd2 WHERE is_current")); + assert_eq!(st[2], "BEGIN TRANSACTION;"); + // close: UPDATE (not MERGE) the prior open version of changed keys, with + // null-safe key matching (IS NOT DISTINCT FROM, not IN — IN drops NULLs) + assert!(st[3].starts_with("UPDATE _wm_target.dim_scd2 SET valid_to = CAST(now() AS TIMESTAMP), is_current = false")); + assert!(st[3].contains( + "WHERE is_current AND EXISTS (SELECT 1 FROM _wm_scd2_changed \ + WHERE _wm_scd2_changed.\"id\" IS NOT DISTINCT FROM _wm_target.dim_scd2.\"id\");" + )); + // open: INSERT the new current version, null-safe key matching + assert!(st[4].starts_with( + "INSERT INTO _wm_target.dim_scd2 SELECT s.*, CAST(now() AS TIMESTAMP) AS valid_from" + )); + assert!(st[4].contains( + "true AS is_current FROM (SELECT id, name FROM dl.src) s WHERE EXISTS \ + (SELECT 1 FROM _wm_scd2_changed c WHERE c.\"id\" IS NOT DISTINCT FROM s.\"id\");" + )); + // consumer-convenience `_current` view: `IF NOT EXISTS` (created once, + // no-op on unchanged reruns) and INSIDE the txn (folded into the write snapshot) + assert_eq!( + st[5], + "CREATE VIEW IF NOT EXISTS _wm_target.dim_scd2_current AS SELECT * FROM _wm_target.dim_scd2 WHERE is_current;" + ); + assert_eq!(st[6], "COMMIT;"); + // no fragile constructs: no MERGE INTO, and no NULL-dropping `IN (SELECT` + assert!(!st.iter().any(|s| s.contains("MERGE INTO"))); + assert!(!st.iter().any(|s| s.contains("IN (SELECT"))); + // soft-delete default: no deleted-key set, no second close + assert!(!st.iter().any(|s| s.contains("_wm_scd2_deleted"))); + } + + #[test] + fn codegen_scd2_explicit_track_projects_key_and_tracked_cols() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name, addr FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec!["name".to_string()], + close_deleted: false, + }, + }; + let st = cg.statements(); + // only key + tracked cols are compared (addr changes don't rotate a version) + assert!(st[1].contains("SELECT \"id\", \"name\" FROM (SELECT id, name, addr FROM dl.src) EXCEPT SELECT \"id\", \"name\" FROM _wm_target.dim WHERE is_current")); + } + + #[test] + fn codegen_scd2_close_deleted_adds_deleted_set_and_second_close() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: true, + }, + }; + let st = cg.statements(); + // the deleted-key set: current keys absent from the snapshot, captured + // before the transaction (like `changed`) + assert!(st.iter().any(|s| s.contains( + "CREATE OR REPLACE TEMP TABLE _wm_scd2_deleted AS SELECT \"id\" FROM \ + (SELECT \"id\" FROM _wm_target.dim WHERE is_current EXCEPT SELECT \"id\" FROM (SELECT id, name FROM dl.src));" + ))); + // a second close UPDATE against the deleted set (null-safe), and NO INSERT + // that reopens deleted keys (the only INSERT filters on `_wm_scd2_changed`) + assert!(st + .iter() + .any(|s| s.starts_with("UPDATE _wm_target.dim SET valid_to") + && s.contains( + "EXISTS (SELECT 1 FROM _wm_scd2_deleted \ + WHERE _wm_scd2_deleted.\"id\" IS NOT DISTINCT FROM _wm_target.dim.\"id\");" + ))); + assert_eq!( + st.iter().filter(|s| s.starts_with("INSERT INTO")).count(), + 1 + ); + assert!(st + .iter() + .find(|s| s.starts_with("INSERT INTO")) + .unwrap() + .contains("_wm_scd2_changed")); + // the deleted close is inside the transaction (between BEGIN and COMMIT) + let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap(); + let commit = st.iter().position(|s| s == "COMMIT;").unwrap(); + let del_close = st + .iter() + .position(|s| s.starts_with("UPDATE") && s.contains("_wm_scd2_deleted")) + .unwrap(); + assert!(begin < del_close && del_close < commit); + } + #[test] fn snapshot_capture_targets_alias() { assert_eq!( diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 4ecbc992bc..97c7d2959c 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -253,6 +253,64 @@ } } }, + { + "name": "materialize scd2 via history flag with key, track and deletes=close", + "code": "// pipeline\n// materialize ducklake://analytics/dim_customer key=id history track=name,tier deletes=close\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim_customer", + "unique_key": "id", + "scd2": true, + "track": ["name", "tier"], + "close_deleted": true + } + } + }, + { + "name": "materialize scd2 keyword alias with key only (track all non-key cols)", + "code": "// materialize scd2 ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id", + "scd2": true + } + } + }, + { + "name": "materialize key without history is plain merge (SCD1, not scd2)", + "code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id" + } + } + }, { "name": "materialize manual escape hatch, first value wins", "code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}", diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 883ddebcbc..96a81f863e 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -59,6 +59,12 @@ struct ExpectedMaterialize { append: bool, #[serde(default)] unique_key: Option, + #[serde(default)] + scd2: bool, + #[serde(default)] + track: Vec, + #[serde(default)] + close_deleted: bool, } #[derive(Deserialize)] @@ -191,6 +197,12 @@ fn pipeline_annotation_fixtures_match() { assert_eq!(m.manual, e.manual, "{ctx}: materialize manual"); assert_eq!(m.append, e.append, "{ctx}: materialize append"); assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key"); + assert_eq!(m.scd2, e.scd2, "{ctx}: materialize scd2"); + assert_eq!(m.track, e.track, "{ctx}: materialize track"); + assert_eq!( + m.close_deleted, e.close_deleted, + "{ctx}: materialize close_deleted" + ); } (got, want) => panic!( "{ctx}: materialize mismatch — got {:?}, want present={}", diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 6553abb5cc..fe86b09b09 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -182,7 +182,7 @@ windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) -workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char) +workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) workspace_diff: source_workspace_id(char), fork_workspace_id(char), path(char), kind(char), ahead(int), behind(int), has_changes(bool), exists_in_source(bool), exists_in_fork(bool) diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index eedb43bc1a..67997bc965 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -146,12 +146,11 @@ async fn list_foldernames( } fn validate_owner(owner: &str) -> Result<()> { - if !owner - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-') - { + if !owner.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-' || c == '.' || c == '@' + }) { return Err(error::Error::BadRequest( - "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, or slashes".to_string(), + "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, slashes, dots, or at-signs".to_string(), )); } Ok(()) diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 9aecd66947..a61977937d 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -408,6 +408,14 @@ async fn list_scripts( .and_then(|s| s.as_str()) .filter(|s| !s.is_empty() && *s != row.path.as_str()) .map(|s| s.to_string()); + // A draft-only pipeline node (`// pipeline`) has no deployed row to carry + // auto_kind, so compute it from the draft content — mirroring the create + // path — so the home page folds it into its pipeline like a deployed member. + let auto_kind = v + .get("content") + .and_then(|s| s.as_str()) + .filter(|c| parse_pipeline_annotations(c).in_pipeline) + .map(|_| "pipeline".to_string()); rows.push(ListableScript { hash: ScriptHash(0), path: row.path, @@ -429,7 +437,7 @@ async fn list_scripts( draft_only: Some(true), has_deploy_errors: false, ws_error_handler_muted: None, - auto_kind: None, + auto_kind, use_codebase: false, deployment_msg: None, kind, @@ -1289,10 +1297,16 @@ async fn create_script_internal<'c>( // but the executor parses the signature from the un-wrapped script, so // `$name` references in the SELECT stay bound at run time. } - // `key=` (merge) and `append` are mutually exclusive reconciliation - // strategies; append (INSERT-only) wins. Surface the conflict rather - // than silently dropping the dedup the author may have intended. - if m.unique_key.is_some() && m.append { + // Reconciliation strategies are mutually exclusive; surface a conflict + // rather than silently dropping behavior the author may have intended. + // Precedence must mirror the runtime (`duckdb_executor` strategy + // derivation): scd2 (`history`) > append > merge (`key=`) > replace. + if m.scd2 && m.append { + tracing::warn!( + "script {}: both `history`/`scd2` and `append` set on // materialize; history wins (SCD2), append ignored", + ns.path + ); + } else if m.unique_key.is_some() && m.append { tracing::warn!( "script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)", ns.path @@ -1335,14 +1349,28 @@ async fn create_script_internal<'c>( let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() { let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind); let mut a = effective_assets.unwrap_or_default(); - if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) { - a.push(windmill_common::assets::AssetWithAltAccessType { - path: m.target_path.clone(), - kind, - access_type: Some(windmill_common::assets::AssetUsageAccessType::W), - alt_access_type: None, - columns: None, - }); + // Produced assets: the managed table, plus — for managed scd2 — the + // `_current` companion view the runtime (re)creates each run. + // Registering the view as a write asset lets `// on + // ducklake://…/_current` subscribers be dispatched by the cascade + // (which fans out from these deploy-time asset rows); without it a + // subscriber on the view would silently never fire. Gated on `!manual`: + // manual mode owns its own DDL and short-circuits before the scd2 codegen + // (no view is created), so registering it there would be a false edge. + let mut targets = vec![m.target_path.clone()]; + if m.scd2 && !m.manual { + targets.push(format!("{}_current", m.target_path)); + } + for path in targets { + if !a.iter().any(|x| x.kind == kind && x.path == path) { + a.push(windmill_common::assets::AssetWithAltAccessType { + path, + kind, + access_type: Some(windmill_common::assets::AssetUsageAccessType::W), + alt_access_type: None, + columns: None, + }); + } } Some(a) } else { diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a7e1322ad9..8339df7b76 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -406,7 +406,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul ) } ObjectSettings::Gcs(gcs) => { - if gcs.service_account_key.is_empty() { + // Mirror `build_gcs_client`'s blank-key check (shared predicate): a blank/`{}` key falls + // back to the instance's ambient credentials there, so it must be rejected here too — + // otherwise an untrusted caller could probe with the server's identity (the very + // SSRF/credential-exfil this function guards against). + if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { return Err(error::Error::NotAuthorized( "Testing GCS storage without a service account key requires a super admin" .to_string(), @@ -2187,6 +2191,26 @@ mod object_storage_test_hardening { ); } + #[tokio::test] + async fn rejects_gcs_blank_service_account_key() { + // A blank key makes build_gcs_client fall back to the instance's ambient credentials, so an + // untrusted caller must not be allowed to test with it. The `serviceAccountKey` field is + // serialized via serde's `as_string` (`to_string` of the JSON value), so the settings UI's + // "no key" empty object arrives as `"{}"` and a null as `"null"` — both must be rejected. + for key in [serde_json::json!({}), serde_json::json!(null)] { + let settings: ObjectSettings = serde_json::from_value(serde_json::json!({ + "type": "Gcs", + "bucket": "b", + "serviceAccountKey": key + })) + .unwrap(); + assert!( + validate_object_storage_test(&settings).await.is_err(), + "blank key {key:?} should be rejected" + ); + } + } + fn ip(s: &str) -> IpAddr { s.parse().unwrap() } diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 2963b91a50..a0dca7576a 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -1552,6 +1552,15 @@ async fn update_user( } if let Some(d) = eu.disabled { + #[cfg(feature = "enterprise")] + if !d { + if let Some(msg) = + windmill_common::ee_oss::check_seat_cap_for_reactivation(&db, &email_to_update) + .await? + { + return Err(Error::BadRequest(msg)); + } + } sqlx::query_scalar!( "UPDATE password SET disabled = $1 WHERE email = $2", d, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 5de3bfd2c4..a00f2bfe13 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6,7 +6,10 @@ * LICENSE-AGPL for a copy of the license. */ -use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_predicate, check_scopes, require_devops_role, require_is_writer, + require_super_admin, ApiAuthed, +}; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; @@ -40,9 +43,10 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_user_against_rule, get_datatable_resource_from_db_unchecked, validate_fork_workspace_id, - DataTable, DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, - ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, + check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked, + validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, + DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, + ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -150,6 +154,9 @@ pub fn workspaced_service() -> Router { .route("/leave", post(leave_workspace)) .route("/get_workspace_name", get(get_workspace_name)) .route("/create_fork", post(create_workspace_fork)) + .route("/attach_dev_workspace", post(attach_dev_workspace)) + .route("/detach_dev_workspace", post(detach_dev_workspace)) + .route("/get_dev_workspace", get(get_dev_workspace)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_color", post(change_workspace_color)) .route( @@ -191,6 +198,7 @@ pub fn workspaced_service() -> Router { .route("/prune_versions", post(prune_versions)) .route("/list_ws_specific", get(list_ws_specific)) .route("/list_ws_specific_versions", get(list_ws_specific_versions)) + .route("/set_ws_specific", post(set_ws_specific)) } pub fn global_service() -> Router { Router::new() @@ -441,6 +449,20 @@ struct CreateWorkspaceFork { /// forked workspace's datatable config to point to the new database. #[serde(default)] forked_datatables: Vec, + /// Create the fork as a persistent dev workspace: the id is not required to carry the + /// `wm-fork-` prefix, and at most one dev workspace may exist per parent. + #[serde(default)] + is_dev_workspace: bool, + /// When creating a dev workspace, lock the parent ("prod") against direct deployment and/or + /// ad-hoc forking, so edits are funneled through the dev workspace. + #[serde(default)] + lock_prod_deploy: bool, + #[serde(default)] + lock_prod_forking: bool, + /// Copy the parent's members (usr rows + group memberships) into the fork so + /// the team can work in it. Defaults off; the dev-workspace UI defaults it on. + #[serde(default)] + copy_members: bool, } #[derive(Deserialize)] @@ -469,6 +491,7 @@ struct UserWorkspace { pub color: Option, pub operator_settings: Option>, pub parent_workspace_id: Option, + pub is_dev_workspace: bool, pub disabled: bool, } @@ -637,6 +660,34 @@ async fn exists_workspace( Ok(Json(exists)) } +/// Whether this workspace already has an active canonical dev workspace. The create-fork UI can't +/// rely on the caller's workspace list to decide this — a dev paired to this prod may exist that the +/// caller isn't a member of — so it asks the server, which sees all children. +#[derive(Serialize)] +struct DevWorkspaceInfo { + id: String, + name: String, +} + +/// This workspace's active canonical dev workspace, if any. The create-fork UI and the dev-workspace +/// settings tab can't rely on the caller's workspace list — a dev paired to this prod may exist that +/// the caller isn't a member of — so they ask the server, which sees all children. Returns its id/name +/// so a prod admin who isn't a dev member can still see the pairing and detach it. +async fn get_dev_workspace( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let dev = sqlx::query_as!( + DevWorkspaceInfo, + "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + &w_id + ) + .fetch_optional(&db) + .await?; + Ok(Json(dev)) +} + async fn list_workspaces( authed: ApiAuthed, Extension(user_db): Extension, @@ -3643,6 +3694,7 @@ async fn user_workspaces( let workspaces = sqlx::query_as!( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, + workspace.is_dev_workspace, CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, usr.disabled FROM workspace @@ -3826,6 +3878,8 @@ async fn create_workspace( } } + validate_workspace_name(&nw.name)?; + let mut tx: Transaction<'_, Postgres> = db.begin().await?; check_w_id_conflict(&mut tx, &nw.id).await?; @@ -4423,6 +4477,28 @@ async fn clone_groups( Ok(()) } +/// Copy the source workspace's members (the `usr` rows, carrying each member's role) into the +/// target so a fork/dev can be a shared environment. Idempotent — skips members the target already +/// has. Group memberships are not handled here: the sole caller is the create-fork path, where +/// `clone_groups` already copies the source's full group structure (including `all` membership). +async fn copy_workspace_members( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via) + SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via + FROM usr WHERE workspace_id = $2 + ON CONFLICT DO NOTHING", + target_workspace_id, + source_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_resource_types( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -4972,7 +5048,27 @@ async fn create_workspace_fork_branch( return Err(Error::PermissionDenied(msg)); } - validate_fork_workspace_id(&nw.id)?; + // Two-phase create for git-synced workspaces: this endpoint only creates the git branch(es) and + // validates up front; it does NOT create the workspace row. The caller follows up with + // `create_workspace_fork`, which inserts the row and applies the dev designation + prod lock + + // member copy. So the dev/lock/copy_members fields here are validated only — they are acted on by + // that second call. Validating early lets a bad request fail before any branch is created. + if nw.is_dev_workspace { + validate_dev_workspace_id(&nw.id)?; + ensure_dev_parent_is_root(&db, &w_id).await?; + // Reject before creating any git branch if the parent already has a dev workspace, + // otherwise the deferred branch-creation job leaves a dangling branch on the synced repos. + ensure_no_existing_dev_workspace(&db, &w_id).await?; + // Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking the + // parent mutates its protection rules), so require admin of the parent regardless of the lock + // flags — mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker + // could claim the dev slot. Enforced in this first phase too so the request fails before any + // git branch is created rather than leaving dangling branches. + require_admin(authed.is_admin, &authed.username)?; + } else { + validate_fork_workspace_id(&nw.id)?; + } + validate_workspace_name(&nw.name)?; // Fail before creating any git branch so a name conflict doesn't leave a // dangling branch on the synced repos. @@ -5117,7 +5213,12 @@ async fn create_workspace_fork( ))); } - validate_fork_workspace_id(&nw.id)?; + if nw.is_dev_workspace { + validate_dev_workspace_id(&nw.id)?; + } else { + validate_fork_workspace_id(&nw.id)?; + } + validate_workspace_name(&nw.name)?; // Check the id conflict before the CE workspace-count limit so that // re-using a taken (possibly archived) fork id reports the actual // conflict instead of a misleading "maximum number of workspaces" error. @@ -5143,18 +5244,29 @@ async fn create_workspace_fork( return Err(Error::PermissionDenied(msg)); } + if nw.is_dev_workspace { + ensure_dev_parent_is_root(&db, &parent_workspace_id).await?; + // Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking prod + // mutates its protection rules), so require admin of the parent regardless of the lock flags — + // mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker could + // claim the dev slot (and, without member copy, prod admins might not even see it to detach). + require_admin(authed.is_admin, &authed.username)?; + ensure_no_existing_dev_workspace(&db, &parent_workspace_id).await?; + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; let forked_id = nw.id; sqlx::query!( "INSERT INTO workspace - (id, name, owner, parent_workspace_id) - VALUES ($1, $2, $3, $4)", + (id, name, owner, parent_workspace_id, is_dev_workspace) + VALUES ($1, $2, $3, $4, $5)", forked_id, nw.name, authed.email, parent_workspace_id, + nw.is_dev_workspace, ) .execute(&mut *tx) .await?; @@ -5169,11 +5281,24 @@ async fn create_workspace_fork( .execute(&mut *tx) .await?; + // Optionally bring the parent's members into the fork (a shared dev env). Dev-only: it's part of the + // dev-workspace feature (and the frontend only offers it there), so the backend enforces it rather + // than trusting the client — copying the parent's whole team into an ordinary throwaway fork isn't + // intended. Dev creation is already admin-gated, so this is transitively admin-only too. Done before + // the explicit creator insert below so the creator (a parent member) is copied with full metadata + // (operator/role/is_service_account/added_via), not the bare row the insert alone would leave. + if nw.copy_members && nw.is_dev_workspace { + copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id).await?; + } + + // Ensure the creator is a member of the fork even without copy_members (or if they aren't a parent + // member). No-op when copy_members already brought their full row. sqlx::query!( "INSERT INTO usr (workspace_id, email, username, is_admin) SELECT $1, email, username, is_admin FROM usr WHERE workspace_id = $3 AND email = $2 + ON CONFLICT DO NOTHING ", forked_id, authed.email, @@ -5196,6 +5321,18 @@ async fn create_workspace_fork( apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; } + // Lock the parent ("prod") so edits are funneled through this dev workspace. + let locked_prod = nw.is_dev_workspace && (nw.lock_prod_deploy || nw.lock_prod_forking); + if locked_prod { + lock_prod_workspace( + &mut tx, + &parent_workspace_id, + nw.lock_prod_deploy, + nw.lock_prod_forking, + ) + .await?; + } + audit_log( &mut *tx, &authed, @@ -5208,9 +5345,235 @@ async fn create_workspace_fork( .await?; tx.commit().await?; + if locked_prod { + windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id); + } + Ok(format!("Created forked workspace {}", &forked_id)) } +#[derive(Deserialize)] +struct AttachDevWorkspace { + dev_workspace_id: String, + #[serde(default)] + lock_prod_deploy: bool, + #[serde(default)] + lock_prod_forking: bool, +} + +#[derive(Deserialize)] +struct DetachDevWorkspace { + dev_workspace_id: String, +} + +/// Pair an existing standalone workspace to this workspace ("prod") as its dev workspace, without +/// cloning any data (both already exist). Sets the dev's parent + deploy_to to prod and, optionally, +/// locks prod against direct deployment. +async fn attach_dev_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Path(prod_w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + if *CLOUD_HOSTED { + return Err(Error::BadRequest( + "Dev workspaces are not available on app.windmill.dev".to_string(), + )); + } + + let dev_w_id = req.dev_workspace_id; + if dev_w_id == prod_w_id { + return Err(Error::BadRequest( + "A workspace cannot be its own dev workspace".to_string(), + )); + } + + // The id is interpolated into a `wm-fork//` branch name like any fork. + validate_dev_workspace_id(&dev_w_id)?; + + let dev = sqlx::query!( + r#"SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1"#, + &dev_w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Workspace {} not found", dev_w_id)))?; + + if dev.deleted { + return Err(Error::BadRequest(format!( + "Workspace {} is archived", + dev_w_id + ))); + } + // A candidate that already belongs to a DIFFERENT parent can't be attached. A candidate already + // parented to this prod is allowed: it's the recovery path after renaming a dev workspace (the + // rename keeps the parent but drops the dev flag), and re-designating an existing fork of this + // prod as its dev. + if dev + .parent_workspace_id + .as_deref() + .is_some_and(|p| p != prod_w_id) + { + return Err(Error::BadRequest(format!( + "Workspace {} is already a fork or dev workspace of another workspace", + dev_w_id + ))); + } + // The candidate can't itself be a prod with its own dev workspace (no nested dev chains). + ensure_no_existing_dev_workspace(&db, &dev_w_id).await?; + + // Prod must be a root workspace, otherwise attaching could form a parent<->child cycle (e.g. + // attaching A as the dev of B when B is already the dev of A), which breaks hierarchy traversal. + let prod_has_parent = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#, + &prod_w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Workspace {} not found", prod_w_id)))?; + if prod_has_parent { + return Err(Error::BadRequest(format!( + "Workspace {} is itself a fork or dev workspace and cannot be a prod workspace", + prod_w_id + ))); + } + + // The caller must be admin of the dev workspace too (or a superadmin). + let is_admin_of_dev = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &dev_w_id, + &authed.email + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if !is_admin_of_dev && !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Attaching workspace '{dev_w_id}' as a dev requires being an admin of it (or a superadmin)" + ))); + } + + ensure_no_existing_dev_workspace(&db, &prod_w_id).await?; + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + &prod_w_id, + &dev_w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + &prod_w_id, + &dev_w_id + ) + .execute(&mut *tx) + .await?; + + if req.lock_prod_deploy || req.lock_prod_forking { + lock_prod_workspace( + &mut tx, + &prod_w_id, + req.lock_prod_deploy, + req.lock_prod_forking, + ) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "workspaces.attach_dev_workspace", + ActionKind::Update, + &prod_w_id, + Some(&dev_w_id), + None, + ) + .await?; + tx.commit().await?; + + // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping + // so per-workspace job tags route to the prod family immediately rather than after the TTL. + windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + + if req.lock_prod_deploy || req.lock_prod_forking { + windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); + } + + Ok(format!( + "Attached {} as dev workspace of {}", + dev_w_id, prod_w_id + )) +} + +/// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the +/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork). +async fn detach_dev_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Path(prod_w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let dev_w_id = req.dev_workspace_id; + let is_dev_of_prod = sqlx::query_scalar!( + r#"SELECT EXISTS( + SELECT 1 FROM workspace + WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace + )"#, + &dev_w_id, + &prod_w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !is_dev_of_prod { + return Err(Error::BadRequest(format!( + "{} is not the dev workspace of {}", + dev_w_id, prod_w_id + ))); + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + &dev_w_id + ) + .execute(&mut *tx) + .await?; + // Only one dev per prod, so detaching it means prod no longer has a dev: drop the lock rule. + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2", + &prod_w_id, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.detach_dev_workspace", + ActionKind::Update, + &prod_w_id, + Some(&dev_w_id), + None, + ) + .await?; + tx.commit().await?; + + windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); + + Ok(format!( + "Detached dev workspace {} from {}", + dev_w_id, prod_w_id + )) +} + async fn edit_workspace( authed: ApiAuthed, Extension(db): Extension, @@ -5250,6 +5613,10 @@ pub(crate) async fn archive_workspace_impl( db: &DB, w_id: &str, username: &str, + // When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the + // prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the + // archive — a later failure can't strand a half-archived dev that's still flagged/locked. + dev_lock_parent: Option<&str>, ) -> Result<(usize, usize, usize)> { // Step 1: Disable all schedules and clear their queued jobs let mut tx = db.begin().await?; @@ -5291,6 +5658,30 @@ pub(crate) async fn archive_workspace_impl( .execute(&mut *tx) .await?; + if let Some(prod) = dev_lock_parent { + // Dissolve the dev pairing atomically with the archive: clear the canonical-dev flag (so the + // archived row no longer occupies the parent's one-dev slot), and drop the prod's lock unless a + // replacement dev already holds it (NOT EXISTS sees the just-cleared flag within this tx, so the + // row being archived doesn't count). + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2 + AND NOT EXISTS ( + SELECT 1 FROM workspace + WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false + )", + prod, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; // Step 2: Get all remaining queued jobs for this workspace (non-schedule jobs) @@ -5334,8 +5725,39 @@ async fn archive_workspace( ) -> Result { require_admin(authed.is_admin, &authed.username)?; + // If this is an attached dev workspace, archiving it leaves the prod with no active dev (the + // unique index and user_workspaces both ignore deleted=true), so clear the prod's + // dev_workspace_lock too. Gate it on prod-admin since it removes prod's protection rule (mirrors + // detach/delete) — a dev-admin who isn't a prod-admin must not be able to unlock prod this way. + let dev_lock_parent: Option = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + if let Some(ref prod) = dev_lock_parent { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + prod, + &authed.email + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if !is_prod_admin + && !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied(format!( + "Archiving dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } + } + + // The dev pairing teardown (clear is_dev + drop the prod lock) runs inside archive_workspace_impl's + // transaction, atomically with `deleted = true`. let (schedules_count, canceled_count, deleted_tokens_count) = - archive_workspace_impl(&db, &w_id, &authed.username).await?; + archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?; // Audit log let mut tx = db.begin().await?; @@ -5370,6 +5792,10 @@ async fn archive_workspace( .await?; tx.commit().await?; + if let Some(prod) = dev_lock_parent { + windmill_common::workspaces::invalidate_protection_rules_cache(&prod); + } + Ok(format!( "Archived workspace {}, disabled {} schedules, canceled {} jobs and deleted {} tokens", &w_id, schedules_count, canceled_count, deleted_tokens_count @@ -6211,6 +6637,119 @@ async fn list_protection_rules( )) } +/// Insert or replace a protection ruleset within an existing transaction. Unlike the +/// `create_protection_rule` handler (which rejects an existing name), this upserts, so it is safe to +/// call programmatically when designating a dev/prod pair. Callers MUST invalidate the +/// protection-rules cache (`invalidate_protection_rules_cache`) after the transaction commits. +async fn upsert_protection_rule( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + name: &str, + rules: ProtectionRules, + bypass_groups: &[String], + bypass_users: &[String], +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (workspace_id, name) + DO UPDATE SET rules = EXCLUDED.rules, + bypass_groups = EXCLUDED.bypass_groups, + bypass_users = EXCLUDED.bypass_users + "#, + w_id, + name, + rules.bits(), + bypass_groups, + bypass_users, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Lock a prod workspace by applying the reserved dev-workspace lock rule with the selected +/// restrictions (block direct deployment and/or ad-hoc forking). Non-admins are then funneled +/// through the one dev workspace; admins bypass the rules (their existing escape hatch). +async fn lock_prod_workspace( + tx: &mut Transaction<'_, Postgres>, + prod_w_id: &str, + block_deploy: bool, + block_forking: bool, +) -> Result<()> { + let mut rules = Vec::new(); + if block_deploy { + rules.push(ProtectionRuleKind::DisableDirectDeployment); + } + if block_forking { + rules.push(ProtectionRuleKind::DisableWorkspaceForking); + } + if rules.is_empty() { + return Ok(()); + } + upsert_protection_rule( + tx, + prod_w_id, + DEV_WORKSPACE_LOCK_RULE_NAME, + ProtectionRules::from(&rules), + &[], + &[], + ) + .await +} + +/// Error out if `parent_w_id` already has an active (non-archived) dev workspace. Mirrors the +/// partial unique index `workspace_canonical_dev_idx` with a friendly message. +async fn ensure_no_existing_dev_workspace(db: &DB, parent_w_id: &str) -> Result<()> { + let existing = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + parent_w_id + ) + .fetch_optional(db) + .await?; + if let Some(existing) = existing { + return Err(Error::BadRequest(format!( + "Workspace '{}' already has a dev workspace ('{}'). Detach it before creating another.", + parent_w_id, existing + ))); + } + Ok(()) +} + +/// A dev workspace pairs with a root prod workspace; nesting dev workspaces (a dev of a dev) isn't +/// supported and would muddle the prod<->dev relationship. +async fn ensure_dev_parent_is_root(db: &DB, parent_w_id: &str) -> Result<()> { + let parent_is_fork = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#, + parent_w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if parent_is_fork { + return Err(Error::BadRequest(format!( + "Cannot create a dev workspace of '{}' because it is itself a fork or dev workspace.", + parent_w_id + ))); + } + Ok(()) +} + +/// `dev_workspace_lock` is owned by the dev-workspace feature (attach/detach/archive/delete create and +/// remove it by name). Reserve it from the public protection-rule API so a user-managed rule can't +/// collide: otherwise the feature's name-based cleanup would clobber the user's rule, or a manual edit +/// could weaken the feature's lock. +fn reject_reserved_rule_name(name: &str) -> Result<()> { + if name == DEV_WORKSPACE_LOCK_RULE_NAME { + return Err(Error::BadRequest(format!( + "'{}' is a reserved protection-rule name managed by the dev workspace feature", + DEV_WORKSPACE_LOCK_RULE_NAME + ))); + } + Ok(()) +} + /// Create a new protection rule async fn create_protection_rule( authed: ApiAuthed, @@ -6219,6 +6758,7 @@ async fn create_protection_rule( Json(req): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&req.name)?; let mut tx = db.begin().await?; @@ -6293,6 +6833,7 @@ async fn update_protection_rule( Json(req): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&rule_name)?; let mut tx = db.begin().await?; @@ -6367,6 +6908,7 @@ async fn delete_protection_rule( Path((w_id, rule_name)): Path<(String, String)>, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&rule_name)?; let mut tx = db.begin().await?; @@ -6511,10 +7053,22 @@ async fn compare_workspaces( })); } + // Honor ws_specific at read time: a workspace-specific resource/variable keeps its own value per + // environment, so it must never appear in the normal diff (the per-item compare suppresses it, + // but a cached `has_changes=true` row is trusted without re-running that compare, so filter those + // here too). Seeding the initial copy onto a side that lacks it is a separate explicit action + // (the "Create in " button on the Workspace-specific list), not part of the diff. The row + // is left intact, so unpinning resurfaces it without a re-tally. let diff_items = sqlx::query_as!( WorkspaceDiffRow, "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff - WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + WHERE source_workspace_id = $1 AND fork_workspace_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM ws_specific ws + WHERE ws.path = workspace_diff.path + AND ws.item_kind = workspace_diff.kind + AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id) + )", source_workspace_id, fork_workspace_id, ) @@ -7242,7 +7796,6 @@ async fn compare_two_resources( .fetch_optional(db) .await?; - // If either side is ws_specific, consider unchanged let source_ws_specific = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)", source_workspace_id, @@ -7261,6 +7814,9 @@ async fn compare_two_resources( .await? .unwrap_or(false); + // A workspace-specific resource keeps its own value per environment, so it never appears in the + // diff (in either direction). Seeding the initial copy onto a side that lacks it is a separate + // explicit action ("Create in "), not a diff entry. if source_ws_specific || target_ws_specific { return Ok(ItemComparison { has_changes: false, @@ -7317,7 +7873,8 @@ async fn compare_two_variables( .fetch_one(db) .await?; - // If either side is ws_specific, consider unchanged + // A workspace-specific variable keeps its own value per environment, so it never appears in the + // diff. Seeding the initial copy onto a side that lacks it is a separate explicit action. if presence.src_ws || presence.tgt_ws { return Ok(ItemComparison { has_changes: false, @@ -7831,6 +8388,19 @@ async fn list_ws_specific( .fetch_all(&mut *tx) .await?; tx.commit().await?; + // RLS gates membership/folder access, but a scoped API token must also be held to its read + // scopes — mirror the resource/variable list endpoints, which filter with these predicates so a + // token lacking `resources:read:*` / `variables:read:*` can't enumerate pinned paths it can't read. + let resource_allowed = build_scope_path_predicate(&authed, "resources", "read"); + let variable_allowed = build_scope_path_predicate(&authed, "variables", "read"); + let items = items + .into_iter() + .filter(|it| match it.item_kind.as_str() { + "resource" => resource_allowed(&it.path), + "variable" => variable_allowed(&it.path), + _ => false, + }) + .collect::>(); Ok(Json(items)) } @@ -7853,6 +8423,17 @@ async fn list_ws_specific_versions( ))); } + // A scoped API token must hold the read scope for this path, like the resource/variable read + // endpoints. Without the scope, report no versions rather than leaking the path's history. + let domain = if q.kind == "resource" { + "resources" + } else { + "variables" + }; + if !build_scope_path_predicate(&authed, domain, "read")(&q.path) { + return Ok(Json(vec![])); + } + let versions: Vec = sqlx::query_scalar!( r#"SELECT ws AS "ws!" FROM list_ws_specific_versions($1, $2, $3, $4)"#, &w_id, @@ -7865,3 +8446,155 @@ async fn list_ws_specific_versions( Ok(Json(versions)) } + +#[derive(Deserialize)] +struct SetWsSpecificBody { + item_kind: String, + path: String, + value: bool, +} + +/// Mark (or unmark) a single resource/variable as workspace-specific. Pinning +/// excludes it from the deploy diff so each environment keeps its own value +/// (see `compare_two_resources`/`compare_two_variables`). Set per-workspace, so +/// the compare page calls this once per side to flag both environments. +async fn set_ws_specific( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(body): Json, +) -> Result { + if body.item_kind != "resource" && body.item_kind != "variable" { + return Err(Error::BadRequest(format!( + "Invalid kind '{}'. Must be 'resource' or 'variable'", + body.item_kind + ))); + } + + // Reject a malformed path with a 400 before the auth check, which indexes the leading segments and + // would otherwise panic (500) on a path like `u` with no segment. Accept all three shared path + // shapes Windmill uses — `u/`, `f/`, `g/` (e.g. seeded `g/all/...` resources). + let segs: Vec<&str> = body.path.split('/').collect(); + if segs.len() < 2 || !matches!(segs[0], "u" | "f" | "g") || segs[1].is_empty() { + return Err(Error::BadRequest(format!( + "Invalid {} path: {}", + body.item_kind, body.path + ))); + } + + // Authorize like the resource/variable editors' own ws_specific toggle: + // actual write access to the item + token scope + the workspace deploy rules. + // `require_owner_of_path` is the real write gate (the resource editor uses it); + // `check_scopes` only constrains scoped tokens (it is a no-op for session/cookie + // logins). Together: a non-admin who can edit the item may pin it, while a + // read-only member is rejected and a locked workspace still blocks non-deployers. + // `require_is_writer` matches the resource/variable editors' write semantics (owner, folder + // writer, or item writer via extra_perms) — not owner-only. + let writer_query = if body.item_kind == "resource" { + "SELECT extra_perms FROM resource WHERE path = $1 AND workspace_id = $2" + } else { + "SELECT extra_perms FROM variable WHERE path = $1 AND workspace_id = $2" + }; + require_is_writer( + &authed, + &body.path, + &w_id, + db.clone(), + writer_query, + &body.item_kind, + ) + .await?; + check_scopes(&authed, || { + format!("{}s:write:{}", body.item_kind, body.path) + })?; + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( + &w_id, + &authed.username, + &authed.groups, + authed.is_admin, + &db, + ) + .await? + { + return Err(Error::PermissionDenied(msg)); + } + + let mut tx = user_db.begin(&authed).await?; + + if body.value { + // Existence guard keeps a dangling marker from being created for a + // path absent in this workspace. + if body.item_kind == "resource" { + sqlx::query!( + "INSERT INTO ws_specific (workspace_id, item_kind, path) + SELECT $1::varchar, 'resource', $2::varchar + WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar) + ON CONFLICT DO NOTHING", + w_id, + body.path, + ) + .execute(&mut *tx) + .await?; + // A resource owns its `$var:` secrets, so pin those too. + windmill_store::resources::mark_linked_variables_ws_specific( + &mut tx, &authed, &w_id, &body.path, + ) + .await?; + } else { + sqlx::query!( + "INSERT INTO ws_specific (workspace_id, item_kind, path) + SELECT $1::varchar, 'variable', $2::varchar + WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar) + ON CONFLICT DO NOTHING", + w_id, + body.path, + ) + .execute(&mut *tx) + .await?; + } + } else { + // Unmark only this item; linked variables stay flagged (they may be + // referenced by other resources) — mirrors the resource-form toggle. + sqlx::query!( + "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3", + w_id, + body.item_kind, + body.path, + ) + .execute(&mut *tx) + .await?; + // While pinned, the item's cached workspace_diff verdict was never recomputed (the compare + // read filter excludes it), so it may now be stale in either direction. Mark it NULL so the + // next compare re-evaluates from scratch — and the now-shared item reappears (or is dropped) + // correctly instead of being stuck on its pre-pin verdict. + sqlx::query!( + "UPDATE workspace_diff SET has_changes = NULL + WHERE path = $2 AND kind = $3 + AND ($1 IN (source_workspace_id, fork_workspace_id))", + w_id, + body.path, + body.item_kind, + ) + .execute(&mut *tx) + .await?; + } + + let value_str = body.value.to_string(); + audit_log( + &mut *tx, + &authed, + &format!("{}s.set_ws_specific", body.item_kind), + ActionKind::Update, + &w_id, + Some(&body.path), + Some([("value", value_str.as_str())].into()), + ) + .await?; + + tx.commit().await?; + Ok(format!( + "Set workspace-specific={} for {} {}", + body.value, body.item_kind, body.path + )) +} diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 3cb2b29f79..47d8c24b92 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use windmill_api_auth::{require_super_admin, ApiAuthed}; -use windmill_common::workspaces::WM_FORK_PREFIX; use windmill_common::DB; use crate::workspaces::{ @@ -26,7 +25,7 @@ use windmill_common::{ db::UserDB, error::{Error, Result}, utils::require_admin, - workspaces::DataTable, + workspaces::{DataTable, DEV_WORKSPACE_LOCK_RULE_NAME, WM_FORK_PREFIX}, }; use windmill_queue::schedule::{get_schedule_opt, push_scheduled_job}; @@ -65,22 +64,42 @@ pub(crate) async fn change_workspace_id( old_id, rw.new_id ); - // Create new workspace with new id and name. A fork that keeps a wm-fork- - // id must carry its parent_workspace_id over, otherwise it becomes a - // parentless "fork of nothing" with no source to compare or merge against. - // A non-fork target id means the workspace is being promoted out of a fork, - // so the parent pointer is intentionally cleared. + // Create new workspace with new id and name. Fork lineage AND the dev designation are preserved + // from the source row, not inferred from the new id's prefix: a prefix-less fork (a dev or + // detached-dev workspace) would otherwise be silently promoted to a root workspace, and a dev + // would lose its flag — leaving its prod locked with no canonical dev. Promoting out of a fork is + // a separate, explicit action — a rename never does it implicitly. info!("Creating new workspace row"); - let new_is_fork = rw.new_id.starts_with(WM_FORK_PREFIX); + let old = sqlx::query!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!", is_dev_workspace + FROM workspace WHERE id = $1"#, + &old_id + ) + .fetch_optional(&mut *tx) + .await?; + let new_is_fork = old.as_ref().map(|o| o.has_parent).unwrap_or(false); + let new_is_dev = new_is_fork && old.as_ref().map(|o| o.is_dev_workspace).unwrap_or(false); + // Neutralize the old row's dev flag BEFORE inserting the new one: the move-and-archive archives + // the old row only later, so without this the new dev row and the not-yet-archived old dev row + // would momentarily both be active under the same parent and trip the one-dev-per-parent index. + if new_is_dev { + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + &old_id + ) + .execute(&mut *tx) + .await?; + } sqlx::query!( - "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id) + "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace) SELECT $1, $2, owner, false, premium, - CASE WHEN $4 THEN parent_workspace_id ELSE NULL END + CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5 FROM workspace WHERE id = $3", &rw.new_id, &rw.new_name, &old_id, - new_is_fork + new_is_fork, + new_is_dev ) .execute(&mut *tx) .await?; @@ -360,8 +379,18 @@ pub(crate) async fn change_workspace_id( // must follow the renamed parent to the new id, otherwise it is left // pointing at the soft-deleted old shell (whose data has moved here). info!("Re-parenting child forks to the new workspace id"); + let reparented_children: Vec = sqlx::query_scalar!( + "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id", + &rw.new_id, + &old_id + ) + .fetch_all(&mut *tx) + .await?; + + // A dev/fork's `deploy_to` points at the prod root, so it must follow the rename too — otherwise + // the child re-parents to the new id but still deploys to the soft-deleted old shell. sqlx::query!( - "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2", + "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2", &rw.new_id, &old_id ) @@ -696,11 +725,18 @@ pub(crate) async fn change_workspace_id( tx.commit().await?; + // The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent + // routing cache so jobs route under the renamed root rather than the old (archived) one until the + // 300s TTL would otherwise expire. + for child in &reparented_children { + windmill_queue::tags::invalidate_fork_parent_cache(child); + } + // Archive old workspace: disable schedules, cancel remaining jobs, set deleted=true // Note: schedules were already moved to new workspace, so this will find 0 schedules info!("Archiving old workspace"); let (_schedules_count, canceled_count, _deleted_tokens_count) = - archive_workspace_impl(&db, &old_id, &authed.username).await?; + archive_workspace_impl(&db, &old_id, &authed.username, None).await?; info!( "Workspace id change completed: moved {} to {}, archived old workspace", @@ -734,15 +770,65 @@ pub(crate) async fn delete_workspace( _ => Ok(w_id), }?; - if dwq.only_delete_forks.unwrap_or(false) && !w_id.starts_with(WM_FORK_PREFIX) { + let is_fork = workspace_is_fork(&db, &w_id).await?; + if dwq.only_delete_forks.unwrap_or(false) && !is_fork { return Err(Error::BadRequest( "Cannot delete this workspace because it is not a workspace fork.".to_string(), )); } let mut tx = db.begin().await?; - if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { - require_super_admin(&db, &authed.email).await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Deleting this workspace requires being the fork's owner or a superadmin".to_string(), + )); + } + + // Don't hard-delete a workspace that still has a dev workspace paired to it: the FK is + // ON DELETE SET NULL, which would orphan the (prefix-less) dev into a parentless, non-fork row + // its owner could no longer self-delete. Require detaching/deleting the dev first. Ordinary + // forks have no such guard — they keep their prefix and stay owner-deletable when orphaned. + // Archived devs (deleted = true) are included: they keep is_dev_workspace = true, so SET NULL on + // their parent would violate the `is_dev ⇒ has parent` CHECK and fail the whole delete with a 500. + if let Some(dev_id) = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + { + return Err(Error::BadRequest(format!( + "Cannot delete workspace '{}' because it has a dev workspace ('{}'). Detach or delete the dev workspace first.", + w_id, dev_id + ))); + } + + // Deleting an attached dev workspace removes the parent prod's dev_workspace_lock (below), so it + // must be a prod-admin action, not just the dev's own owner (dev ownership can diverge from + // prod's) — mirrors detach_dev_workspace, which is prod-admin gated. + if let Some(prod) = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &prod, + &authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if !is_prod_admin && !is_super_admin_email(&db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Deleting dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } } sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id) @@ -920,10 +1006,31 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + // If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop + // the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod + // stays locked against direct deploy/forking with no dev workspace left to make changes in. + let dev_lock_parent: Option = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; + if let Some(ref parent) = dev_lock_parent { + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2", + parent, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + } + // Record under the instance-level "admins" workspace. The per-workspace audit // rows are deleted along with the workspace, so this instance-level entry is the // only durable, superadmin-discoverable record of who deleted the workspace. @@ -939,6 +1046,10 @@ pub(crate) async fn delete_workspace( .await?; tx.commit().await?; + if let Some(parent) = dev_lock_parent { + windmill_common::workspaces::invalidate_protection_rules_cache(&parent); + } + Ok(format!("Deleted workspace {}", &w_id)) } @@ -957,9 +1068,15 @@ pub async fn drop_forked_datatable_databases( Json(req): Json, ) -> Result>> { // Same permission check as delete_workspace: fork owner or super admin + let is_fork = workspace_is_fork(&db, &w_id).await?; let mut tx = db.begin().await?; - if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { - require_super_admin(&db, &authed.email).await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Dropping forked datatable databases requires being the fork's owner or a superadmin" + .to_string(), + )); } tx.commit().await?; @@ -1103,3 +1220,21 @@ async fn is_workspace_owner( .await?; Ok(owner.map(|o| o == authed.email).unwrap_or(false)) } + +/// Whether a workspace is a fork or dev workspace. Both forks and dev workspaces set +/// `parent_workspace_id`, but a `wm-fork-` workspace can outlive its parent (the FK is +/// `ON DELETE SET NULL`), so also treat the prefix as fork-ness — otherwise an orphaned fork would +/// lose owner-self-delete. Used to gate owner-self-delete, which is permitted for forks/dev +/// workspaces but requires superadmin otherwise. +async fn workspace_is_fork(db: &DB, w_id: &str) -> Result { + if w_id.starts_with(WM_FORK_PREFIX) { + return Ok(true); + } + Ok(sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8913253d25..9be2175f53 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.742.0 + version: 1.744.0 title: Windmill API contact: @@ -1184,6 +1184,89 @@ paths: schema: type: string + /w/{workspace}/workspaces/attach_dev_workspace: + post: + summary: attach an existing workspace as the dev workspace of this (prod) workspace + operationId: attachDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + dev_workspace_id: + type: string + lock_prod_deploy: + type: boolean + lock_prod_forking: + type: boolean + required: + - dev_workspace_id + responses: + "200": + description: dev workspace attached + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/detach_dev_workspace: + post: + summary: detach the dev workspace from this (prod) workspace + operationId: detachDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + dev_workspace_id: + type: string + required: + - dev_workspace_id + responses: + "200": + description: dev workspace detached + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/get_dev_workspace: + get: + summary: this workspace's active canonical dev workspace, if any + operationId: getDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: the dev workspace, or null if none + content: + application/json: + schema: + nullable: true + type: object + properties: + id: + type: string + name: + type: string + required: + - id + - name + /workspaces/exists: post: summary: exists workspace @@ -6238,6 +6321,42 @@ paths: items: type: string + /w/{workspace}/workspaces/set_ws_specific: + post: + summary: mark or unmark an item as workspace-specific + operationId: setWsSpecific + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + item_kind: + type: string + enum: + - resource + - variable + path: + type: string + value: + type: boolean + required: + - item_kind + - path + - value + responses: + "200": + description: workspace-specific flag updated + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/public_app_rate_limit: post: summary: Set public app rate limit for this workspace @@ -21355,6 +21474,54 @@ paths: format: int64 description: Number of pipeline-member scripts in the folder + /w/{workspace}/assets/partitions: + get: + summary: List materialized partitions for a ducklake asset + operationId: listAssetPartitions + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/`) + schema: + type: string + responses: + "200": + description: per-partition materialization status + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/MaterializedPartition" + + /w/{workspace}/assets/asset_schemas: + get: + summary: List captured output-schema versions for a ducklake asset + operationId: listAssetSchemas + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + responses: + "200": + description: captured schema versions, newest first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AssetSchemaVersion" + /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -27756,6 +27923,8 @@ components: parent_workspace_id: type: string nullable: true + is_dev_workspace: + type: boolean created_by: type: string nullable: true @@ -27766,6 +27935,7 @@ components: - name - username - color + - is_dev_workspace - disabled required: - email @@ -27807,6 +27977,18 @@ components: new_dbname: type: string description: "New database name for the fork" + is_dev_workspace: + type: boolean + description: "Create the fork as a persistent dev workspace (id not required to carry the wm-fork- prefix; at most one per parent)" + lock_prod_deploy: + type: boolean + description: "When creating a dev workspace, lock the parent (prod) against direct deployment" + lock_prod_forking: + type: boolean + description: "When creating a dev workspace, prevent forking the parent (prod)" + copy_members: + type: boolean + description: "Copy the parent's members (users + group memberships) into the fork so the team can work in it" required: - id - name @@ -29491,6 +29673,65 @@ components: - ducklake - datatable - volume + MaterializedPartition: + type: object + required: [asset_kind, asset_path, partition, status, materialized_at] + properties: + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + partition: + type: string + status: + type: string + enum: [running, materialized, failed] + snapshot_id: + type: integer + format: int64 + nullable: true + row_count: + type: integer + format: int64 + nullable: true + job_id: + type: string + format: uuid + nullable: true + materialized_at: + type: string + format: date-time + error: + type: string + nullable: true + AssetSchemaVersion: + type: object + required: [version, columns, captured_at] + properties: + version: + type: integer + format: int64 + columns: + type: array + items: + type: object + required: [name, type] + properties: + name: + type: string + type: + type: string + snapshot_id: + type: integer + format: int64 + nullable: true + job_id: + type: string + format: uuid + nullable: true + captured_at: + type: string + format: date-time Asset: type: object properties: diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 2a0c34ae8d..2184c1548f 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -382,12 +382,18 @@ pub fn build_query_string( .map(|value| { // Use the original name for the query parameter key let original_name = get_original_name(param_name, query_field_renames); - let value_str = value.to_string(); - let str_val = value_str.trim_matches('"'); + // For string values, use the raw content: to_string() would JSON-encode + // it, and stripping the outer quotes leaves inner quotes backslash-escaped + // (e.g. `{\"k\":\"v\"}`), which breaks downstream JSON parsing of params + // like `args`/`result`. Non-string values keep their JSON serialization. + let str_val = value + .as_str() + .map(|s| s.to_string()) + .unwrap_or_else(|| value.to_string()); format!( "{}={}", urlencoding::encode(&original_name), - urlencoding::encode(str_val) + urlencoding::encode(&str_val) ) }) }) @@ -624,4 +630,55 @@ mod tests { .expect("legitimate path should substitute"); assert_eq!(result, "/w/dev/scripts/get/p/u/alice/my_script"); } + + fn single_query_schema(param: &str) -> Option { + Some(json!({ + "type": "object", + "properties": { param: { "type": "string" } } + })) + } + + #[test] + fn build_query_string_preserves_json_string_content() { + // A string param carrying JSON (e.g. the `args` filter on listJobs) must be + // emitted as its raw content so the backend can `serde_json::from_str` it. + let mut args = serde_json::Map::new(); + args.insert("args".to_string(), json!("{\"key\":\"val\"}")); + + let qs = build_query_string(&args, &single_query_schema("args"), &None); + + // No backslash escaping: %5C must not appear; the encoded braces/quotes are exact. + assert_eq!(qs, "?args=%7B%22key%22%3A%22val%22%7D"); + assert!( + !qs.contains("%5C"), + "must not contain backslash escapes: {qs}" + ); + } + + #[test] + fn build_query_string_keeps_non_string_serialization() { + let mut args = serde_json::Map::new(); + args.insert("per_page".to_string(), json!(42)); + assert_eq!( + build_query_string(&args, &single_query_schema("per_page"), &None), + "?per_page=42" + ); + + let mut args = serde_json::Map::new(); + args.insert("running".to_string(), json!(true)); + assert_eq!( + build_query_string(&args, &single_query_schema("running"), &None), + "?running=true" + ); + } + + #[test] + fn build_query_string_encodes_plain_string() { + let mut args = serde_json::Map::new(); + args.insert("path".to_string(), json!("u/alice/my script")); + assert_eq!( + build_query_string(&args, &single_query_schema("path"), &None), + "?path=u%2Falice%2Fmy%20script" + ); + } } diff --git a/backend/windmill-common/src/ee_oss.rs b/backend/windmill-common/src/ee_oss.rs index e68bc00c6e..26b9c87b7e 100644 --- a/backend/windmill-common/src/ee_oss.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -20,6 +20,7 @@ lazy_static::lazy_static! { pub static ref LICENSE_KEY: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); pub static ref LICENSE_OFFLINE_METADATA: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref LICENSE_OFFLINE_OVER_CU_CAP: AtomicBool = AtomicBool::new(false); + pub static ref LICENSE_OFFLINE_OVER_SEAT_CAP: AtomicBool = AtomicBool::new(false); pub static ref LICENSE_OFFLINE_LAST_STATUS: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref LICENSE_OFFLINE_LAST_CHECKED_AT: arc_swap::ArcSwap>> = arc_swap::ArcSwap::from_pointee(None); } @@ -62,6 +63,14 @@ pub async fn check_seat_cap_for_new_user( Ok(None) } +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn check_seat_cap_for_reactivation( + _db: &DB, + _email: &str, +) -> anyhow::Result> { + Ok(None) +} + #[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn compute_instance_hash(_db: &DB) -> anyhow::Result> { // Implementation is not open source diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index be336a84e1..aaead77925 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -564,6 +564,132 @@ mod validate_dbname_tests { } } +#[cfg(test)] +mod pg_tls_tests { + use super::PgDatabase; + + // A syntactically valid (self-signed) certificate, used only to exercise the + // "root certificate supplied" branch — its contents are never validated here. + const VALID_PEM: &str = "-----BEGIN CERTIFICATE-----\n\ +MIIDETCCAfmgAwIBAgIUX/yHsMoWBljFzJr5Xh7V2I6ykMEwDQYJKoZIhvcNAQEL\n\ +BQAwGDEWMBQGA1UEAwwNd2luZG1pbGwtdGVzdDAeFw0yNjA2MjkwOTUwNTlaFw0z\n\ +NjA2MjYwOTUwNTlaMBgxFjAUBgNVBAMMDXdpbmRtaWxsLXRlc3QwggEiMA0GCSqG\n\ +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvF2hMw8adQGG6EnDk8GsOIoHT+kLN1W0F\n\ +yYFwH1wGVmzVP1YNfUts8aQfMtl/ZjW7SQlvKeK+18id4fVNYvZpbFhj66IsKMOU\n\ +MnJHcC6X/IAdhANyhM1fcrS6YupanAKOhLPk4HYRD5tGI4Y1vzTnQKGffIZ0bof7\n\ +3GtCiJLv8wrJKszeoKPtdFazdW+CYePbFq3Owc7HMo8CwA7A5TsgcowELhCfYwZv\n\ +Pn/9v+NDHQO0jJclH7qK221RkbqZGD+nPJ4rUm7oRi0vfApBQZ0FFJZjiki/Kg2+\n\ +RACb6Ud/LOeRBerKQHbN8KeYnGafCaIC4s/XytVwxAz+kgK1qyl7AgMBAAGjUzBR\n\ +MB0GA1UdDgQWBBRo2Jby4SZlrwMNbhA4bswZcBNRyjAfBgNVHSMEGDAWgBRo2Jby\n\ +4SZlrwMNbhA4bswZcBNRyjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\n\ +A4IBAQBlED+FQW3GB3Wa1NdVN252vihuFNnbq81yvhf4T7dfAxwkxI9jiM+ZWCw2\n\ +g59FbLupj8Rwun5gE2H/9M8ZunISdlwaMH5nyDJlbRjttPfY1cEoyGEY+UXIslfg\n\ +BoiI5rOtz9R2qurxEic1VtEVfXhEuWwCG86vCBDdHrL/qqqUJEx/P8qyC7uVc8XC\n\ +uclnJVL7x1ax0jTmEPur9K+DQn2ws01mzpq2QwSunibpDL5D5xM1oYekv0tQFEkT\n\ +ta9ELulniZau8zUAtwqwecxodzl+KO8NYj0a9PGgAM64dMqkRtRA8P4UP350Nag3\n\ ++hOq1qpWD7yPVyycx/KCilICOKVf\n\ +-----END CERTIFICATE-----\n"; + + fn pg(sslmode: Option<&str>, root_cert: Option<&str>) -> PgDatabase { + PgDatabase { + host: "db.example.com".to_string(), + user: Some("u".to_string()), + password: Some("p".to_string()), + port: Some(5432), + sslmode: sslmode.map(|s| s.to_string()), + dbname: "mydb".to_string(), + root_certificate_pem: root_cert.map(|s| s.to_string()), + accept_invalid_certs: None, + use_iam_auth: None, + region: None, + } + } + + /// Whether the connector enforces certificate verification for the given config. + fn verifies( + sslmode: Option<&str>, + root_cert: Option<&str>, + accept_invalid_certs: Option, + ) -> bool { + let mut builder = native_tls::TlsConnector::builder(); + PgDatabase::configure_pg_tls_verification( + &mut builder, + sslmode, + root_cert, + accept_invalid_certs, + ) + .unwrap() + } + + #[test] + fn verify_modes_enforce_verification_when_explicitly_requested() { + // accept_invalid_certs=Some(false) is what newly created resources carry: it + // verifies even with no custom cert (against the OS trust store). + assert!(verifies(Some("verify-full"), None, Some(false))); + assert!(verifies(Some("verify-ca"), None, Some(false))); + assert!(verifies(Some("verify-full"), Some(""), Some(false))); + assert!(verifies(Some("verify-full"), Some(VALID_PEM), Some(false))); + assert!(verifies(Some("verify-ca"), Some(VALID_PEM), Some(false))); + } + + #[test] + fn verify_modes_unset_fall_back_to_legacy_behavior() { + // Unset (None): verify iff a root cert is present — preserves the behavior of + // resources that predate the flag (incl. git-synced), so upgrades don't break. + assert!(!verifies(Some("verify-full"), None, None)); + assert!(!verifies(Some("verify-ca"), None, None)); + assert!(!verifies(Some("verify-full"), Some(""), None)); + assert!(verifies(Some("verify-full"), Some(VALID_PEM), None)); + assert!(verifies(Some("verify-ca"), Some(VALID_PEM), None)); + } + + #[test] + fn accept_invalid_certs_true_disables_verification_for_verify_modes() { + assert!(!verifies(Some("verify-full"), Some(VALID_PEM), Some(true))); + assert!(!verifies(Some("verify-ca"), None, Some(true))); + } + + #[test] + fn accept_invalid_certs_is_ignored_outside_verify_modes() { + // require never consults the flag: it verifies iff a cert is present, and + // encrypts-without-verifying otherwise, regardless of accept_invalid_certs. + assert!(!verifies(Some("require"), None, Some(false))); + assert!(!verifies(Some("require"), None, Some(true))); + assert!(!verifies(None, None, Some(true))); + assert!(verifies(Some("require"), Some(VALID_PEM), Some(true))); + assert!(verifies(Some("require"), Some(VALID_PEM), None)); + } + + #[test] + fn invalid_pem_is_rejected() { + let mut builder = native_tls::TlsConnector::builder(); + let err = PgDatabase::configure_pg_tls_verification( + &mut builder, + Some("verify-full"), + Some("not a certificate"), + Some(false), + ); + assert!(err.is_err()); + } + + #[test] + fn to_uri_collapses_verify_modes_for_tokio_postgres() { + // to_uri() feeds tokio-postgres, which only parses disable/prefer/require; + // verify-* therefore map to require there (verification is connector-driven). + for mode in ["require", "verify-ca", "verify-full"] { + assert!( + pg(Some(mode), None).to_uri().contains("sslmode=require"), + "{mode} should map to sslmode=require in to_uri()" + ); + } + assert!(pg(Some("disable"), None) + .to_uri() + .contains("sslmode=disable")); + assert!(pg(Some("allow"), None).to_uri().contains("sslmode=prefer")); + assert!(pg(None, None).to_uri().contains("sslmode=prefer")); + } +} + #[derive(Serialize, Debug)] pub struct PrepareQueryColumnInfo { pub name: String, @@ -588,6 +714,12 @@ pub struct PgDatabase { pub sslmode: Option, pub dbname: String, pub root_certificate_pem: Option, + /// Only meaningful for sslmode verify-ca/verify-full. `Some(true)` accepts any + /// server certificate (no chain or hostname check); `Some(false)` enforces + /// verification. `None` falls back to legacy behavior — verify only when a root + /// certificate is present — so resources that predate this flag (including + /// git-synced ones, whose source never sets it) keep working unchanged. + pub accept_invalid_certs: Option, pub use_iam_auth: Option, pub region: Option, } @@ -676,10 +808,94 @@ impl PgDatabase { } } + /// True when sslmode requests verification (verify-ca/verify-full) but the + /// effective configuration disables it, so the server's identity is not + /// checked. Mirrors the verify-* decision in `configure_pg_tls_verification`. + pub fn verify_mode_skips_verification(&self) -> bool { + matches!( + self.sslmode.as_deref(), + Some("verify-ca") | Some("verify-full") + ) && self.accept_invalid_certs.unwrap_or( + self.root_certificate_pem + .as_deref() + .unwrap_or("") + .is_empty(), + ) + } + + /// Configure certificate and hostname verification on a native-tls connector + /// according to the requested Postgres `sslmode`. The crates.io tokio-postgres + /// build only parses disable/prefer/require, so verify-ca and verify-full are + /// enforced here, on the connector, rather than through the connection URI. + /// + /// verify-full — verify the certificate chain AND that it matches the host. + /// verify-ca — verify the chain only; libpq does not check the hostname. + /// require / other — encrypt without verifying identity, unless a root + /// certificate is supplied (then verify the chain). + /// + /// `accept_invalid_certs` only applies to verify-ca/verify-full: `Some(true)` + /// accepts any certificate, `Some(false)` enforces verification, and `None` + /// falls back to the legacy behavior — verify only when a root certificate is + /// present — so resources predating the flag (including git-synced ones, whose + /// source never sets it) keep working unchanged. Verification uses the OS trust + /// store plus any supplied root certificate. Returns false when the connector + /// was set to accept any certificate, so callers can surface that an unverified + /// connection is being made. + fn configure_pg_tls_verification( + builder: &mut native_tls::TlsConnectorBuilder, + sslmode: Option<&str>, + root_certificate_pem: Option<&str>, + accept_invalid_certs: Option, + ) -> Result { + use native_tls::Certificate; + + let custom_root = match root_certificate_pem { + Some(pem) if !pem.is_empty() => Some( + Certificate::from_pem(pem.as_bytes()) + .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, + ), + _ => None, + }; + + match sslmode { + Some("verify-full") | Some("verify-ca") => { + // Unset falls back to the legacy behavior: verify iff a cert is present. + if accept_invalid_certs.unwrap_or(custom_root.is_none()) { + builder + .danger_accept_invalid_certs(true) + .danger_accept_invalid_hostnames(true); + return Ok(false); + } + if let Some(cert) = custom_root { + builder.add_root_certificate(cert); + } + if sslmode == Some("verify-ca") { + // verify-ca verifies the chain but, per libpq, not the hostname. + builder.danger_accept_invalid_hostnames(true); + } + Ok(true) + } + _ => { + // "require": accept_invalid_certs does not apply. Encrypt but do not + // verify identity, unless an explicit root certificate was supplied + // (then verify the chain). + if let Some(cert) = custom_root { + builder.add_root_certificate(cert); + Ok(true) + } else { + builder + .danger_accept_invalid_certs(true) + .danger_accept_invalid_hostnames(true); + Ok(false) + } + } + } + } + async fn connect_inner( &self, ) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { - use native_tls::{Certificate, TlsConnector}; + use native_tls::TlsConnector; use postgres_native_tls::MakeTlsConnector; use tokio_postgres::tls::NoTls; let ssl_mode_is_require = matches!( @@ -690,21 +906,17 @@ impl PgDatabase { if ssl_mode_is_require { tracing::info!("Creating new connection"); let mut connector = TlsConnector::builder(); - if let Some(root_certificate_pem) = &self.root_certificate_pem { - if !root_certificate_pem.is_empty() { - connector.add_root_certificate( - Certificate::from_pem(root_certificate_pem.as_bytes()).map_err(|e| { - error::Error::BadConfig(format!("Invalid Certs: {e:#}")) - })?, - ); - } else { - connector.danger_accept_invalid_certs(true); - connector.danger_accept_invalid_hostnames(true); - } - } else { - connector - .danger_accept_invalid_certs(true) - .danger_accept_invalid_hostnames(true); + Self::configure_pg_tls_verification( + &mut connector, + self.sslmode.as_deref(), + self.root_certificate_pem.as_deref(), + self.accept_invalid_certs, + )?; + if self.verify_mode_skips_verification() { + tracing::warn!( + "Postgres connection with sslmode={} is not verifying the server certificate (accept_invalid_certs is set, or no root certificate is configured and the resource predates that flag). Set accept_invalid_certs=false or provide root_certificate_pem to enforce verification.", + self.sslmode.as_deref().unwrap_or("") + ); } let (client, connection) = tokio::time::timeout( @@ -759,23 +971,16 @@ impl PgDatabase { error::Error::InternalErr(format!("IAM token generation failed: {e:#}")) })?; - // RDS IAM auth requires SSL + // RDS IAM auth requires SSL. let mut connector = TlsConnector::builder(); - if let Some(root_certificate_pem) = &self.root_certificate_pem { - if !root_certificate_pem.is_empty() { - connector.add_root_certificate( - native_tls::Certificate::from_pem(root_certificate_pem.as_bytes()) - .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, - ); - } else { - connector.danger_accept_invalid_certs(true); - connector.danger_accept_invalid_hostnames(true); - } - } else { - tracing::warn!("IAM RDS auth without root certificate: TLS certificate verification is disabled. Consider providing root_certificate_pem for production use."); - connector - .danger_accept_invalid_certs(true) - .danger_accept_invalid_hostnames(true); + let verified = Self::configure_pg_tls_verification( + &mut connector, + self.sslmode.as_deref(), + self.root_certificate_pem.as_deref(), + self.accept_invalid_certs, + )?; + if !verified { + tracing::warn!("IAM RDS auth without certificate verification: TLS certificate verification is disabled. Provide root_certificate_pem (and set sslmode=verify-full) to enforce verification."); } tracing::info!("Creating new IAM RDS connection to {}", &self.host); @@ -840,6 +1045,7 @@ impl PgDatabase { dbname, sslmode, root_certificate_pem: None, + accept_invalid_certs: None, use_iam_auth: None, region: None, }) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 09195bfea2..f0113aea5c 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -184,7 +184,37 @@ pub const WM_FORK_PREFIX: &str = "wm-fork-"; /// layer because the actual branch creation runs in a deferred git-sync worker job — without /// this check, the API returns 200 and the failure only surfaces later in the worker. pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> { - if !id.starts_with(WM_FORK_PREFIX) { + validate_workspace_branch_id(id, true) +} + +/// Like [`validate_fork_workspace_id`] but does not require the `wm-fork-` prefix. Used for dev +/// workspaces, whose id is an ordinary (prefix-less) workspace id but must still be git-branch-safe +/// because it is interpolated into a `wm-fork//` branch name like any fork. +pub fn validate_dev_workspace_id(id: &str) -> error::Result<()> { + validate_workspace_branch_id(id, false) +} + +/// The `workspace.name` column is `character varying(50)`, so a name longer than 50 characters +/// triggers a raw `value too long for type character varying(50)` SQL error on insert. Validate +/// up front to return a clear message instead. +pub fn validate_workspace_name(name: &str) -> error::Result<()> { + if name.chars().count() > 50 { + return Err(Error::BadRequest(format!( + "Workspace name is too long ({} chars). Maximum length is 50 characters.", + name.chars().count() + ))); + } + Ok(()) +} + +fn validate_workspace_branch_id(id: &str, require_fork_prefix: bool) -> error::Result<()> { + if id.is_empty() { + return Err(Error::BadRequest( + "Workspace id cannot be empty".to_string(), + )); + } + + if require_fork_prefix && !id.starts_with(WM_FORK_PREFIX) { return Err(Error::BadRequest(format!( "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", id, WM_FORK_PREFIX @@ -193,8 +223,9 @@ pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> { if id.len() > 50 { return Err(Error::BadRequest(format!( - "Fork workspace id `{}` is too long ({} chars). Maximum length is 50 characters (including the '{}' prefix).", - id, id.len(), WM_FORK_PREFIX + "Workspace id `{}` is too long ({} chars). Maximum length is 50 characters.", + id, + id.len() ))); } @@ -558,6 +589,11 @@ pub enum RuleCheckResult { /// - User is not in bypass lists /// /// Returns `Err` if the rule is not found +/// Reserved protection-rule name applied to a prod workspace paired with a dev workspace. It carries +/// `DisableDirectDeployment` + `DisableWorkspaceForking` and is auto-managed by the dev-workspace +/// feature (applied on pairing, removed on detach). +pub const DEV_WORKSPACE_LOCK_RULE_NAME: &str = "dev_workspace_lock"; + pub async fn check_user_against_rule( workspace_id: &str, rule: &ProtectionRuleKind, @@ -915,6 +951,30 @@ mod tests { assert!(validate_fork_workspace_id("wm-fork-foo/bar.lock").is_err()); } + #[test] + fn test_validate_dev_workspace_id_accepts_prefixless_valid() { + // Dev workspaces use ordinary, prefix-less ids but must stay git-branch-safe. + validate_dev_workspace_id("dev").unwrap(); + validate_dev_workspace_id("my-dev-workspace").unwrap(); + validate_dev_workspace_id("staging.42").unwrap(); + // The fork prefix is allowed but not required. + validate_dev_workspace_id("wm-fork-dev").unwrap(); + } + + #[test] + fn test_validate_dev_workspace_id_rejects_empty_and_git_unsafe() { + assert!(validate_dev_workspace_id("").is_err()); + assert!(validate_dev_workspace_id("dev workspace").is_err()); + assert!(validate_dev_workspace_id("dev..staging").is_err()); + assert!(validate_dev_workspace_id("dev/.x").is_err()); + assert!(validate_dev_workspace_id("dev.lock").is_err()); + } + + #[test] + fn test_validate_fork_workspace_id_rejects_empty() { + assert!(validate_fork_workspace_id("").is_err()); + } + #[test] fn test_validate_fork_workspace_id_rejects_too_long() { let long_id = format!("wm-fork-{}", "a".repeat(43)); diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cf30efe116..553eb69917 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -494,6 +494,23 @@ fn build_azure_blob_client( return Ok(Arc::new(store)); } +/// Whether a GCS `service_account_key` carries no static credentials, in which case the client +/// should fall back to the instance's ambient credentials (GKE Workload Identity / metadata server) +/// instead of being handed an unparseable key. Besides an empty/whitespace string, the settings UI +/// stores "no key" as an empty JSON object `{}` (and `serde_json` may yield `null`), so treat those +/// as absent too. Shared with the connectivity-test SSRF guard so both agree on what "no key" means. +pub fn gcs_service_account_key_is_blank(service_account_key: &str) -> bool { + let trimmed = service_account_key.trim(); + if trimmed.is_empty() { + return true; + } + match serde_json::from_str::(trimmed) { + Ok(serde_json::Value::Null) => true, + Ok(serde_json::Value::Object(map)) => map.is_empty(), + _ => false, + } +} + #[cfg(feature = "parquet")] async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result> { let gcs_resource = gcs_resource_ref.clone(); @@ -509,7 +526,12 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result (parent_workspace_id, cached_at). - // `parent_workspace_id` is essentially immutable once a fork is created, so a multi-minute TTL - // is safe. `None` means the lookup found no parent (or the DB call failed); we still cache it - // briefly so that forks missing a parent do not hammer the DB. + // `parent_workspace_id` is stable for the lifetime of a fork EXCEPT across attach/detach of a + // dev workspace, which set/keep it; those paths call `invalidate_fork_parent_cache` so routing + // doesn't lag. `None` means the lookup found no parent (or the DB call failed); we still cache + // it briefly so that forks missing a parent do not hammer the DB. static ref FORK_PARENT_CACHE: quick_cache::sync::Cache, std::time::Instant)> = quick_cache::sync::Cache::new(500); } +/// Drop the cached fork->parent mapping for a workspace. Call after mutating `parent_workspace_id` +/// (attaching/detaching a dev workspace) so per-workspace job tags resolve to the new parent +/// immediately instead of after the cache TTL. +pub fn invalidate_fork_parent_cache(workspace_id: &str) { + FORK_PARENT_CACHE.remove(workspace_id); +} + /// Returns `Some(effective_workspace_tag_id)` if jobs of `workspace_id` should use workspace- /// specific tags, where `effective_workspace_tag_id` is the string embedded in the tag. For forks, /// this is always the parent workspace id, optionally suffixed with `-fork` (controlled by the @@ -26,16 +34,14 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool) -> Optio return None; } - let is_fork = workspace_id.starts_with(WM_FORK_PREFIX); - - // For forks, always resolve to the parent workspace id; regular workspaces avoid the lookup. - let effective_ws_id: String = if is_fork { - lookup_fork_parent(workspace_id, db) - .await - .unwrap_or_else(|| workspace_id.to_string()) // no parent found -> fall back to fork's own id - } else { - workspace_id.to_string() - }; + // Resolve to the parent workspace id when the workspace is a fork or dev workspace (both set + // parent_workspace_id). The lookup caches its `None` result, so non-forks stay cheap after warmup + // (and the common case is already short-circuited by the global toggle above). + let parent = lookup_fork_parent(workspace_id, db).await; + // A `wm-fork-` workspace can outlive its parent (the FK is `ON DELETE SET NULL`), so keep + // treating the prefix as fork-ness for the `-fork` suffix even when the parent link is gone. + let is_fork = parent.is_some() || workspace_id.starts_with(WM_FORK_PREFIX); + let effective_ws_id: String = parent.unwrap_or_else(|| workspace_id.to_string()); // Whitelist check is against the resolved (parent) id so that including a parent in the // whitelist transparently covers all of its forks. @@ -62,8 +68,10 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool) -> Optio }) } -/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set (or the -/// DB lookup failed). Backed by a short-TTL cache to avoid a DB round-trip per job push. +/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set. Backed by a +/// short-TTL cache to avoid a DB round-trip per job push. A transient DB error returns `None` for +/// this call but is NOT cached, so the next push retries instead of misrouting a (prefix-less) dev +/// workspace's jobs for the whole TTL. async fn lookup_fork_parent(fork_id: &str, db: &Pool) -> Option { if let Some((parent, cached_at)) = FORK_PARENT_CACHE.get(fork_id) { if cached_at.elapsed().as_secs() < FORK_PARENT_CACHE_TTL_SECS { @@ -78,8 +86,11 @@ async fn lookup_fork_parent(fork_id: &str, db: &Pool) -> Option Some(parent), - _ => None, + Ok(opt) => opt.flatten(), + Err(e) => { + tracing::warn!("failed to look up fork parent for {fork_id}: {e:#}"); + return None; + } }; FORK_PARENT_CACHE.insert( diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index b9cea536f3..ad00567639 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -1088,21 +1088,48 @@ async fn create_resource( .execute(&db) .await?; } - sqlx::query!( - "INSERT INTO resource - (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) - VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", - w_id, - resource.path, - raw_json as sqlx::types::Json<&RawValue>, - resource.description, - resource.resource_type, - authed.username, - resource.labels.as_deref() as Option<&[String]> - ) - .execute(&mut *tx) - .await?; + if update_if_exists { + sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) + VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username, + resource.labels.as_deref() as Option<&[String]> + ) + .execute(&mut *tx) + .await?; + } else { + // Create-only (the default): DO NOTHING + a row-count guard, so a path that appears between + // check_path_conflict above and this insert is rejected rather than overwritten. A plain + // DO UPDATE here would clobber a concurrently-created resource, breaking create-only callers + // (e.g. Compare & Deploy "Create in "). + let inserted = sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) + VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username, + resource.labels.as_deref() as Option<&[String]> + ) + .execute(&mut *tx) + .await?; + if inserted.rows_affected() == 0 { + return Err(Error::BadRequest(format!( + "Resource {} already exists", + resource.path + ))); + } + } // Mirror update_resource: Some(true) inserts, Some(false) clears (only // meaningful on the upsert path, since a pure create has no existing row), @@ -1385,7 +1412,12 @@ fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { } } -async fn mark_linked_variables_ws_specific( +/// Marks every variable referenced by the resource at `resource_path` as workspace-specific. +/// +/// AUTH CONTRACT: this mutates `ws_specific` and does NOT check authorization itself. The caller +/// MUST verify that `authed` has write access to the resource at `resource_path` in `w_id` (e.g. via +/// `require_owner_of_path`) before calling it. +pub async fn mark_linked_variables_ws_specific( tx: &mut Transaction<'_, Postgres>, authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 2758fe85ff..292246c62c 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -938,6 +938,143 @@ remote_tmp={job_dir}/.ansible/tmp Ok(()) } +/// Read a colon-separated path list (e.g. `roles_path`, `collections_path`) from +/// the `[defaults]` section of an ansible.cfg. Returns the raw entries as written, +/// unresolved. Deliberately minimal: no inline-comment or continuation handling, +/// which ansible's configparser also does not apply to these values. `key` and `=` +/// or `:` as the delimiter are both matched (Python configparser accepts either), +/// with the value itself split on `:` (`os.pathsep`). +fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> { + let mut in_defaults = false; + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('[') && trimmed.ends_with(']') { + in_defaults = trimmed[1..trimmed.len() - 1] + .trim() + .eq_ignore_ascii_case("defaults"); + continue; + } + if !in_defaults || trimmed.starts_with('#') || trimmed.starts_with(';') { + continue; + } + // configparser delimits key/value on the first `=` or `:`, whichever + // comes first; the remaining `:` in the value are path separators. + let sep = trimmed.find('=').into_iter().chain(trimmed.find(':')).min(); + if let Some(sep) = sep { + let (k, rest) = trimmed.split_at(sep); + let v = &rest[1..]; + if k.trim().eq_ignore_ascii_case(key) { + let entries: Vec = v + .split(':') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + return (!entries.is_empty()).then_some(entries); + } + } + } + None +} + +/// Prepend Windmill's dependency install dir to the repo cfg's declared path list. +/// Relative entries from the repo cfg are resolved against `cfg_dir` to match how +/// ansible resolves them relative to the config file's own directory. +fn resolve_and_prepend_path( + base: String, + repo_entries: Option>, + cfg_dir: &str, +) -> String { + let mut paths = vec![base]; + if let Some(entries) = repo_entries { + for e in entries { + if e.starts_with('/') || e.starts_with('~') { + paths.push(e); + } else { + paths.push(format!("{cfg_dir}/{e}")); + } + } + } + paths.join(":") +} + +/// Build the environment overrides applied when delegating to a git repo that +/// ships its own ansible.cfg. See the call site for the layering rationale. +async fn build_ansible_cfg_override_envs( + cfg_path: &str, + job_dir: &str, + vault_password_file_exists: bool, + reqs: Option<&AnsibleRequirements>, +) -> error::Result> { + let mut envs = vec![ + ("ANSIBLE_CONFIG".to_string(), cfg_path.to_string()), + // Runtime-bound: reference the ephemeral job dir, cannot be set statically. + ("ANSIBLE_HOME".to_string(), format!("{job_dir}/.ansible")), + ( + "ANSIBLE_LOCAL_TEMP".to_string(), + format!("{job_dir}/.ansible/tmp"), + ), + ( + "ANSIBLE_REMOTE_TEMP".to_string(), + format!("{job_dir}/.ansible/tmp"), + ), + ]; + + // Vault: Windmill manages the secret, so its config wins over the repo cfg. + if vault_password_file_exists { + envs.push(( + "ANSIBLE_VAULT_PASSWORD_FILE".to_string(), + format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}"), + )); + } + if let Some(vault_ids) = reqs.map(|r| &r.vault_id).filter(|v| !v.is_empty()) { + // Defense in depth: entries are validated at parse time, but re-check + // here since they are interpolated raw into the env value. + for vault_id in vault_ids { + validate_vault_id(vault_id)?; + } + envs.push(( + "ANSIBLE_VAULT_IDENTITY_LIST".to_string(), + vault_ids.join(","), + )); + } + + // Dependency search paths: additive. Windmill installs galaxy roles into + // `{job_dir}/roles` and collections into `{job_dir}`; prepend those to the + // repo cfg's declared paths so both Windmill-installed and repo deps resolve. + let cfg_dir = std::path::Path::new(cfg_path) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or(job_dir); + let cfg_content = tokio::fs::read_to_string(cfg_path).await.map_err(|e| { + windmill_common::error::Error::internal_err(format!( + "Failed to read delegated ansible.cfg at `{cfg_path}`: {e}" + )) + })?; + + envs.push(( + "ANSIBLE_ROLES_PATH".to_string(), + resolve_and_prepend_path( + format!("{job_dir}/roles"), + parse_ansible_cfg_path_list(&cfg_content, "roles_path"), + cfg_dir, + ), + )); + envs.push(( + "ANSIBLE_COLLECTIONS_PATH".to_string(), + resolve_and_prepend_path( + job_dir.to_string(), + // Also probe the deprecated plural ini alias; env vars replace (not + // merge) the cfg value, so a repo using it would otherwise be dropped. + parse_ansible_cfg_path_list(&cfg_content, "collections_path") + .or_else(|| parse_ansible_cfg_path_list(&cfg_content, "collections_paths")), + cfg_dir, + ), + )); + + Ok(envs) +} + pub async fn get_git_ssh_cmd( reqs: &AnsibleRequirements, job_dir: &str, @@ -1136,6 +1273,9 @@ pub async fn handle_ansible_job( let mut nsjail_extra_mounts = vec![]; let mut playbook_override = None; + // Absolute path of a repo-provided `ansible.cfg` to use as the effective + // config, set when `delegate_to_git_repo.ansible_cfg` is provided. + let mut ansible_config_override: Option = None; if let Some(r) = reqs.as_ref() { nsjail_extra_mounts = create_file_resources( @@ -1301,6 +1441,23 @@ pub async fn handle_ansible_job( inventories.push(format!("{}/{}", &repo.target_path, inv)); } + if let Some(cfg_rel) = delegated_git_repo.ansible_cfg.as_ref() { + let cfg_rel = interpolate_template( + cfg_rel, + interpolated_args.as_ref(), + "delegate_to_git_repo.ansible_cfg", + )?; + validate_relative_path(&cfg_rel, "delegate_to_git_repo.ansible_cfg")?; + let cfg_path = format!("{}/{}/{}", job_dir, &repo.target_path, cfg_rel); + if !tokio::fs::try_exists(&cfg_path).await.unwrap_or(false) { + return Err(windmill_common::error::Error::BadRequest(format!( + "delegate_to_git_repo.ansible_cfg: no ansible.cfg found in the cloned repo at `{}/{}`", + &repo.target_path, cfg_rel + ))); + } + ansible_config_override = Some(cfg_path); + } + if delegated_git_repo.install_requirements { install_requirements_from_cloned_repo( &repo.target_path, @@ -1450,6 +1607,32 @@ pub async fn handle_ansible_job( create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?; + // When the run delegates to a git repo that ships its own ansible.cfg, that + // file becomes the effective config (ansible loads exactly one config file and + // does not merge). These env vars layer Windmill's runtime-bound settings back + // on top — env vars outrank ansible.cfg. Only applied on the non-sandboxed + // path: git-repo delegation clones into `job_dir` which the nsjail profile does + // not mount, so it already requires DISABLE_NSJAIL. + let ansible_env_overrides = match ansible_config_override.as_ref() { + Some(cfg_path) => { + if is_sandboxing_enabled() { + tracing::warn!( + "delegate_to_git_repo.ansible_cfg is set but sandboxing is enabled; \ + git-repo delegation requires DISABLE_NSJAIL, the ansible.cfg override \ + will not take effect" + ); + } + build_ansible_cfg_override_envs( + cfg_path, + job_dir, + vault_password_file_exists, + reqs.as_ref(), + ) + .await? + } + None => vec![], + }; + let mut reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let additional_python_paths_folders = additional_python_paths.join(":"); @@ -1560,6 +1743,7 @@ fi .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) + .envs(ansible_env_overrides) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1856,4 +2040,230 @@ mod tests { assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err()); assert!(!dir.path().join("ansible.cfg").exists()); } + + #[test] + fn test_parse_ansible_cfg_path_list() { + let cfg = "\ +[defaults] +roles_path = roles:extra/roles +collections_path=/opt/collections +host_key_checking = False + +[inventory] +roles_path = ignored/section +"; + assert_eq!( + parse_ansible_cfg_path_list(cfg, "roles_path"), + Some(vec!["roles".to_string(), "extra/roles".to_string()]) + ); + assert_eq!( + parse_ansible_cfg_path_list(cfg, "collections_path"), + Some(vec!["/opt/collections".to_string()]) + ); + // Keys only in another section are not picked up. + assert_eq!(parse_ansible_cfg_path_list(cfg, "library"), None); + } + + #[test] + fn test_parse_ansible_cfg_path_list_ignores_comments() { + let cfg = "\ +[defaults] +# roles_path = commented +; roles_path = also_commented +"; + assert_eq!(parse_ansible_cfg_path_list(cfg, "roles_path"), None); + } + + #[test] + fn test_parse_ansible_cfg_path_list_colon_delimiter() { + // configparser accepts `:` as a key/value delimiter, and the value can + // itself be a `:`-separated list. + let cfg = "\ +[defaults] +roles_path: my_roles +collections_path : a/col:b/col +"; + assert_eq!( + parse_ansible_cfg_path_list(cfg, "roles_path"), + Some(vec!["my_roles".to_string()]) + ); + assert_eq!( + parse_ansible_cfg_path_list(cfg, "collections_path"), + Some(vec!["a/col".to_string(), "b/col".to_string()]) + ); + } + + #[test] + fn test_resolve_and_prepend_path() { + // No repo entries: only Windmill's install dir. + assert_eq!( + resolve_and_prepend_path("/job/roles".to_string(), None, "/job/repo"), + "/job/roles" + ); + // Relative repo entries resolve against the cfg dir; absolute/~ kept as-is. + assert_eq!( + resolve_and_prepend_path( + "/job/roles".to_string(), + Some(vec![ + "roles".to_string(), + "/abs/roles".to_string(), + "~/r".to_string() + ]), + "/job/repo/config" + ), + "/job/roles:/job/repo/config/roles:/abs/roles:~/r" + ); + } + + fn ansible_playbook_available() -> bool { + std::process::Command::new("ansible-playbook") + .arg("--version") + .output() + .is_ok() + } + + /// End-to-end: with a delegated repo that ships its own `ansible.cfg` pointing + /// `roles_path` at an in-repo directory, the override env vars must make the + /// real `ansible-playbook` resolve a role it otherwise cannot. Requires the + /// `ansible-playbook` binary; self-skips when absent (e.g. standard CI). Run on + /// a worker devbox with `cargo test -p windmill-worker --features python`. + #[tokio::test] + async fn test_ansible_cfg_override_resolves_repo_roles_e2e() { + if !ansible_playbook_available() { + eprintln!( + "SKIP test_ansible_cfg_override_resolves_repo_roles_e2e: ansible-playbook not found on PATH" + ); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo = dir.path().join(DELEGATE_GIT_REPO_TARGET); + let role_tasks = repo.join("my_roles/greet/tasks"); + std::fs::create_dir_all(&role_tasks).unwrap(); + + std::fs::write( + repo.join("ansible.cfg"), + "[defaults]\nroles_path = my_roles\n", + ) + .unwrap(); + std::fs::write( + role_tasks.join("main.yml"), + "- debug:\n msg: \"hello from greet role\"\n", + ) + .unwrap(); + let play = "- hosts: localhost\n connection: local\n gather_facts: false\n roles:\n - greet\n"; + std::fs::write(repo.join("play.yml"), play).unwrap(); + + // Windmill's own generated cfg (the negative-control config that exists today). + create_ansible_cfg(None, job_dir, false).unwrap(); + + let playbook = format!("{DELEGATE_GIT_REPO_TARGET}/play.yml"); + let run = |envs: Vec<(String, String)>| { + std::process::Command::new("ansible-playbook") + .arg(&playbook) + .current_dir(job_dir) + .envs(envs) + .output() + .unwrap() + }; + + // Negative control: today's behavior (Windmill cfg via cwd, no override) — + // the role lives in the repo subdir and is not found. + let cfg_path = repo.join("ansible.cfg"); + let before = run(vec![]); + assert!( + !before.status.success(), + "without the override the repo role must NOT resolve; stdout={}", + String::from_utf8_lossy(&before.stdout) + ); + + // With the override: ANSIBLE_CONFIG points at the repo cfg and roles_path + // is honored, so the role runs. + let envs = + build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) + .await + .unwrap(); + let after = run(envs); + let stdout = String::from_utf8_lossy(&after.stdout); + assert!( + after.status.success() && stdout.contains("hello from greet role"), + "with the override the repo role must resolve; status={:?} stdout={stdout} stderr={}", + after.status, + String::from_utf8_lossy(&after.stderr) + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join("delegate_git_repository"); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + std::fs::write(&cfg_path, "[defaults]\nroles_path = my_roles\n").unwrap(); + let cfg_path = cfg_path.to_str().unwrap(); + + let reqs = AnsibleRequirements { + vault_id: vec!["dev@vault_pass.txt".to_string()], + ..Default::default() + }; + let envs = build_ansible_cfg_override_envs(cfg_path, job_dir, true, Some(&reqs)) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + + assert_eq!( + map.get("ANSIBLE_CONFIG").map(|s| s.as_str()), + Some(cfg_path) + ); + assert_eq!( + map.get("ANSIBLE_HOME"), + Some(&format!("{job_dir}/.ansible")) + ); + assert_eq!( + map.get("ANSIBLE_VAULT_PASSWORD_FILE"), + Some(&format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}")) + ); + assert_eq!( + map.get("ANSIBLE_VAULT_IDENTITY_LIST").map(|s| s.as_str()), + Some("dev@vault_pass.txt") + ); + // Windmill's `{job_dir}/roles` is prepended to the repo cfg's own path, + // which is resolved against the cfg directory. + assert_eq!( + map.get("ANSIBLE_ROLES_PATH"), + Some(&format!( + "{job_dir}/roles:{}/my_roles", + repo_dir.to_str().unwrap() + )) + ); + // No collections_path in the repo cfg → only Windmill's job dir. + assert_eq!( + map.get("ANSIBLE_COLLECTIONS_PATH").map(|s| s.as_str()), + Some(job_dir) + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_collections_paths_alias() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join("delegate_git_repository"); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + // Deprecated plural alias must still be picked up so the repo's collections + // are not silently dropped when the env override replaces the cfg value. + std::fs::write(&cfg_path, "[defaults]\ncollections_paths = my_cols\n").unwrap(); + + let envs = + build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + assert_eq!( + map.get("ANSIBLE_COLLECTIONS_PATH"), + Some(&format!("{job_dir}:{}/my_cols", repo_dir.to_str().unwrap())) + ); + } } diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e4b94acabf..a932be102d 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -195,7 +195,21 @@ fn build_materialized_query( for s in plan.setup.iter_mut() { *s = substitute(s); } - let strategy = if m.append { + let strategy = if m.scd2 { + // SCD2 needs a natural key to identify an entity across versions, and its + // diff/close/open shape has no partition-scoped form in v1. + let key = m.unique_key.clone().ok_or_else(|| { + Error::ExecutionErr( + "materialize scd2: requires a natural key — add `key=`".to_string(), + ) + })?; + if partitioned { + return Err(Error::ExecutionErr( + "materialize scd2: `// partitioned` is not supported with scd2 in v1".to_string(), + )); + } + MaterializeStrategy::Scd2 { key, track: m.track.clone(), close_deleted: m.close_deleted } + } else if m.append { MaterializeStrategy::Append } else if let Some(uk) = m.unique_key { MaterializeStrategy::Merge { unique_key: uk } @@ -1520,6 +1534,88 @@ mod tests { assert!(!rewritten.contains("-- $file")); } + // SCD2 managed mode wraps the SELECT into the diff → close-old → open-new + // shape (unit-covered in the parser's codegen tests); here we pin the + // executor-level wiring: the natural key flows through and the wrap is + // generated (not the manual track-only path). + #[test] + fn materialize_scd2_wraps_with_history() { + // Primary spelling: `key=history` on a merge. + let script = "-- materialize ducklake://main/dim key=id history track=name\n\ + SELECT id, name FROM dl.src"; + let (rewritten, _) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("scd2 is managed — must rewrite"); + assert!( + rewritten.contains("valid_from"), + "adds SCD2 columns:\n{rewritten}" + ); + assert!(rewritten.contains("is_current")); + assert!( + rewritten.contains("_wm_scd2_changed"), + "captures changed keys" + ); + assert!( + rewritten.contains("UPDATE _wm_target.dim SET valid_to"), + "closes prior version" + ); + assert!( + rewritten.contains("CREATE VIEW IF NOT EXISTS _wm_target.dim_current"), + "emits the consumer-convenience current view" + ); + // default is soft-delete — no deleted-key set without `deletes=close` + assert!(!rewritten.contains("_wm_scd2_deleted")); + assert!(!rewritten.contains("MERGE INTO")); + } + + #[test] + fn materialize_scd2_deletes_close_wires_through() { + let script = "-- materialize ducklake://main/dim key=id history deletes=close\n\ + SELECT id, name FROM dl.src"; + let (rewritten, _) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("scd2 is managed — must rewrite"); + assert!( + rewritten.contains("_wm_scd2_deleted"), + "deletes=close adds the vanished-key set + close:\n{rewritten}" + ); + } + + #[test] + fn materialize_scd2_requires_key() { + let script = "-- materialize scd2 ducklake://main/dim\nSELECT id, name FROM dl.src"; + let err = match build_materialized_query(script, None, &std::collections::HashMap::new()) { + Err(e) => e, + Ok(_) => panic!("scd2 without key must error"), + }; + assert!( + format!("{err}").contains("requires a natural key"), + "got: {err}" + ); + } + + #[test] + fn materialize_scd2_rejects_partitioned() { + let script = "-- pipeline\n-- partitioned daily\n\ + -- materialize scd2 ducklake://main/dim key=id\nSELECT id, name FROM dl.src"; + let err = match build_materialized_query( + script, + Some("2026-07-01"), + &std::collections::HashMap::new(), + ) { + Err(e) => e, + Ok(_) => panic!("partitioned + scd2 must error"), + }; + assert!( + format!("{err}").contains("not supported with scd2"), + "got: {err}" + ); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index a1a26d0630..98c1b5bba7 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -615,6 +615,25 @@ pub async fn do_postgresql( return Err(Error::BadRequest("Missing database argument".to_string())); }; + // Surface in the job logs (not just the worker logs) when a verify-ca/verify-full + // resource is connecting without actually verifying the server certificate, so the + // person running the query can see and fix the misconfiguration. + if database.verify_mode_skips_verification() { + windmill_queue::append_logs( + &job.id, + &job.workspace_id, + format!( + "warning: sslmode={} but the server's TLS certificate is not being verified \ + (accept_invalid_certs is enabled, or no root certificate is configured). Set \ + accept_invalid_certs to false or provide root_certificate_pem to verify the \ + server identity.\n", + database.sslmode.as_deref().unwrap_or("") + ), + conn, + ) + .await; + } + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let collection_strategy = if annotations.raw_output || annotations.return_last_result { // raw_output emits a single envelope from the last statement, so the @@ -629,10 +648,26 @@ pub async fn do_postgresql( // Include use_iam_auth in cache key to distinguish IAM vs non-IAM connections to the same host. // The cache key is static (doesn't include the token), which is correct because PostgreSQL // connections remain valid after initial auth — fresh tokens are generated on cache miss. + // + // to_uri() collapses require/verify-ca/verify-full to the same string, so the TLS verification + // inputs are folded into the key separately. Without this a connection established under a + // weaker sslmode (or a different root cert) could be reused for a stricter request, undoing + // the verification configured in PgDatabase::configure_pg_tls_verification. + let tls_disc = { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + database.sslmode.hash(&mut h); + database.root_certificate_pem.hash(&mut h); + database.accept_invalid_certs.hash(&mut h); + h.finish() + }; + // to_uri() already ends with `?sslmode=...`, so append further key segments + // with `&` to keep database_string a well-formed URI (it is only ever a cache + // key, but a malformed one would mislead anyone who later logs or parses it). let database_string = if use_iam_auth { - format!("{}?iam=true", database.to_uri()) + format!("{}&iam=true&tls={tls_disc:x}", database.to_uri()) } else { - database.to_uri() + format!("{}&tls={tls_disc:x}", database.to_uri()) }; let database_string_clone = database_string.clone(); diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f8cc748bdb..d1dad43038 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.742.0"; +export const VERSION = "v1.744.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bun.lock b/cli/bun.lock index c94551beb3..50bcc48222 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -19,6 +19,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -288,6 +289,8 @@ "utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="], + "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.740.0", "", {}, "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw=="], + "windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="], "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.510.1", "", {}, "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ=="], diff --git a/cli/package-lock.json b/cli/package-lock.json index c502b3b25e..949b44e948 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -20,6 +20,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -1409,6 +1410,11 @@ "node": ">= 4" } }, + "node_modules/windmill-parser-wasm-asset": { + "version": "1.740.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.740.0.tgz", + "integrity": "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw==" + }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-csharp/-/windmill-parser-wasm-csharp-1.510.1.tgz", diff --git a/cli/package.json b/cli/package.json index 2f46e3829b..1cf937d95e 100644 --- a/cli/package.json +++ b/cli/package.json @@ -28,6 +28,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", diff --git a/cli/src/commands/pipeline/boundedCascade.ts b/cli/src/commands/pipeline/boundedCascade.ts index 27380d799a..d928a72b64 100644 --- a/cli/src/commands/pipeline/boundedCascade.ts +++ b/cli/src/commands/pipeline/boundedCascade.ts @@ -40,10 +40,14 @@ export const isScriptNode = (id: string): boolean => id.startsWith(SCRIPT_PREFIX export const scriptPathOf = (id: string): string => id.slice(SCRIPT_PREFIX.length); const assetNodeId = (kind: string, path: string): string => `${kind}:${path}`; -// Native trigger kinds that fan out per-event — never bounded-run starts. -// `webhook` / `data_upload` have no trigger row in the graph payload, so a root -// whose only entry is one of those reads as a manual root. -const EVENT_TRIGGER_KINDS = new Set([ +// Native trigger kinds whose scripts must NOT be auto-run by the CLI cascade: +// event triggers fan out per external event, and `webhook`/`data_upload` are UI +// entrypoints that need caller-supplied input (a request body / an uploaded +// S3Object) — previewing any of them with empty args runs the wrong thing. +// The deployed graph omits `webhook`/`data_upload` rows, but the local graph +// emits them, so a `// on data_upload` script would otherwise read as a manual +// root and get auto-run without its upload argument. +const NON_AUTORUN_TRIGGER_KINDS = new Set([ "kafka", "mqtt", "nats", @@ -51,6 +55,8 @@ const EVENT_TRIGGER_KINDS = new Set([ "sqs", "gcp", "email", + "webhook", + "data_upload", ]); /** Resolve an asset URI (`datatable://x`, `s3://b/k`, …) to its node id. */ @@ -121,6 +127,38 @@ function closure(adj: Map>, start: string): Set { export const descendants = (dag: LineageDag, n: string): Set => closure(dag.down, n); export const ancestors = (dag: LineageDag, n: string): Set => closure(dag.up, n); +/** + * Nodes reachable from `starts` over the lineage DAG, treating `barriers` as cut + * points: a barrier node is neither included NOR traversed through. So a node + * reachable ONLY via a barrier is excluded, while one also reachable via another + * path stays. Used for whole-pipeline runs to keep event handlers AND their + * event-only downstream closure out (subtracting only the handler would leave a + * consumer whose producer was skipped, which topoOrder would then run with + * missing/stale inputs). + */ +export function reachableCutting( + dag: LineageDag, + starts: Iterable, + barriers: Set, +): Set { + const seen = new Set(); + const queue: string[] = []; + for (const s of starts) { + if (barriers.has(s) || seen.has(s)) continue; + seen.add(s); + queue.push(s); + } + while (queue.length > 0) { + const n = queue.shift()!; + for (const next of dag.down.get(n) ?? []) { + if (barriers.has(next) || seen.has(next)) continue; + seen.add(next); + queue.push(next); + } + } + return seen; +} + export type BoundedResult = { nodes: Set; reachableEnds: string[]; @@ -151,19 +189,35 @@ export function boundedSet(dag: LineageDag, start: string, ends: string[]): Boun export function validStarts(g: BCGraph): Set { const subscribers = new Set(); const scheduleScripts = new Set(); - const eventScripts = new Set(); + const nonAutorunScripts = new Set(); for (const t of g.triggers ?? []) { if (t.runnable_kind !== "script") continue; if (t.trigger_kind === "asset") subscribers.add(t.runnable_path); else if (t.trigger_kind === "schedule") scheduleScripts.add(t.runnable_path); - else if (EVENT_TRIGGER_KINDS.has(t.trigger_kind)) eventScripts.add(t.runnable_path); + else if (NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) nonAutorunScripts.add(t.runnable_path); } const out = new Set(); for (const r of g.runnables ?? []) { if (r.usage_kind !== "script") continue; const p = r.path; if (scheduleScripts.has(p)) out.add(scriptNodeId(p)); - else if (!subscribers.has(p) && !eventScripts.has(p)) out.add(scriptNodeId(p)); + else if (!subscribers.has(p) && !nonAutorunScripts.has(p)) out.add(scriptNodeId(p)); + } + return out; +} + +/** + * Script node ids that carry a trigger requiring caller-supplied input or + * per-event fanout (kafka/mqtt/nats/postgres/sqs/gcp/email/webhook/data_upload). + * These can't be run with empty args, so a whole-pipeline run must exclude them + * even when they're a lineage descendant of a valid start (not just a root). + */ +export function nonAutorunTriggerScripts(g: BCGraph): Set { + const out = new Set(); + for (const t of g.triggers ?? []) { + if (t.runnable_kind === "script" && NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) { + out.add(scriptNodeId(t.runnable_path)); + } } return out; } diff --git a/cli/src/commands/pipeline/dev.ts b/cli/src/commands/pipeline/dev.ts new file mode 100644 index 0000000000..3d11dfc2bd --- /dev/null +++ b/cli/src/commands/pipeline/dev.ts @@ -0,0 +1,290 @@ +// `wmill pipeline dev [folder]` — live-preview a data pipeline from local files. +// +// The pipeline analog of `wmill dev` / `wmill app dev`: watch a folder of +// `// pipeline` scripts, rebuild the asset graph from the working tree on every +// save, and push it over a WebSocket to the `/pipeline_dev` page, which renders +// the same graph editor the UI uses and runs the cascade via preview (no deploy). +// Editing stays in the user's own editor; the page live-reloads. + +import { Command } from "@cliffy/command"; +import { colors } from "@cliffy/ansi/colors"; +import * as http from "node:http"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { createHash, randomBytes } from "node:crypto"; +import process from "node:process"; +import * as open from "open"; +import { WebSocket, WebSocketServer } from "ws"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { + mergeConfigWithConfigFile, + type SyncOptions, +} from "../../core/conf.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; +import { resolveBindPort } from "../../utils/port-probe.ts"; +import { getConfigDirPath } from "../../../windmill-utils-internal/src/config/config.ts"; +import { buildLocalPipelineGraph, workspaceRoot } from "./localGraph.ts"; + +const PORT = 3201; +// Bind loopback only: each WS frame ships the folder's full script source +// (`scripts[].content` + `temp_script_refs`) with no auth, so it must not be +// reachable from the LAN. The webview connects via `ws://localhost`, and an SSH +// `-L` / devbox port-forward targets 127.0.0.1 on the host, so both still work. +const LISTEN_HOST = "127.0.0.1"; + +interface PipelineDevOpts extends GlobalOptions, SyncOptions { + port?: number; + open?: boolean; + defaultTs?: "bun" | "deno"; + frontend?: string; +} + +// The WS token gates access to local source. Persist it under the user-private +// config dir (0600) so a `pipeline dev` restart reuses it — an already-open +// `/pipeline_dev` page auto-reconnecting with the token from its URL then +// survives the restart, instead of every upgrade being rejected by +// `verifyClient` until the freshly printed URL is reopened. Scoped by +// remote+workspace+root+folder+port (NOT port alone): a same-session restart +// reconnects, but any *different* session — another folder, workspace, remote, or +// local checkout on the same port — gets a different token, so a stale tab can't +// reconnect and receive another session's source. Falls back to an ephemeral +// token if the config dir can't be read/written. +async function stableWsToken( + remote: string, + workspaceId: string, + root: string, + folder: string, + port: number, +): Promise { + // Hash the tuple (NUL-delimited so no value can spoof the boundary) → a + // collision-resistant, filesystem-safe key. A plain sanitized join would let + // different values collide (`a/b` and `a_b` → same name), which would reuse a + // token across sessions and reintroduce the cross-session leak. + const key = createHash("sha256") + .update(`${remote}\0${workspaceId}\0${root}\0${folder}\0${port}`) + .digest("hex") + .slice(0, 32); + let tokenFile: string | undefined; + try { + tokenFile = path.join(await getConfigDirPath(), `pipeline-dev-${key}.token`); + const existing = fs.readFileSync(tokenFile, "utf-8").trim(); + if (existing) return existing; + } catch { + // no reusable token file yet (or config dir unavailable) — mint a fresh one + } + const token = randomBytes(24).toString("base64url"); + if (tokenFile) { + try { + fs.writeFileSync(tokenFile, token, { mode: 0o600 }); + } catch { + // best-effort persistence; fall back to the in-memory token + } + } + return token; +} + +// Resolve the target folder: explicit arg, else auto-detect when cwd sits inside +// `f//…` (the supported `cd f/my_pipeline && wmill pipeline dev` form). +function resolveFolder(root: string, folderArg?: string): string | undefined { + if (folderArg) return folderArg.replace(/^f\//, "").replace(/\/$/, ""); + const rel = path.relative(root, process.cwd()).replaceAll("\\", "/"); + if (rel === "" || rel.startsWith("..")) return undefined; + const segs = rel.split("/"); + if (segs[0] === "f" && segs[1]) return segs[1]; + return undefined; +} + +async function dev(opts: PipelineDevOpts, folderArg?: string) { + const root = workspaceRoot(); + const folder = resolveFolder(root, folderArg); + if (!folder) { + log.error( + colors.red( + "Could not determine the pipeline folder. Pass it explicitly " + + "(`wmill pipeline dev `) or run from inside an `f/` directory.", + ), + ); + process.exit(1); + } + const folderDir = path.join(root, "f", folder); + if (!fs.existsSync(folderDir)) { + log.error(colors.red(`Folder not found on disk: ${folderDir}`)); + process.exit(1); + } + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const merged = await mergeConfigWithConfigFile(opts); + const codebases = await listSyncCodebases(merged); + + // Resolve relative imports from local (not-yet-deployed) content for previews. + // Snapshot at startup like `wmill dev`; restart to refresh. Degrades to + // undefined on older backends. + let tempScriptRefs: Record | undefined; + try { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + merged, + codebases, + { kind: "all" }, + ); + } catch { + // best-effort + } + + const EMPTY_GRAPH = { runnables: [], assets: [], edges: [], triggers: [] }; + async function buildBundle() { + const { graph, scripts } = await buildLocalPipelineGraph({ + root, + folder: folder!, + defaultTs: merged.defaultTs, + }); + return { + type: "pipeline" as const, + folder, + graph, + scripts, + temp_script_refs: tempScriptRefs, + }; + } + + // Don't let a transient build error (e.g. a half-written file) abort startup — + // serve an empty graph and recover on the next save. + let current: Awaited>; + try { + current = await buildBundle(); + } catch (e: any) { + log.error(colors.red(`Initial graph build failed: ${e.message}`)); + current = { type: "pipeline", folder, graph: EMPTY_GRAPH, scripts: [], temp_script_refs: tempScriptRefs }; + } + log.info( + colors.blue( + `Watching f/${folder} — ${current.scripts.length} pipeline script(s)`, + ), + ); + + const clients = new Set(); + function broadcast() { + const msg = JSON.stringify(current); + for (const ws of clients) { + if (ws.readyState === WebSocket.OPEN) ws.send(msg); + } + } + + // Debounced rebuild on any change under the folder. + let timer: ReturnType | undefined; + const watcher = fs.watch(folderDir, { recursive: true }); + watcher.on("change", (_ev, filename) => { + if (filename && filename.toString().endsWith(".lock")) return; + if (timer) clearTimeout(timer); + timer = setTimeout(async () => { + timer = undefined; + try { + current = await buildBundle(); + log.info(colors.cyan(`↻ rebuilt graph (${current.scripts.length} scripts)`)); + broadcast(); + } catch (e: any) { + log.error(colors.red(`Failed to rebuild pipeline graph: ${e.message}`)); + } + }, 150); + }); + watcher.on("error", (e) => log.error(colors.red(`Watcher error: ${e.message}`))); + + const port = await resolveBindPort(opts.port ?? PORT, "wmill pipeline dev", { + info: (m) => log.info(m), + warn: (m) => log.warn(m), + }); + + const server = http.createServer((_req, res) => { + res.writeHead(200); + res.end(); + }); + // Loopback bind keeps the LAN out, but any browser tab can still open a + // `ws://localhost:` connection — and each frame ships the folder's full + // script source. Gate the upgrade on an unguessable per-session token (carried + // in the dev-page URL) so a stray page on the predictable dev port can't + // exfiltrate the source. base64url → safe as a query value. + // Stable across restarts (scoped to remote+workspace+root+folder+port) so an + // already-open page reconnects after a CLI restart (see stableWsToken), not + // just after a transient WS drop. + const wsToken = await stableWsToken( + workspace.remote, + workspace.workspaceId, + root, + folder, + port, + ); + const wss = new WebSocketServer({ + server, + verifyClient: (info) => { + try { + const u = new URL(info.req.url ?? "", "http://localhost"); + return u.searchParams.get("token") === wsToken; + } catch { + return false; + } + }, + }); + wss.on("connection", (ws: WebSocket) => { + clients.add(ws); + // Push the current bundle immediately so the page renders without waiting + // for the first file change. + try { + ws.send(JSON.stringify(current)); + } catch { + // ignore + } + ws.on("close", () => clients.delete(ws)); + ws.on("error", () => clients.delete(ws)); + }); + + // The `/pipeline_dev` page is served by the frontend, not the backend. By + // default we open it on the workspace remote, but that 404s on a remote whose + // deployed frontend predates this route — `--frontend` points the page at a + // locally-run frontend (`REMOTE= npm run dev`) while the API/token + // still target the remote. Normalize to a single trailing slash either way. + const pageBase = (opts.frontend ?? workspace.remote).replace(/\/?$/, "/"); + const url = + `${pageBase}pipeline_dev?workspace=${workspace.workspaceId}` + + `&wm_token=${workspace.token}&folder=${encodeURIComponent(folder)}&port=${port}` + + `&ws_token=${encodeURIComponent(wsToken)}`; + + server.listen(port, LISTEN_HOST, () => { + log.info(colors.green.bold(`🚀 Pipeline dev server on ws://localhost:${port}/ws`)); + log.info(colors.gray(`Open: ${url}`)); + if (opts.open !== false) { + open.default(url).catch((e: any) => + log.warn(colors.yellow(`Failed to open browser: ${e.message}`)), + ); + } + }); + + process.on("SIGINT", () => { + log.info(colors.yellow("\n🛑 Shutting down…")); + watcher.close(); + for (const ws of clients) ws.close(); + server.close(); + process.exit(0); + }); +} + +const command = new Command() + .description( + "Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).", + ) + .arguments("[folder:string]") + .option("--port ", "Port for the dev WebSocket server.") + .option("--no-open", "Do not open the browser automatically.") + .option( + "--frontend ", + "Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.", + ) + .action(dev as any); + +export default command; diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts new file mode 100644 index 0000000000..57fb70618a --- /dev/null +++ b/cli/src/commands/pipeline/docs.ts @@ -0,0 +1,242 @@ +// `wmill pipeline docs ` — generate a PIPELINE.md (+ AGENTS.md / CLAUDE.md +// pointer) describing a folder's pipeline so an editor or agentic loop has the +// same context the UI surfaces: the asset DAG, per-script triggers/IO, and the +// schemas of the datatables the pipeline touches. Mirrors the app docs pattern +// (`app/generate_agents.ts:regenerateAgentDocs`) but scoped to a pipeline folder. + +import { writeFile } from "node:fs/promises"; +import { existsSync, readFileSync } from "node:fs"; +import * as path from "node:path"; +import { OpenAPI } from "../../../gen/index.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as log from "../../core/log.ts"; +import { colors } from "@cliffy/ansi/colors"; +import { GlobalOptions } from "../../types.ts"; +import { + type AssetGraph, + buildLocalPipelineGraph, + workspaceRoot, +} from "./localGraph.ts"; + +const ASSET_KINDS = "s3object,ducklake,datatable,volume"; + +function assetUri(kind: string, p: string): string { + const prefix = kind === "s3object" ? "s3" : kind; + return `${prefix}://${p}`; +} + +async function fetchDeployedGraph( + workspaceId: string, + folder: string, +): Promise { + const res = await fetch( + `${OpenAPI.BASE}/w/${workspaceId}/assets/graph?folder=${encodeURIComponent(folder)}&asset_kinds=${ASSET_KINDS}`, + { headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` } }, + ); + if (!res.ok) { + throw new Error(`GET assets/graph -> ${res.status}: ${await res.text()}`); + } + return (await res.json()) as AssetGraph; +} + +// Render the pipeline graph as a markdown document. +function generatePipelineMarkdown( + folder: string, + graph: AssetGraph, + datatableSchemas: any[], + local: boolean, +): string { + const writesByScript = new Map(); + const readsByScript = new Map(); + for (const e of graph.edges) { + if (e.runnable_kind !== "script") continue; + const uri = assetUri(e.asset_kind, e.asset_path); + if (e.access_type === "w" || e.access_type === "rw") { + (writesByScript.get(e.runnable_path) ?? writesByScript.set(e.runnable_path, []).get(e.runnable_path)!).push(uri); + } + if (e.access_type === "r" || e.access_type === "rw" || e.access_type === undefined) { + (readsByScript.get(e.runnable_path) ?? readsByScript.set(e.runnable_path, []).get(e.runnable_path)!).push(uri); + } + } + const onByScript = new Map(); + const nativeByScript = new Map(); + for (const t of graph.triggers) { + if (t.runnable_kind !== "script") continue; + if (t.trigger_kind === "asset") { + const at = t as Extract; + (onByScript.get(at.runnable_path) ?? onByScript.set(at.runnable_path, []).get(at.runnable_path)!).push(assetUri(at.asset_kind, at.asset_path)); + } else { + (nativeByScript.get(t.runnable_path) ?? nativeByScript.set(t.runnable_path, []).get(t.runnable_path)!).push(t.trigger_kind); + } + } + + const scripts = graph.runnables.filter((r) => r.usage_kind === "script").map((r) => r.path).sort(); + + let md = `# Pipeline \`f/${folder}\` + +${local ? "_Built from local working-tree files (`// pipeline` scripts)._" : "_Built from the deployed workspace asset graph._"} + +A data pipeline is a folder of scripts marked \`// pipeline\` and wired by asset +annotations. A script subscribes to upstream data with \`// on \` and +produces data by reading/writing assets in its body (\`datatable://\`, +\`ducklake://\`, \`s3://\`, \`volume://\`). The cascade runs a producer, then every +downstream subscriber, in topological order. + +- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"} + +## Scripts + +`; + + for (const s of scripts) { + const on = onByScript.get(s) ?? []; + const native = nativeByScript.get(s) ?? []; + const writes = [...new Set(writesByScript.get(s) ?? [])].sort(); + const reads = [...new Set(readsByScript.get(s) ?? [])].sort(); + md += `### \`${s}\`\n\n`; + if (native.length) md += `- **Triggers:** ${native.map((n) => `\`${n}\``).join(", ")}\n`; + if (on.length) md += `- **On (subscribes to):** ${on.map((u) => `\`${u}\``).join(", ")}\n`; + if (reads.length) md += `- **Reads:** ${reads.map((u) => `\`${u}\``).join(", ")}\n`; + if (writes.length) md += `- **Writes:** ${writes.map((u) => `\`${u}\``).join(", ")}\n`; + if (!native.length && !on.length && !reads.length && !writes.length) { + md += `- _No declared triggers or asset IO._\n`; + } + md += `\n`; + } + + // Datatable schemas, restricted to datatables this pipeline references. + const referencedDatatables = new Set( + graph.assets.filter((a) => a.kind === "datatable").map((a) => a.path.split("/")[0]), + ); + const relevant = (datatableSchemas ?? []).filter((dt: any) => referencedDatatables.has(dt.datatable_name)); + if (relevant.length > 0) { + md += `## Datatable schemas\n\n`; + for (const dt of relevant) { + md += `### Datatable \`${dt.datatable_name}\`\n\n`; + if (dt.error) { + md += `> ⚠️ ${dt.error}\n\n`; + continue; + } + for (const [schemaName, tables] of Object.entries(dt.schemas ?? {})) { + for (const [tableName, columns] of Object.entries(tables as Record)) { + const ref = schemaName === "public" + ? `${dt.datatable_name}/${tableName}` + : `${dt.datatable_name}/${schemaName}:${tableName}`; + md += `- \`datatable://${ref}\`\n`; + for (const [col, type] of Object.entries(columns as Record)) { + md += ` - \`${col}\`: ${type}\n`; + } + } + } + md += `\n`; + } + } + + md += `## Working with this pipeline + +- **Inspect the graph:** \`wmill pipeline show ${folder} --local\` +- **Run the cascade locally (no deploy):** \`wmill pipeline run ${folder} --local\` + (optionally \`--from + +{#if isDev && parentId} +
+

+ This is a dev workspace paired with root workspace {parentId}. Promote changes + from the home page banner or the Compare & Deploy page. +

+
+ +
+
+{:else if pairedDev} +
+

+ This workspace's dev workspace is {pairedDev.name} ({pairedDev.id}). Edits to this + workspace are redirected there. +

+
+ {#if pairedDev.isMember} + + {/if} + +
+
+{:else if parentId} +

+ Dev workspace pairing is only available for root workspaces. This workspace is a fork of + {parentId}. +

+{:else} +
+

+ Pair this workspace with a dev workspace: the same code with a different environment (resource + and variable values). Edits are made in the dev workspace and promoted here. +

+
+ Attach an existing workspace as dev +
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. + +\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- \`// materialize append\` → INSERT-append rows (incremental); +- \`// materialize key=\` → merge/upsert on \`\`. + +\`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. + +\`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. +2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`), language chosen for the work: \`duckdb\` or \`postgresql\` for SQL-shaped data work, \`bun\`/\`python3\` for general transforms. SQL-heavy lakehouse steps usually use \`duckdb\`. +3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the \`/pipeline/\` editor open, prefer the dedicated \`build_pipeline_node\` / \`edit_pipeline_node\` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake table): + +\`\`\`sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +\`\`\` + +Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): + +\`\`\`sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +\`\`\` +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope @@ -3025,11 +3087,20 @@ inspect asset-driven pipelines (scripts marked \`// pipeline\`, wired by \`// on - \`--json\` - Output as JSON (for piping to jq) - \`pipeline show \` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal - \`--json\` - Output the raw asset graph as JSON + - \`--local\` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed. - \`pipeline run \` - run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s) - \`--from \` - Start script (short name or path). Defaults to the folder's sole schedule/manual root. - \`--to \` - End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream. - \`--dry-run\` - Print the topological run plan without executing. - \`--json\` - Output the plan as JSON (for piping to jq). + - \`--local\` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files. + - \`--upload \` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable. +- \`pipeline docs \` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop + - \`--local\` - Build the graph from local working-tree files instead of the deployed workspace. +- \`pipeline dev [folder:string]\` - Live-preview a data pipeline from local files: watch an \`f/\` of \`// pipeline\` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy). + - \`--port \` - Port for the dev WebSocket server. + - \`--no-open\` - Do not open the browser automatically. + - \`--frontend \` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page. ### protection-rules diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 6973aaa137..7c34e85706 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -424,11 +424,20 @@ inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal - `--json` - Output the raw asset graph as JSON + - `--local` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed. - `pipeline run ` - run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s) - `--from ` - Start script (short name or path). Defaults to the folder's sole schedule/manual root. - `--to ` - End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream. - `--dry-run` - Print the topological run plan without executing. - `--json` - Output the plan as JSON (for piping to jq). + - `--local` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files. + - `--upload ` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable. +- `pipeline docs ` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop + - `--local` - Build the graph from local working-tree files instead of the deployed workspace. +- `pipeline dev [folder:string]` - Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy). + - `--port ` - Port for the dev WebSocket server. + - `--no-open` - Do not open the browser automatically. + - `--frontend ` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page. ### protection-rules diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md new file mode 100644 index 0000000000..a0f2e2492a --- /dev/null +++ b/system_prompts/base/pipeline-base.md @@ -0,0 +1,60 @@ +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`. + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. +- **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. + +## Materialize (the managed output) + +> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. + +`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- `// materialize append` → INSERT-append rows (incremental); +- `// materialize key=` → merge/upsert on ``. + +`// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time. + +`materialize` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: `f//`. The folder is the pipeline. +2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`. +3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table): + +```sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +``` + +Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): + +```sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 6ee1971f43..98c03c4367 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -2369,6 +2369,7 @@ def main(): flow_base = read_markdown_file(base_dir / "flow-base.md") resources_base = read_markdown_file(base_dir / "resources.md") raw_app_base = read_markdown_file(base_dir / "raw-app.md") + pipeline_base = read_markdown_file(base_dir / "pipeline-base.md") workflow_as_code_base = read_markdown_file(base_dir / "workflow-as-code.md") flow_cli = read_markdown_file(base_dir / "flow-cli.md") flow_chat_special_modules = read_markdown_file(base_dir / "flow-chat-special-modules.md") @@ -2435,6 +2436,7 @@ def main(): 'FLOW_BASE': flow_base, 'RESOURCES_BASE': resources_base, 'RAW_APP_BASE': raw_app_base, + 'PIPELINE_BASE': pipeline_base, 'WORKFLOW_AS_CODE_BASE': workflow_as_code_base, 'FLOW_CHAT_SPECIAL_MODULES': flow_chat_special_modules, @@ -2534,6 +2536,11 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } +// Helper for data pipeline authoring (chat consumers) +export function getPipelinePrompt(): string { + return prompts.PIPELINE_BASE; +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2586,6 +2593,7 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; +export declare function getPipelinePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7ae2023a7c..ca1b6c98ae 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.742.0", + "version": "1.744.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8edc248c45..1b8efd04d8 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.742.0", + "version": "1.744.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index bee795f5f0..48207fbc7d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.742.0 +1.744.0