starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-19 16:02:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into explore-git-sync-improvements

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
This commit is contained in:
hugocasa
2026-07-01 16:40:12 +02:00
parent ab181519c3 289017bcb2
commit 60570ef9ef
206 changed files with 11446 additions and 2327 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+37
View File
@@ -1,5 +1,42 @@
# Changelog
## [1.744.0](https://github.com/windmill-labs/windmill/compare/v1.743.0...v1.744.0) (2026-07-01)
### Features
* add copy-to-clipboard button to rendered Mermaid diagrams in AI chat ([#9838](https://github.com/windmill-labs/windmill/issues/9838)) ([a27e814](https://github.com/windmill-labs/windmill/commit/a27e814a03c615259381eaf684aa90d56569b0af))
* add dev workspaces paired with a lockable prod workspace ([#9793](https://github.com/windmill-labs/windmill/issues/9793)) ([b4b0c6a](https://github.com/windmill-labs/windmill/commit/b4b0c6a93e52152251fadefe319773faf42549b2))
* **ansible:** support repo-provided ansible.cfg in delegate_to_git_repo ([#9851](https://github.com/windmill-labs/windmill/issues/9851)) ([68bf0da](https://github.com/windmill-labs/windmill/commit/68bf0daf5815307cda6ce23214dd5159b6aa33b4))
* **licensing:** enforce offline license seat cap ([#9845](https://github.com/windmill-labs/windmill/issues/9845)) ([83f3d7f](https://github.com/windmill-labs/windmill/commit/83f3d7f910b331c09f60cc9ff556728afa3dec07))
* **object-store:** make GCS service account key optional for Workload Identity ([#9842](https://github.com/windmill-labs/windmill/issues/9842)) ([83ed011](https://github.com/windmill-labs/windmill/commit/83ed011e264f20ffa66a7bf933f2fe3615cf6b67))
* **pipeline:** local development for data pipelines (CLI --local + pipeline dev preview) ([#9840](https://github.com/windmill-labs/windmill/issues/9840)) ([74f579e](https://github.com/windmill-labs/windmill/commit/74f579e6d9ef08e74460f904a4c22ed9d6a3b5b0))
* **pipelines:** add managed SCD2 history materialize strategy ([#9850](https://github.com/windmill-labs/windmill/issues/9850)) ([5a66127](https://github.com/windmill-labs/windmill/commit/5a661279a3690e2393b9b16996f5d1a5a509259c))
### Bug Fixes
* **ai-chat:** replay anthropic turns verbatim to keep thinking valid ([#9843](https://github.com/windmill-labs/windmill/issues/9843)) ([a37a144](https://github.com/windmill-labs/windmill/commit/a37a144e81cf6b3de935688a617e9d0e1756004a))
* grant dispatch_event table to windmill roles ([#9852](https://github.com/windmill-labs/windmill/issues/9852)) ([f05b50d](https://github.com/windmill-labs/windmill/commit/f05b50d29ac2fdbb808a97057fb92c8e425b4a2f))
* grant workspace_diff, materialized_partition, debounce_stale_data to windmill roles ([#9853](https://github.com/windmill-labs/windmill/issues/9853)) ([293647d](https://github.com/windmill-labs/windmill/commit/293647de4c13cb8468cbd81ff1924cba90e164b4))
* honor verify-ca/verify-full sslmode for postgres connections ([#9835](https://github.com/windmill-labs/windmill/issues/9835)) ([bf6be96](https://github.com/windmill-labs/windmill/commit/bf6be967fa8c74e1299cf63f813c1cfa34b97f3e))
* **mcp:** stop double-escaping string query params in build_query_string ([#9855](https://github.com/windmill-labs/windmill/issues/9855)) ([1c46f89](https://github.com/windmill-labs/windmill/commit/1c46f899ca03edf62053f4f14d65b4eabff4255d))
* **s3_proxy:** preserve URL-encoding on Hive-partition proxy writes ([#9848](https://github.com/windmill-labs/windmill/issues/9848)) ([6b79bdd](https://github.com/windmill-labs/windmill/commit/6b79bddd42fe55f891c17cb71a7e36ee31337bac))
* validate workspace name length (max 50 chars) on create and fork ([#9854](https://github.com/windmill-labs/windmill/issues/9854)) ([b52972d](https://github.com/windmill-labs/windmill/commit/b52972d0de89004e98d18241d238ca028e4eecba))
## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29)
### Features
* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005))
* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0))
### Bug Fixes
* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b))
## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28)
ai_evals/cases/global.yaml
+60
View File
@@ -1223,6 +1223,66 @@
- places it in team_a (writable by this non-admin user) and not team_b (read-only)
- leaves the result as a draft only
- id: global-test-pipeline-create-node
prompt: |-
Set up the first step of a data pipeline at `f/evals/global/orders_ingest`.
On a schedule, it should pull raw orders and land them in a managed DuckLake
table so later steps can build on it. Keep it as an AI draft only — don't
deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/orders_ingest
valueIncludes:
- pipeline
forbiddenDrafts:
- type: flow
pathStartsWith: f/evals/global/
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- write_flow
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- builds a data pipeline node as a script (not a flow)
- marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`)
- declares a schedule trigger and writes its output to a managed DuckLake table
- leaves the result as an AI draft and does not deploy or save it
- id: global-test-pipeline-two-node-chain
prompt: |-
Build a small data pipeline in the `f/evals/global` folder: one step that
ingests orders into a DuckLake table, and a second step that reads that table
and writes a daily order-count rollup table. Wire the second step to run off
the first step's output. Keep everything as drafts — don't deploy.
initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json
runtime:
maxTurns: 14
validate:
draftCountAtLeast: 2
forbiddenDrafts:
- type: flow
pathStartsWith: f/evals/global/
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- write_flow
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates two data pipeline nodes as scripts (not a flow) in f/evals/global
- both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`)
- the first ingests orders into a DuckLake table
- the second reads that same table and writes a daily rollup, wired to the first step's output asset
- leaves both as AI drafts without deploying
- id: global-path5-create-folder-then-draft
prompt: |-
Create a new shared folder called "analytics" for our data work, then draft a
backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) FROM del",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941"
}
backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json
Generated
-38
View File
@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n DELETE FROM asset\n WHERE (workspace_id, path, kind) IN (\n SELECT workspace_id, path, kind FROM (\n SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[], \n $2::varchar[], \n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id \n AND a.path = limits.path \n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"VarcharArray",
"VarcharArray",
{
"Custom": {
"name": "asset_kind[]",
"kind": {
"Array": {
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
}
}
}
}
}
},
"Int4Array"
]
},
"nullable": []
},
"hash": "02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0"
}
backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json → backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2",
"query": "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e"
"hash": "0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b"
}
backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json
Generated
+5 -5
View File
@@ -46,11 +46,11 @@
]
},
"nullable": [
false,
false,
false,
false,
false,
true,
true,
true,
true,
true,
true,
true
]
backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "x",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Uuid",
"Varchar",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346"
}
backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a"
}
backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\", is_dev_workspace\n FROM workspace WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_parent!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "is_dev_workspace",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
false
]
},
"hash": "16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531"
}
backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json
Generated
-27
View File
@@ -1,27 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "asset_usage_kind",
"kind": {
"Enum": [
"script",
"flow",
"job"
]
}
}
}
]
},
"nullable": []
},
"hash": "1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6"
}
backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, name)\n DO UPDATE SET rules = EXCLUDED.rules,\n bypass_groups = EXCLUDED.bypass_groups,\n bypass_users = EXCLUDED.bypass_users\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Int4",
"TextArray",
"TextArray"
]
},
"nullable": []
},
"hash": "1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242"
}
backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json
Generated
+4 -2
View File
@@ -38,7 +38,8 @@
"google",
"ci_test",
"github",
"azure"
"azure",
"asset"
]
}
}
@@ -75,7 +76,8 @@
"google",
"ci_test",
"github",
"azure"
"azure",
"asset"
]
}
}
backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) as \"c!\" FROM del",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "c!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4"
}
backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5\n FROM workspace WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee"
}
backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json → backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2",
"query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM ws_specific ws\n WHERE ws.path = workspace_diff.path\n AND ws.item_kind = workspace_diff.kind\n AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id)\n )",
"describe": {
"columns": [
{
@@ -55,5 +55,5 @@
true
]
},
"hash": "0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1"
"hash": "2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf"
}
backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'resource', $2::varchar\n WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1"
}
backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (parent_workspace_id IS NOT NULL) AS \"is_fork!\" FROM workspace WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "is_fork!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd"
}
backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ORDER BY path, created_at DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "content!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db"
}
backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca"
}
backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json → backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id)\n VALUES ($1, $2, $3, $4)",
"query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id, is_dev_workspace)\n VALUES ($1, $2, $3, $4, $5)",
"describe": {
"columns": [],
"parameters": {
@@ -8,10 +8,11 @@
"Varchar",
"Varchar",
"Varchar",
"Varchar"
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c"
"hash": "41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056"
}
backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json → backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ",
"query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ON CONFLICT DO NOTHING\n ",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078"
"hash": "42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7"
}
backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings)\n VALUES ($1, $2, $3)\n ON CONFLICT (hash)\n DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Int8",
"Int8"
]
},
"nullable": []
},
"hash": "451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a"
}
backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "WITH potential AS (\n SELECT email, operator FROM usr WHERE is_service_account IS false\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user\n WHERE email NOT IN (SELECT email FROM password WHERE disabled IS true)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "authors!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "operators!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44"
}
backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace SET is_dev_workspace = false WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a"
}
backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM v2_job WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f"
}
backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json → backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t",
"query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1 AND is_service_account IS false\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t",
"describe": {
"columns": [
{
@@ -18,5 +18,5 @@
null
]
},
"hash": "9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9"
"hash": "5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72"
}
backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json
Generated
-40
View File
@@ -1,40 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH mine AS (\n SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1\n ), claim_self AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE id = $1 AND consumed_at IS NULL\n RETURNING debounce_batch\n ), claim_rest AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)\n AND id <> $1 AND consumed_at IS NULL\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM mine) AS \"had_row!\",\n (SELECT debounce_batch FROM claim_self) AS claimed_batch,\n (SELECT consumed_by FROM mine) AS prev_consumed_by,\n ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "had_row!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "claimed_batch",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "prev_consumed_by",
"type_info": "Uuid"
},
{
"ordinal": 3,
"name": "claimed_ids!",
"type_info": "UuidArray"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null,
null,
null,
null
]
},
"hash": "57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89"
}
backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json → backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json
Generated
+9 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false",
"query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false",
"describe": {
"columns": [
{
@@ -30,11 +30,16 @@
},
{
"ordinal": 5,
"name": "is_dev_workspace",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "operator_settings",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"ordinal": 7,
"name": "disabled",
"type_info": "Bool"
}
@@ -50,9 +55,10 @@
false,
true,
true,
false,
null,
false
]
},
"hash": "c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6"
"hash": "63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19"
}
backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b"
}
backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'\n RETURNING 1\n ) SELECT count(*) FROM del",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18"
}
backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "runnable_path!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "kind!",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true,
null
]
},
"hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc"
}
backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\" FROM workspace WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_parent!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c"
}
backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523"
}
backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "parent_workspace_id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true
]
},
"hash": "7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376"
}
backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', '1970-01-01T00:00:00+00:00')\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23"
}
backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace SELECT $1, $2, owner, false, premium FROM workspace WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745"
}
backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "parent_workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "deleted",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
true,
false
]
},
"hash": "8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38"
}
backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "runnable_settings_handle",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true
]
},
"hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2"
}
backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec"
}
backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'variable', $2::varchar\n WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c"
}
backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT status = 'success' AS \"success!\"\n FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "success!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Uuid",
"Int8"
]
},
"nullable": [
null
]
},
"hash": "910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b"
}
backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0"
}
backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END\n FROM workspace WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d"
}
backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
},
"hash": "a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077"
}
backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Text",
"Varchar",
"Varchar",
"TextArray"
]
},
"nullable": []
},
"hash": "a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e"
}
backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b"
}
backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via\n FROM usr WHERE workspace_id = $2\n ON CONFLICT DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142"
}
backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_diff SET has_changes = NULL\n WHERE path = $2 AND kind = $3\n AND ($1 IN (source_workspace_id, fork_workspace_id))",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db"
}
backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(\n SELECT 1 FROM workspace\n WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace\n )",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de"
}
backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch, consumed_at) VALUES\n ($1, nextval('debounce_batch_seq'), now() - interval '20 minutes'),\n ($2, nextval('debounce_batch_seq'), now() - interval '1 minute'),\n ($3, nextval('debounce_batch_seq'), NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed"
}
backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>$2 FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266"
}
backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2\n AND NOT EXISTS (\n SELECT 1 FROM workspace\n WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5"
}
backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "is_admin",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996"
}
backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": [
false
]
},
"hash": "d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8"
}
backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json
Generated
-43
View File
@@ -1,43 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "kind!: AssetKind",
"type_info": {
"Custom": {
"name": "asset_kind",
"kind": {
"Enum": [
"s3object",
"resource",
"variable",
"ducklake",
"datatable",
"volume"
]
}
}
}
},
{
"ordinal": 1,
"name": "path!",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b"
}
backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Text",
"Varchar",
"Varchar",
"TextArray"
]
},
"nullable": []
},
"hash": "deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef"
}
backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json
Generated
-26
View File
@@ -1,26 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH potential AS (\n SELECT email, operator FROM usr\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "authors!",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "operators!",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455"
}
backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json
Generated
-38
View File
@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT status = 'success' AS \"success!\",\n result AS \"result: Json<Box<RawValue>>\",\n started_at AS \"started_at!\"FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "success!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "result: Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 2,
"name": "started_at!",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Uuid",
"Int8"
]
},
"nullable": [
null,
true,
true
]
},
"hash": "e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976"
}
backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT concurrency_settings, debouncing_settings FROM runnable_settings WHERE hash = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "concurrency_settings",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "debouncing_settings",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
true
]
},
"hash": "ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01"
}
backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Text",
"Varchar",
"Varchar",
"TextArray"
]
},
"nullable": []
},
"hash": "f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c"
}
backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d"
}
backend/Cargo.lock
Generated
+97 -96
View File
@@ -195,9 +195,9 @@ dependencies = [
[[package]]
name = "arc-swap"
version = "1.9.1"
version = "1.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207"
checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
dependencies = [
"rustversion",
]
@@ -775,9 +775,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
version = "1.17.0"
version = "1.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -785,14 +785,15 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
version = "0.41.0"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
dependencies = [
"cc",
"cmake",
"dunce",
"fs_extra",
"pkg-config",
]
[[package]]
@@ -2601,9 +2602,9 @@ dependencies = [
[[package]]
name = "curl-sys"
version = "0.4.89+curl-8.20.0"
version = "0.4.90+curl-8.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22"
checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897"
dependencies = [
"cc",
"libc",
@@ -6085,9 +6086,9 @@ dependencies = [
[[package]]
name = "io-uring"
version = "0.7.12"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62"
checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0"
dependencies = [
"bitflags 2.13.0",
"cfg-if",
@@ -9998,9 +9999,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
version = "1.14.1"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
dependencies = [
"web-time",
"zeroize",
@@ -12188,9 +12189,9 @@ dependencies = [
[[package]]
name = "time"
version = "0.3.51"
version = "0.3.53"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327"
checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
dependencies = [
"deranged",
"num-conv",
@@ -12208,9 +12209,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
[[package]]
name = "time-macros"
version = "0.2.30"
version = "0.2.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935"
checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
dependencies = [
"num-conv",
"time-core",
@@ -12922,9 +12923,9 @@ dependencies = [
[[package]]
name = "triomphe"
version = "0.1.15"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39"
checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae"
dependencies = [
"serde",
"stable_deref_trait",
@@ -13734,7 +13735,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-nats",
@@ -13817,7 +13818,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"async-stream",
"async-trait",
@@ -13850,7 +13851,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -13863,7 +13864,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"argon2",
@@ -14001,7 +14002,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14024,7 +14025,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14039,7 +14040,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14065,7 +14066,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -14075,7 +14076,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14092,7 +14093,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -14114,7 +14115,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14137,7 +14138,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14153,7 +14154,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14174,7 +14175,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14195,7 +14196,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14209,7 +14210,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14244,7 +14245,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14269,7 +14270,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"flate2",
@@ -14287,7 +14288,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14309,7 +14310,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14329,7 +14330,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14366,7 +14367,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14394,7 +14395,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"lazy_static",
"serde",
@@ -14406,7 +14407,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -14431,7 +14432,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14445,7 +14446,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14478,7 +14479,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"chrono",
"lazy_static",
@@ -14492,7 +14493,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -14511,7 +14512,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -14613,7 +14614,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"chrono",
"itertools 0.14.0",
@@ -14632,7 +14633,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"regex",
"serde",
@@ -14647,7 +14648,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -14671,7 +14672,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"futures",
@@ -14688,7 +14689,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -14704,7 +14705,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -14725,7 +14726,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -14756,7 +14757,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -14781,7 +14782,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-stream",
@@ -14815,7 +14816,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"futures",
@@ -14833,7 +14834,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -14842,7 +14843,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14854,7 +14855,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14866,7 +14867,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"gosyn",
@@ -14878,7 +14879,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14890,7 +14891,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14902,7 +14903,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -14913,7 +14914,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14924,7 +14925,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -14936,7 +14937,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -14947,7 +14948,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -14969,7 +14970,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -14981,7 +14982,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -14995,7 +14996,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15012,7 +15013,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15025,7 +15026,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde",
@@ -15037,7 +15038,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -15055,7 +15056,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -15071,7 +15072,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -15087,7 +15088,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde",
@@ -15098,7 +15099,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15137,7 +15138,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"const_format",
@@ -15176,7 +15177,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -15187,7 +15188,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -15221,7 +15222,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15245,7 +15246,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15278,7 +15279,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15311,7 +15312,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15331,7 +15332,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15365,7 +15366,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15401,7 +15402,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15424,7 +15425,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15448,7 +15449,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15472,7 +15473,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15507,7 +15508,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15535,7 +15536,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15560,7 +15561,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"bitflags 2.13.0",
@@ -15579,7 +15580,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -15689,7 +15690,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"bytes",
"futures",
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.742.0"
version = "1.744.0"
authors.workspace = true
edition.workspace = true
@@ -87,7 +87,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.742.0"
version = "1.744.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/migrations/20260624161218_dev_workspace.down.sql
+3
View File
@@ -0,0 +1,3 @@
ALTER TABLE workspace DROP CONSTRAINT IF EXISTS workspace_dev_requires_parent;
DROP INDEX IF EXISTS workspace_canonical_dev_idx;
ALTER TABLE workspace DROP COLUMN is_dev_workspace;
backend/migrations/20260624161218_dev_workspace.up.sql
+14
View File
@@ -0,0 +1,14 @@
-- A dev workspace is a fork (parent_workspace_id set) that is the standing editable
-- environment paired with its parent ("prod"), as opposed to a throwaway fork.
ALTER TABLE workspace ADD COLUMN is_dev_workspace BOOLEAN NOT NULL DEFAULT false;
-- At most one active canonical dev workspace per parent (one editable source per prod).
-- Excludes soft-deleted (archived) workspaces so a new dev can replace an archived one.
CREATE UNIQUE INDEX workspace_canonical_dev_idx ON workspace (parent_workspace_id)
WHERE is_dev_workspace AND deleted = false;
-- A dev workspace is a fork, so it must have a parent. Enforce the invariant at the schema level so
-- no path (or manual write) can persist a "root dev workspace". No backfill needed: the column is
-- added above with default false, so no existing row can violate this at creation time.
ALTER TABLE workspace ADD CONSTRAINT workspace_dev_requires_parent
CHECK (NOT is_dev_workspace OR parent_workspace_id IS NOT NULL);
backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql
+4
View File
@@ -0,0 +1,4 @@
REVOKE ALL ON dispatch_event FROM windmill_user;
REVOKE ALL ON dispatch_event FROM windmill_admin;
REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_user;
REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_admin;
backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql
+15
View File
@@ -0,0 +1,15 @@
-- The dispatch_event table (migration 20260523055641_dispatch_event) was
-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user
-- and windmill_admin. Those default privileges only apply to objects created by
-- the role that set them (migration 20250205131523), so deployments whose
-- migration runner is a different role leave dispatch_event ungranted. Direct
-- writes run as the invoking role -- the dispatcher insert (asset_dispatch.rs)
-- and the DELETE in delete_jobs (windmill-common/src/jobs.rs), reached whenever
-- a job's side rows are reaped, e.g. on schedule disable -- and fail with
-- "permission denied for table dispatch_event". Grant explicitly to guarantee
-- access regardless of who ran the migrations (same fix as notify_event in
-- 20260619091631 and script_trigger in 20260619112847).
GRANT ALL ON dispatch_event TO windmill_user;
GRANT ALL ON dispatch_event TO windmill_admin;
GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_user;
GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_admin;
backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql
+6
View File
@@ -0,0 +1,6 @@
REVOKE ALL ON workspace_diff FROM windmill_user;
REVOKE ALL ON workspace_diff FROM windmill_admin;
REVOKE ALL ON materialized_partition FROM windmill_user;
REVOKE ALL ON materialized_partition FROM windmill_admin;
REVOKE ALL ON debounce_stale_data FROM windmill_user;
REVOKE ALL ON debounce_stale_data FROM windmill_admin;
backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql
+22
View File
@@ -0,0 +1,22 @@
-- Same grant gap fixed for notify_event (20260619091631), script_trigger
-- (20260619112847), and dispatch_event: tables created after the one-time
-- GRANT ALL in 20250205131523 rely on ALTER DEFAULT PRIVILEGES, which only
-- applies to objects created by the role that set them. On deployments whose
-- migration runner is a different role, these tables end up ungranted, and
-- writes that run under the RLS role (a transaction opened via
-- user_db.begin(&authed) -> SET LOCAL ROLE windmill_user/windmill_admin) fail
-- with "permission denied for table <name>".
--
-- Each table below has a confirmed write on a user_db transaction:
-- * workspace_diff -- UPDATE in set_ws_specific (workspaces.rs)
-- * materialized_partition -- INSERT via record_materialization (assets API)
-- * debounce_stale_data -- DELETE in resume_suspended_trigger_jobs
-- (global_handler.rs), the same tx that reaps a
-- job's side rows
-- None has a sequence, so only table grants are needed.
GRANT ALL ON workspace_diff TO windmill_user;
GRANT ALL ON workspace_diff TO windmill_admin;
GRANT ALL ON materialized_partition TO windmill_user;
GRANT ALL ON materialized_partition TO windmill_admin;
GRANT ALL ON debounce_stale_data TO windmill_user;
GRANT ALL ON debounce_stale_data TO windmill_admin;
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6272,7 +6272,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"convert_case",
"serde",
@@ -6293,7 +6293,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6305,7 +6305,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6317,7 +6317,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6329,7 +6329,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6341,7 +6341,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6353,7 +6353,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6364,7 +6364,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6375,7 +6375,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6387,7 +6387,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6398,7 +6398,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6420,7 +6420,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6432,7 +6432,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6446,7 +6446,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6463,7 +6463,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde",
@@ -6488,7 +6488,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6522,7 +6522,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6538,7 +6538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6570,7 +6570,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"serde",
@@ -6581,7 +6581,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.742.0"
version = "1.744.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.742.0"
version = "1.744.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/parsers/windmill-parser-yaml/src/lib.rs
+44
View File
@@ -269,6 +269,12 @@ pub struct DelegateToGitRepoDetails {
pub commit: Option<String>,
pub inventories_location: Option<String>,
pub vars_location: Option<String>,
/// Path (relative to the cloned repo root) of an `ansible.cfg` to use as the
/// effective config for the run. When set, Windmill points `ANSIBLE_CONFIG` at
/// it so the repo's own settings (roles paths, inventory plugins, callbacks…)
/// apply, and only injects the settings that depend on runtime state it alone
/// controls (temp/home dirs, vault password) on top.
pub ansible_cfg: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub install_requirements: bool,
}
@@ -629,6 +635,10 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option<DelegateToGitRep
.get(&Yaml::String("vars_location".to_string()))
.and_then(|s| s.as_str())
.map(|s| s.to_string());
let ansible_cfg = v
.get(&Yaml::String("ansible_cfg".to_string()))
.and_then(|s| s.as_str())
.map(|s| s.to_string());
let install_requirements = v
.get(&Yaml::String("install_requirements".to_string()))
.and_then(|s| s.as_bool())
@@ -640,6 +650,7 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option<DelegateToGitRep
commit,
inventories_location,
vars_location,
ansible_cfg,
install_requirements,
});
}
@@ -1049,6 +1060,39 @@ delegate_to_git_repo:
assert_eq!(d.playbook.as_deref(), Some("site.yml"));
}
#[test]
fn test_parse_delegate_ansible_cfg() {
let p = r#"
---
delegate_to_git_repo:
resource: u/admin/repo
playbook: site.yml
ansible_cfg: config/ansible.cfg
---
- name: Test
hosts: all
"#;
let (_, reqs, _) = parse_ansible_reqs(p).unwrap();
let d = reqs.unwrap().delegate_to_git_repo.unwrap();
assert_eq!(d.ansible_cfg.as_deref(), Some("config/ansible.cfg"));
}
#[test]
fn test_parse_delegate_ansible_cfg_absent() {
let p = r#"
---
delegate_to_git_repo:
resource: u/admin/repo
playbook: site.yml
---
- name: Test
hosts: all
"#;
let (_, reqs, _) = parse_ansible_reqs(p).unwrap();
let d = reqs.unwrap().delegate_to_git_repo.unwrap();
assert_eq!(d.ansible_cfg, None);
}
#[test]
fn test_parse_install_requirements_true() {
let p = r#"
backend/parsers/windmill-parser/src/asset_parser.rs
+130 -24
View File
@@ -224,15 +224,21 @@ pub struct RetrySpec {
pub delay: Option<String>,
}
// `// materialize [manual] <asset> [append] [key=<col>]` — declares that this
// script produces a *managed* materialization of `<asset>` (a `ducklake://`
// table). By default the runtime generates the write DDL around the script's
// single trailing `SELECT` and owns idempotency, partition-state and snapshot
// capture. `manual` is the escape hatch: the script writes its own DDL and the
// runtime only records state (track-only). The reconciliation strategy options
// (`append`, `key=<col>`) apply to managed mode: none → DELETE-by-partition +
// INSERT (replace); `key=<col>` → MERGE (dedup within slice); `append` →
// `// materialize [manual] <asset> [append] [key=<col>] [history] [track=<c1,c2>]`
// — declares that this script produces a *managed* materialization of `<asset>`
// (a `ducklake://` table). By default the runtime generates the write DDL around
// the script's single trailing `SELECT` and owns idempotency, partition-state
// and snapshot capture. `manual` is the escape hatch: the script writes its own
// DDL and the runtime only records state (track-only). The reconciliation
// strategy options apply to managed mode: none → DELETE-by-partition + INSERT
// (replace); `key=<col>` → MERGE (dedup within slice, SCD type 1); `append` →
// INSERT-only. `append` wins if both are given (deploy-time warning).
// `key=<col> history` upgrades the merge to SCD type 2: the SELECT is the current
// snapshot (one row per key), and a change to any tracked column (`track=`,
// default all non-key) closes the prior version and opens a new one, keeping full
// history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a
// recognized alias for `history`. `deletes=close` (scd2 only) also closes a key
// that disappears from the snapshot; default leaves absent keys current.
#[derive(Serialize, Debug, PartialEq, Clone)]
pub struct MaterializeSpec {
pub target_kind: AssetKind,
@@ -243,6 +249,20 @@ pub struct MaterializeSpec {
pub append: bool,
#[serde(skip_serializing_if = "Option::is_none", default)]
pub unique_key: Option<String>,
// `scd2` managed history mode: the SELECT is the current snapshot (one row
// per `unique_key`), and the runtime maintains a Slowly-Changing-Dimension
// type-2 history (`valid_from`/`valid_to`/`is_current`). `unique_key` (the
// `key=` opt) is the natural key; `track` lists the columns whose change
// opens a new version (empty ⇒ all non-key columns). Managed mode only.
#[serde(skip_serializing_if = "std::ops::Not::not", default)]
pub scd2: bool,
#[serde(skip_serializing_if = "Vec::is_empty", default)]
pub track: Vec<String>,
// scd2 only: `deletes=close` opts into hard-delete-close — a key that
// disappears from the snapshot has its current version closed (dbt's
// `hard_deletes=close`). Default (false) leaves absent keys current.
#[serde(skip_serializing_if = "std::ops::Not::not", default)]
pub close_deleted: bool,
}
// `// data_test <kind> …` — a data-quality assertion run against the
@@ -800,19 +820,31 @@ fn parse_retry_spec(s: &str) -> Option<RetrySpec> {
Some(RetrySpec { count, delay })
}
// Parse a `// materialize [manual] <asset> [append] [key=<col>]` right-hand
// side. An optional leading `manual` token (whitespace-delimited) opts out of
// managed mode (track-only). The next whitespace token is the target asset URI
// Parse a `// materialize [manual] <asset> [append] [key=<col>] [history]
// [track=<c1,c2>]` right-hand side. An optional leading `manual` token opts out
// of managed mode (track-only); a leading `scd2` token is an alias for the
// `history` flag. The next whitespace token is the target asset URI
// (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the
// remainder are strategy options — bare `append` and `key=<col>` (merge key),
// which apply to managed mode only. A missing/empty target yields `None` (the
// annotation is dropped, fail-safe).
// remainder are strategy options — bare `append`, bare `history` (SCD type-2 on
// a keyed merge), `key=<col>` (merge/scd2 key), `track=<c1,c2,…>` (scd2 tracked
// columns), and `deletes=close` (scd2 hard-delete-close) — which apply to managed
// mode only. A missing/empty target yields `None` (the annotation is dropped,
// fail-safe).
fn parse_materialize_spec(s: &str) -> Option<MaterializeSpec> {
let (manual, rest) = match s.strip_prefix("manual") {
Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => {
(true, after.trim_start())
}
_ => (false, s),
// One optional leading mode keyword: `manual` (escape hatch, track-only) or
// `scd2` (alias for the `history` flag below). A missing keyword is the
// default managed mode.
fn strip_mode<'a>(s: &'a str, kw: &str) -> Option<&'a str> {
s.strip_prefix(kw)
.filter(|after| after.is_empty() || after.starts_with(char::is_whitespace))
.map(|after| after.trim_start())
}
let (manual, scd2_kw, rest) = if let Some(after) = strip_mode(s, "manual") {
(true, false, after)
} else if let Some(after) = strip_mode(s, "scd2") {
(false, true, after)
} else {
(false, false, s)
};
let mut it = rest.trim().splitn(2, char::is_whitespace);
let asset_tok = it.next()?;
@@ -822,11 +854,38 @@ fn parse_materialize_spec(s: &str) -> Option<MaterializeSpec> {
return None;
}
let append = opts_str.split_whitespace().any(|t| t == "append");
let unique_key = parse_kv_opts(opts_str)
.get("key")
.filter(|k| !k.is_empty())
.cloned();
Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key })
// SCD type-2 history mode. The primary spelling is the bare `history` flag on
// a keyed merge (`key=<col> history`) — it reads as "a keyed upsert that keeps
// history"; the leading `scd2` keyword is a recognized alias for the same.
let scd2 = scd2_kw || opts_str.split_whitespace().any(|t| t == "history");
let opts = parse_kv_opts(opts_str);
let unique_key = opts.get("key").filter(|k| !k.is_empty()).cloned();
// `track=<c1,c2,…>` (scd2 only): comma-separated columns whose change opens a
// new version. Empty entries dropped; an empty list ⇒ track all non-key cols.
// Like every `=`-option here the value is whitespace-terminated, so the list
// must contain no spaces (`track=a,b`, not `track=a, b` — the rest is dropped).
let track = opts
.get("track")
.map(|v| {
v.split(',')
.map(|c| c.trim().to_string())
.filter(|c| !c.is_empty())
.collect::<Vec<_>>()
})
.unwrap_or_default();
// `deletes=close` (scd2 only) opts into hard-delete-close; any other value
// (or absence) keeps the soft-delete default.
let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false);
Some(MaterializeSpec {
target_kind,
target_path: path.to_string(),
manual,
append,
unique_key,
scd2,
track,
close_deleted,
})
}
// Parse a `// data_test <kind> …` right-hand side into one `DataTest`. The
@@ -1457,6 +1516,53 @@ mod pipeline_annotation_tests {
assert_eq!(m.unique_key, None);
}
#[test]
fn materialize_scd2_history_flag_with_key_and_track() {
// Primary spelling: `key=<col> history` on a merge.
let out = parse_pipeline_annotations(
"// materialize ducklake://a/dim key=id history track=name,tier",
);
let m = out.materialize.expect("materialize");
assert!(m.scd2);
assert!(!m.manual);
assert_eq!(m.unique_key.as_deref(), Some("id"));
assert_eq!(m.track, vec!["name".to_string(), "tier".to_string()]);
}
#[test]
fn materialize_scd2_keyword_is_alias_for_history() {
let out = parse_pipeline_annotations("// materialize scd2 ducklake://a/dim key=id");
let m = out.materialize.expect("materialize");
assert!(m.scd2);
assert_eq!(m.unique_key.as_deref(), Some("id"));
assert!(m.track.is_empty());
// soft-delete default
assert!(!m.close_deleted);
}
#[test]
fn materialize_scd2_deletes_close_opt() {
let out = parse_pipeline_annotations(
"// materialize ducklake://a/dim key=id history deletes=close",
);
let m = out.materialize.expect("materialize");
assert!(m.scd2);
assert!(m.close_deleted);
// any other value keeps the soft-delete default
let out = parse_pipeline_annotations(
"// materialize ducklake://a/dim key=id history deletes=ignore",
);
assert!(!out.materialize.expect("materialize").close_deleted);
}
#[test]
fn materialize_key_without_history_is_plain_merge() {
let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id");
let m = out.materialize.expect("materialize");
assert!(!m.scd2, "no history flag ⇒ SCD1 merge, not scd2");
assert_eq!(m.unique_key.as_deref(), Some("id"));
}
#[test]
fn materialize_default_syntax_shorthand() {
let out = parse_pipeline_annotations("// materialize ducklake");
backend/parsers/windmill-parser/src/sql_materialize.rs
+310 -3
View File
@@ -327,20 +327,45 @@ fn snippet(stmt: &str) -> String {
// ---------------------------------------------------------------------------
/// How a (partition of a) materialized table is reconciled on each run.
/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else
/// `unique_key` → `Merge`, else `Replace`.
/// Derived at deploy from the annotation: `key=<col> history` (or the `scd2`
/// alias) → `Scd2`, else `append` → `Append`, else `unique_key` → `Merge`, else
/// `Replace`.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum MaterializeStrategy {
/// DELETE the current partition, then INSERT — partition becomes exactly
/// what the SELECT returned. Full-refresh of the slice.
Replace,
/// Upsert within the slice on `unique_key` (delete-by-key + insert); rows
/// absent from the SELECT are left in place.
/// absent from the SELECT are left in place. This is SCD type 1: a changed
/// row overwrites the prior value, keeping no history.
Merge { unique_key: String },
/// INSERT only — immutable event-log semantics.
Append,
/// Slowly Changing Dimension type 2: the SELECT is the *current* snapshot
/// (one row per `key`); a change to any tracked column closes the prior
/// version (`valid_to`/`is_current=false`) and opens a new one, so the full
/// history is preserved. `track` empty ⇒ every non-key column is tracked.
/// `close_deleted` (opt-in `deletes=close`) also closes the current version
/// of a key that disappears from the snapshot (dbt's `hard_deletes=close`);
/// default (false) leaves absent keys current (soft delete).
/// Unpartitioned only (the worker rejects `// partitioned` + scd2).
Scd2 { key: String, track: Vec<String>, close_deleted: bool },
}
/// SCD2 metadata columns appended to the managed history table. Fixed names so
/// the generated diff/close/open SQL and any `// data_test` on them agree.
const SCD2_VALID_FROM: &str = "valid_from";
const SCD2_VALID_TO: &str = "valid_to";
const SCD2_IS_CURRENT: &str = "is_current";
/// Connection-local temp table holding the keys whose version must be rotated
/// this run (changed + new). Captured before the write so the close and the
/// open see the same set. `_wm_` prefix so it never collides with user tables.
const SCD2_CHANGED_KEYS: &str = "_wm_scd2_changed";
/// Connection-local temp table holding the keys that disappeared from the
/// snapshot this run (present-and-current in the table, absent from the SELECT).
/// Only used when `close_deleted` (`deletes=close`) is set.
const SCD2_DELETED_KEYS: &str = "_wm_scd2_deleted";
/// Inputs to materialization codegen, all resolved at run time by the worker.
/// Pure: produces SQL text; executes nothing.
#[derive(Debug, Clone)]
@@ -377,6 +402,13 @@ impl<'a> MaterializeCodegen<'a> {
let pval = self.partition_value_sql;
let mut out = Vec::new();
// SCD2 has a shape unlike the DELETE/INSERT strategies (diff → close old
// → open new) and does not support partitioning (rejected at the worker),
// so it is generated up front by its own helper.
if let MaterializeStrategy::Scd2 { key, track, close_deleted } = &self.strategy {
return self.scd2_statements(key, track, *close_deleted);
}
// Whole-table replace: rebuild the table to match the SELECT's *current*
// schema each run with one atomic `CREATE OR REPLACE` (which DuckLake
// still snapshots). This is the only path that survives a changed SELECT
@@ -441,10 +473,160 @@ impl<'a> MaterializeCodegen<'a> {
));
out.push(format!("INSERT INTO {t} {source};"));
}
// Handled by the early return above (scd2 has no partitioned form).
MaterializeStrategy::Scd2 { .. } => unreachable!("scd2 handled before this match"),
}
out.push("COMMIT;".to_string());
out
}
/// SCD2 codegen: the incoming SELECT is the *current desired snapshot* (one
/// row per `key`); we diff it against the live current rows, close the prior
/// version of every changed/new key, and open a fresh one — so history is
/// kept. `track` empty ⇒ every non-key column is tracked for change
/// detection.
///
/// Shape (all one transaction for the mutation, mirroring the other
/// strategies so a partial failure leaves the prior snapshot intact):
/// 1. bootstrap the table (business columns + `valid_from/valid_to/
/// is_current`), idempotent;
/// 2. capture changed+new keys into a connection-local temp table *before*
/// the write — the close below flips `is_current`, so recomputing the
/// diff after it would see a different set;
/// 3. close the prior open version of those keys (`UPDATE` — not `MERGE`:
/// DuckLake's MERGE is the unreliable path, plain UPDATE works);
/// 3b. when `close_deleted`, also capture the keys that vanished from the
/// snapshot and close their current version (no reopen) — dbt's
/// `hard_deletes=close`;
/// 4. open a new current version from the snapshot;
/// 5. create the `<dim>_current` convenience view once (`IF NOT EXISTS`),
/// inside the same transaction so it doesn't advance the DuckLake snapshot
/// past the data write the summary records (and so an unchanged rerun,
/// whose UPDATE/INSERT touch no rows, stays a true no-op).
///
/// Close/open match keys with `IS NOT DISTINCT FROM` (via a correlated
/// `EXISTS`), not `key IN (…)`: SQL `IN` never matches `NULL`, so a `NULL`
/// natural key would be flagged as changed yet silently skipped by both the
/// close and the open, dropping the row. Null-safe matching materializes it
/// instead (a `NULL` key is still ill-formed for a dimension — guard it with
/// `// data_test not_null <key>` — but it must not vanish).
///
/// Without `close_deleted`, keys present in the table but absent from the
/// SELECT are left current (soft delete — dbt's `hard_deletes=ignore` default;
/// with `close_deleted` they are closed instead — see step 3b). The effective
/// timestamp is `now()`, which DuckDB fixes to
/// the transaction start, so `valid_from`/`valid_to` are consistent within a
/// run without a nondeterministic per-statement clock.
///
/// Reserved columns: `valid_from`/`valid_to`/`is_current` are appended to the
/// user's SELECT with these fixed names (kept clean so consumers write
/// `WHERE is_current` / `ASOF JOIN … >= valid_from`). A SELECT that already
/// projects one of them is a v1 constraint violation — the bootstrap then
/// produces a duplicate output column and the run fails at execution
/// (documented; not statically checkable here since the SELECT's columns
/// aren't known at codegen time).
fn scd2_statements(&self, key: &str, track: &[String], close_deleted: bool) -> Vec<String> {
let t = self.target_qualified;
let sel = self.select_sql;
let k = quote_ident(key);
let vf = SCD2_VALID_FROM;
let vt = SCD2_VALID_TO;
let ic = SCD2_IS_CURRENT;
let changed = SCD2_CHANGED_KEYS;
let deleted = SCD2_DELETED_KEYS;
// Transaction-stable effective timestamp (see doc above). Cast to plain
// TIMESTAMP so it matches the bootstrapped column type (now() is TZ-aware).
let ts = "CAST(now() AS TIMESTAMP)";
// Projection compared to detect change. Empty `track` ⇒ all business
// columns via `* EXCLUDE (<scd cols>)` on the table side (which carries
// the extra metadata columns) and `*` on the snapshot side. An explicit
// `track` ⇒ key + those columns on both sides. `EXCEPT` treats NULLs as
// equal, so an unchanged NULL is not read as a change.
let (src_proj, tgt_proj) = if track.is_empty() {
(
format!("SELECT * FROM ({sel})"),
format!("SELECT * EXCLUDE ({vf}, {vt}, {ic}) FROM {t} WHERE {ic}"),
)
} else {
let cols = std::iter::once(key)
.chain(track.iter().map(String::as_str))
.map(quote_ident)
.collect::<Vec<_>>()
.join(", ");
(
format!("SELECT {cols} FROM ({sel})"),
format!("SELECT {cols} FROM {t} WHERE {ic}"),
)
};
let mut out = vec![
format!(
"CREATE TABLE IF NOT EXISTS {t} AS SELECT *, \
CAST(NULL AS TIMESTAMP) AS {vf}, \
CAST(NULL AS TIMESTAMP) AS {vt}, \
CAST(NULL AS BOOLEAN) AS {ic} FROM ({sel}) WHERE false;"
),
format!(
"CREATE OR REPLACE TEMP TABLE {changed} AS \
SELECT {k} FROM ({src_proj} EXCEPT {tgt_proj});"
),
];
// Hard-delete-close (`deletes=close`): the keys that vanished from the
// snapshot — present-and-current in the table, absent from the SELECT.
// Captured before the transaction (like `changed`) and disjoint from it (a
// key is either in the snapshot or not), so the two closes never overlap.
if close_deleted {
out.push(format!(
"CREATE OR REPLACE TEMP TABLE {deleted} AS \
SELECT {k} FROM (SELECT {k} FROM {t} WHERE {ic} EXCEPT SELECT {k} FROM ({sel}));"
));
}
out.push("BEGIN TRANSACTION;".to_string());
out.push(format!(
"UPDATE {t} SET {vt} = {ts}, {ic} = false \
WHERE {ic} AND EXISTS (SELECT 1 FROM {changed} \
WHERE {changed}.{k} IS NOT DISTINCT FROM {t}.{k});"
));
// Close vanished keys — no matching INSERT below, so they close without
// reopening. A key that later reappears isn't in `WHERE is_current`, so the
// `changed` diff treats it as new and opens a fresh version (a validity gap
// between the delete and the reactivation — correct SCD2).
if close_deleted {
out.push(format!(
"UPDATE {t} SET {vt} = {ts}, {ic} = false \
WHERE {ic} AND EXISTS (SELECT 1 FROM {deleted} \
WHERE {deleted}.{k} IS NOT DISTINCT FROM {t}.{k});"
));
}
out.push(format!(
"INSERT INTO {t} SELECT s.*, {ts} AS {vf}, CAST(NULL AS TIMESTAMP) AS {vt}, \
true AS {ic} FROM ({sel}) s WHERE EXISTS (SELECT 1 FROM {changed} c \
WHERE c.{k} IS NOT DISTINCT FROM s.{k});"
));
out.push(
// Consumer convenience: a `<dim>_current` view (the live slice) so the
// common "just the latest version" read needs no `WHERE is_current`,
// and downstream scripts can `// on` / read it directly. For the
// effective-dated payoff, consumers `ASOF JOIN <dim> ON fact.key =
// dim.<key> AND fact.ts >= dim.valid_from`.
//
// `CREATE VIEW IF NOT EXISTS` (not `OR REPLACE`), created inside the
// write transaction, on purpose: the view definition never changes
// (`SELECT * WHERE is_current` always reflects live data), and a
// catalog write advances the DuckLake snapshot — so `OR REPLACE` on
// every run would (a) advance the snapshot on an otherwise no-op
// unchanged run and (b) make the summary's `max(snapshot_id)` record
// the view DDL instead of the data write. `IF NOT EXISTS` creates it
// once (folded into the first data-write snapshot) and is a true no-op
// afterwards. The `<dim>_current` name is reserved: if a real table by
// that name already exists, `IF NOT EXISTS` skips silently (no view,
// no error) — documented as a reserved suffix.
format!("CREATE VIEW IF NOT EXISTS {t}_current AS SELECT * FROM {t} WHERE {ic};"),
);
out.push("COMMIT;".to_string());
out
}
}
/// The read that captures the DuckLake snapshot id produced by the write, for
@@ -1111,6 +1293,131 @@ mod tests {
);
}
#[test]
fn codegen_scd2_default_track_closes_old_opens_new() {
// Empty `track` ⇒ diff on all business columns via `* EXCLUDE (scd cols)`.
let cg = MaterializeCodegen {
target_qualified: "_wm_target.dim_scd2",
select_sql: "SELECT id, name FROM dl.src",
partition_col: "_wm_partition",
partition_value_sql: "''",
partitioned: false,
strategy: MaterializeStrategy::Scd2 {
key: "id".to_string(),
track: vec![],
close_deleted: false,
},
};
let st = cg.statements();
// bootstrap adds the three SCD metadata columns
assert!(st[0].starts_with("CREATE TABLE IF NOT EXISTS _wm_target.dim_scd2 AS SELECT *,"));
assert!(st[0].contains("AS valid_from"));
assert!(st[0].contains("AS valid_to"));
assert!(st[0].contains("AS is_current"));
// changed-key set captured before the transaction, all cols compared
assert!(
st[1].contains("CREATE OR REPLACE TEMP TABLE _wm_scd2_changed AS SELECT \"id\" FROM")
);
assert!(st[1].contains("SELECT * FROM (SELECT id, name FROM dl.src) EXCEPT"));
assert!(st[1].contains("SELECT * EXCLUDE (valid_from, valid_to, is_current) FROM _wm_target.dim_scd2 WHERE is_current"));
assert_eq!(st[2], "BEGIN TRANSACTION;");
// close: UPDATE (not MERGE) the prior open version of changed keys, with
// null-safe key matching (IS NOT DISTINCT FROM, not IN — IN drops NULLs)
assert!(st[3].starts_with("UPDATE _wm_target.dim_scd2 SET valid_to = CAST(now() AS TIMESTAMP), is_current = false"));
assert!(st[3].contains(
"WHERE is_current AND EXISTS (SELECT 1 FROM _wm_scd2_changed \
WHERE _wm_scd2_changed.\"id\" IS NOT DISTINCT FROM _wm_target.dim_scd2.\"id\");"
));
// open: INSERT the new current version, null-safe key matching
assert!(st[4].starts_with(
"INSERT INTO _wm_target.dim_scd2 SELECT s.*, CAST(now() AS TIMESTAMP) AS valid_from"
));
assert!(st[4].contains(
"true AS is_current FROM (SELECT id, name FROM dl.src) s WHERE EXISTS \
(SELECT 1 FROM _wm_scd2_changed c WHERE c.\"id\" IS NOT DISTINCT FROM s.\"id\");"
));
// consumer-convenience `<dim>_current` view: `IF NOT EXISTS` (created once,
// no-op on unchanged reruns) and INSIDE the txn (folded into the write snapshot)
assert_eq!(
st[5],
"CREATE VIEW IF NOT EXISTS _wm_target.dim_scd2_current AS SELECT * FROM _wm_target.dim_scd2 WHERE is_current;"
);
assert_eq!(st[6], "COMMIT;");
// no fragile constructs: no MERGE INTO, and no NULL-dropping `IN (SELECT`
assert!(!st.iter().any(|s| s.contains("MERGE INTO")));
assert!(!st.iter().any(|s| s.contains("IN (SELECT")));
// soft-delete default: no deleted-key set, no second close
assert!(!st.iter().any(|s| s.contains("_wm_scd2_deleted")));
}
#[test]
fn codegen_scd2_explicit_track_projects_key_and_tracked_cols() {
let cg = MaterializeCodegen {
target_qualified: "_wm_target.dim",
select_sql: "SELECT id, name, addr FROM dl.src",
partition_col: "_wm_partition",
partition_value_sql: "''",
partitioned: false,
strategy: MaterializeStrategy::Scd2 {
key: "id".to_string(),
track: vec!["name".to_string()],
close_deleted: false,
},
};
let st = cg.statements();
// only key + tracked cols are compared (addr changes don't rotate a version)
assert!(st[1].contains("SELECT \"id\", \"name\" FROM (SELECT id, name, addr FROM dl.src) EXCEPT SELECT \"id\", \"name\" FROM _wm_target.dim WHERE is_current"));
}
#[test]
fn codegen_scd2_close_deleted_adds_deleted_set_and_second_close() {
let cg = MaterializeCodegen {
target_qualified: "_wm_target.dim",
select_sql: "SELECT id, name FROM dl.src",
partition_col: "_wm_partition",
partition_value_sql: "''",
partitioned: false,
strategy: MaterializeStrategy::Scd2 {
key: "id".to_string(),
track: vec![],
close_deleted: true,
},
};
let st = cg.statements();
// the deleted-key set: current keys absent from the snapshot, captured
// before the transaction (like `changed`)
assert!(st.iter().any(|s| s.contains(
"CREATE OR REPLACE TEMP TABLE _wm_scd2_deleted AS SELECT \"id\" FROM \
(SELECT \"id\" FROM _wm_target.dim WHERE is_current EXCEPT SELECT \"id\" FROM (SELECT id, name FROM dl.src));"
)));
// a second close UPDATE against the deleted set (null-safe), and NO INSERT
// that reopens deleted keys (the only INSERT filters on `_wm_scd2_changed`)
assert!(st
.iter()
.any(|s| s.starts_with("UPDATE _wm_target.dim SET valid_to")
&& s.contains(
"EXISTS (SELECT 1 FROM _wm_scd2_deleted \
WHERE _wm_scd2_deleted.\"id\" IS NOT DISTINCT FROM _wm_target.dim.\"id\");"
)));
assert_eq!(
st.iter().filter(|s| s.starts_with("INSERT INTO")).count(),
1
);
assert!(st
.iter()
.find(|s| s.starts_with("INSERT INTO"))
.unwrap()
.contains("_wm_scd2_changed"));
// the deleted close is inside the transaction (between BEGIN and COMMIT)
let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap();
let commit = st.iter().position(|s| s == "COMMIT;").unwrap();
let del_close = st
.iter()
.position(|s| s.starts_with("UPDATE") && s.contains("_wm_scd2_deleted"))
.unwrap();
assert!(begin < del_close && del_close < commit);
}
#[test]
fn snapshot_capture_targets_alias() {
assert_eq!(
backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json
Vendored
+58
View File
@@ -253,6 +253,64 @@
}
}
},
{
"name": "materialize scd2 via history flag with key, track and deletes=close",
"code": "// pipeline\n// materialize ducklake://analytics/dim_customer key=id history track=name,tier deletes=close\nSELECT 1;",
"expected": {
"in_pipeline": true,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "analytics/dim_customer",
"unique_key": "id",
"scd2": true,
"track": ["name", "tier"],
"close_deleted": true
}
}
},
{
"name": "materialize scd2 keyword alias with key only (track all non-key cols)",
"code": "// materialize scd2 ducklake://analytics/dim key=id\nSELECT 1;",
"expected": {
"in_pipeline": false,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "analytics/dim",
"unique_key": "id",
"scd2": true
}
}
},
{
"name": "materialize key without history is plain merge (SCD1, not scd2)",
"code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;",
"expected": {
"in_pipeline": false,
"asset_triggers": [],
"native_triggers": [],
"partition": null,
"freshness": null,
"tag": null,
"retry": null,
"materialize": {
"target_kind": "ducklake",
"target_path": "analytics/dim",
"unique_key": "id"
}
}
},
{
"name": "materialize manual escape hatch, first value wins",
"code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}",
backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs
+12
View File
@@ -59,6 +59,12 @@ struct ExpectedMaterialize {
append: bool,
#[serde(default)]
unique_key: Option<String>,
#[serde(default)]
scd2: bool,
#[serde(default)]
track: Vec<String>,
#[serde(default)]
close_deleted: bool,
}
#[derive(Deserialize)]
@@ -191,6 +197,12 @@ fn pipeline_annotation_fixtures_match() {
assert_eq!(m.manual, e.manual, "{ctx}: materialize manual");
assert_eq!(m.append, e.append, "{ctx}: materialize append");
assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key");
assert_eq!(m.scd2, e.scd2, "{ctx}: materialize scd2");
assert_eq!(m.track, e.track, "{ctx}: materialize track");
assert_eq!(
m.close_deleted, e.close_deleted,
"{ctx}: materialize close_deleted"
);
}
(got, want) => panic!(
"{ctx}: materialize mismatch — got {:?}, want present={}",
backend/summarized_schema.txt
+1 -1
View File
@@ -182,7 +182,7 @@ windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char)
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
workspace_diff: source_workspace_id(char), fork_workspace_id(char), path(char), kind(char), ahead(int), behind(int), has_changes(bool), exists_in_source(bool), exists_in_fork(bool)
backend/windmill-api-groups/src/folders.rs
+4 -5
View File
@@ -146,12 +146,11 @@ async fn list_foldernames(
}
fn validate_owner(owner: &str) -> Result<()> {
if !owner
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-')
{
if !owner.chars().all(|c| {
c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-' || c == '.' || c == '@'
}) {
return Err(error::Error::BadRequest(
"Invalid owner: must contain only alphanumeric characters, underscores, hyphens, or slashes".to_string(),
"Invalid owner: must contain only alphanumeric characters, underscores, hyphens, slashes, dots, or at-signs".to_string(),
));
}
Ok(())
backend/windmill-api-scripts/src/scripts.rs
+41 -13
View File
@@ -408,6 +408,14 @@ async fn list_scripts(
.and_then(|s| s.as_str())
.filter(|s| !s.is_empty() && *s != row.path.as_str())
.map(|s| s.to_string());
// A draft-only pipeline node (`// pipeline`) has no deployed row to carry
// auto_kind, so compute it from the draft content — mirroring the create
// path — so the home page folds it into its pipeline like a deployed member.
let auto_kind = v
.get("content")
.and_then(|s| s.as_str())
.filter(|c| parse_pipeline_annotations(c).in_pipeline)
.map(|_| "pipeline".to_string());
rows.push(ListableScript {
hash: ScriptHash(0),
path: row.path,
@@ -429,7 +437,7 @@ async fn list_scripts(
draft_only: Some(true),
has_deploy_errors: false,
ws_error_handler_muted: None,
auto_kind: None,
auto_kind,
use_codebase: false,
deployment_msg: None,
kind,
@@ -1289,10 +1297,16 @@ async fn create_script_internal<'c>(
// but the executor parses the signature from the un-wrapped script, so
// `$name` references in the SELECT stay bound at run time.
}
// `key=` (merge) and `append` are mutually exclusive reconciliation
// strategies; append (INSERT-only) wins. Surface the conflict rather
// than silently dropping the dedup the author may have intended.
if m.unique_key.is_some() && m.append {
// Reconciliation strategies are mutually exclusive; surface a conflict
// rather than silently dropping behavior the author may have intended.
// Precedence must mirror the runtime (`duckdb_executor` strategy
// derivation): scd2 (`history`) > append > merge (`key=`) > replace.
if m.scd2 && m.append {
tracing::warn!(
"script {}: both `history`/`scd2` and `append` set on // materialize; history wins (SCD2), append ignored",
ns.path
);
} else if m.unique_key.is_some() && m.append {
tracing::warn!(
"script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)",
ns.path
@@ -1335,14 +1349,28 @@ async fn create_script_internal<'c>(
let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() {
let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind);
let mut a = effective_assets.unwrap_or_default();
if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) {
a.push(windmill_common::assets::AssetWithAltAccessType {
path: m.target_path.clone(),
kind,
access_type: Some(windmill_common::assets::AssetUsageAccessType::W),
alt_access_type: None,
columns: None,
});
// Produced assets: the managed table, plus — for managed scd2 — the
// `<dim>_current` companion view the runtime (re)creates each run.
// Registering the view as a write asset lets `// on
// ducklake://…/<dim>_current` subscribers be dispatched by the cascade
// (which fans out from these deploy-time asset rows); without it a
// subscriber on the view would silently never fire. Gated on `!manual`:
// manual mode owns its own DDL and short-circuits before the scd2 codegen
// (no view is created), so registering it there would be a false edge.
let mut targets = vec![m.target_path.clone()];
if m.scd2 && !m.manual {
targets.push(format!("{}_current", m.target_path));
}
for path in targets {
if !a.iter().any(|x| x.kind == kind && x.path == path) {
a.push(windmill_common::assets::AssetWithAltAccessType {
path,
kind,
access_type: Some(windmill_common::assets::AssetUsageAccessType::W),
alt_access_type: None,
columns: None,
});
}
}
Some(a)
} else {
backend/windmill-api-settings/src/lib.rs
+25 -1
View File
@@ -406,7 +406,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul
)
}
ObjectSettings::Gcs(gcs) => {
if gcs.service_account_key.is_empty() {
// Mirror `build_gcs_client`'s blank-key check (shared predicate): a blank/`{}` key falls
// back to the instance's ambient credentials there, so it must be rejected here too —
// otherwise an untrusted caller could probe with the server's identity (the very
// SSRF/credential-exfil this function guards against).
if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) {
return Err(error::Error::NotAuthorized(
"Testing GCS storage without a service account key requires a super admin"
.to_string(),
@@ -2187,6 +2191,26 @@ mod object_storage_test_hardening {
);
}
#[tokio::test]
async fn rejects_gcs_blank_service_account_key() {
// A blank key makes build_gcs_client fall back to the instance's ambient credentials, so an
// untrusted caller must not be allowed to test with it. The `serviceAccountKey` field is
// serialized via serde's `as_string` (`to_string` of the JSON value), so the settings UI's
// "no key" empty object arrives as `"{}"` and a null as `"null"` — both must be rejected.
for key in [serde_json::json!({}), serde_json::json!(null)] {
let settings: ObjectSettings = serde_json::from_value(serde_json::json!({
"type": "Gcs",
"bucket": "b",
"serviceAccountKey": key
}))
.unwrap();
assert!(
validate_object_storage_test(&settings).await.is_err(),
"blank key {key:?} should be rejected"
);
}
}
fn ip(s: &str) -> IpAddr {
s.parse().unwrap()
}
backend/windmill-api-users/src/users.rs
+9
View File
@@ -1552,6 +1552,15 @@ async fn update_user(
}
if let Some(d) = eu.disabled {
#[cfg(feature = "enterprise")]
if !d {
if let Some(msg) =
windmill_common::ee_oss::check_seat_cap_for_reactivation(&db, &email_to_update)
.await?
{
return Err(Error::BadRequest(msg));
}
}
sqlx::query_scalar!(
"UPDATE password SET disabled = $1 WHERE email = $2",
d,
backend/windmill-api-workspaces/src/workspaces.rs
+745 -12
View File
@@ -6,7 +6,10 @@
* LICENSE-AGPL for a copy of the license.
*/
use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed};
use windmill_api_auth::{
build_scope_path_predicate, check_scopes, require_devops_role, require_is_writer,
require_super_admin, ApiAuthed,
};
use windmill_api_users::users::WorkspaceInvite;
use windmill_common::email_oss::send_email_if_possible;
use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME};
@@ -40,9 +43,10 @@ use windmill_common::workspaces::GitRepositorySettings;
#[cfg(feature = "enterprise")]
use windmill_common::workspaces::WorkspaceDeploymentUISettings;
use windmill_common::workspaces::{
check_user_against_rule, get_datatable_resource_from_db_unchecked, validate_fork_workspace_id,
DataTable, DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind,
ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings,
check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked,
validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable,
DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules,
ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME,
};
use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
use windmill_common::PgDatabase;
@@ -150,6 +154,9 @@ pub fn workspaced_service() -> Router {
.route("/leave", post(leave_workspace))
.route("/get_workspace_name", get(get_workspace_name))
.route("/create_fork", post(create_workspace_fork))
.route("/attach_dev_workspace", post(attach_dev_workspace))
.route("/detach_dev_workspace", post(detach_dev_workspace))
.route("/get_dev_workspace", get(get_dev_workspace))
.route("/change_workspace_name", post(change_workspace_name))
.route("/change_workspace_color", post(change_workspace_color))
.route(
@@ -191,6 +198,7 @@ pub fn workspaced_service() -> Router {
.route("/prune_versions", post(prune_versions))
.route("/list_ws_specific", get(list_ws_specific))
.route("/list_ws_specific_versions", get(list_ws_specific_versions))
.route("/set_ws_specific", post(set_ws_specific))
}
pub fn global_service() -> Router {
Router::new()
@@ -441,6 +449,20 @@ struct CreateWorkspaceFork {
/// forked workspace's datatable config to point to the new database.
#[serde(default)]
forked_datatables: Vec<ForkedDatatableInfo>,
/// Create the fork as a persistent dev workspace: the id is not required to carry the
/// `wm-fork-` prefix, and at most one dev workspace may exist per parent.
#[serde(default)]
is_dev_workspace: bool,
/// When creating a dev workspace, lock the parent ("prod") against direct deployment and/or
/// ad-hoc forking, so edits are funneled through the dev workspace.
#[serde(default)]
lock_prod_deploy: bool,
#[serde(default)]
lock_prod_forking: bool,
/// Copy the parent's members (usr rows + group memberships) into the fork so
/// the team can work in it. Defaults off; the dev-workspace UI defaults it on.
#[serde(default)]
copy_members: bool,
}
#[derive(Deserialize)]
@@ -469,6 +491,7 @@ struct UserWorkspace {
pub color: Option<String>,
pub operator_settings: Option<Option<serde_json::Value>>,
pub parent_workspace_id: Option<String>,
pub is_dev_workspace: bool,
pub disabled: bool,
}
@@ -637,6 +660,34 @@ async fn exists_workspace(
Ok(Json(exists))
}
/// Whether this workspace already has an active canonical dev workspace. The create-fork UI can't
/// rely on the caller's workspace list to decide this — a dev paired to this prod may exist that the
/// caller isn't a member of — so it asks the server, which sees all children.
#[derive(Serialize)]
struct DevWorkspaceInfo {
id: String,
name: String,
}
/// This workspace's active canonical dev workspace, if any. The create-fork UI and the dev-workspace
/// settings tab can't rely on the caller's workspace list — a dev paired to this prod may exist that
/// the caller isn't a member of — so they ask the server, which sees all children. Returns its id/name
/// so a prod admin who isn't a dev member can still see the pairing and detach it.
async fn get_dev_workspace(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Option<DevWorkspaceInfo>> {
let dev = sqlx::query_as!(
DevWorkspaceInfo,
"SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false",
&w_id
)
.fetch_optional(&db)
.await?;
Ok(Json(dev))
}
async fn list_workspaces(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -3643,6 +3694,7 @@ async fn user_workspaces(
let workspaces = sqlx::query_as!(
UserWorkspace,
"SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,
workspace.is_dev_workspace,
CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,
usr.disabled
FROM workspace
@@ -3826,6 +3878,8 @@ async fn create_workspace(
}
}
validate_workspace_name(&nw.name)?;
let mut tx: Transaction<'_, Postgres> = db.begin().await?;
check_w_id_conflict(&mut tx, &nw.id).await?;
@@ -4423,6 +4477,28 @@ async fn clone_groups(
Ok(())
}
/// Copy the source workspace's members (the `usr` rows, carrying each member's role) into the
/// target so a fork/dev can be a shared environment. Idempotent — skips members the target already
/// has. Group memberships are not handled here: the sole caller is the create-fork path, where
/// `clone_groups` already copies the source's full group structure (including `all` membership).
async fn copy_workspace_members(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
target_workspace_id: &str,
) -> Result<()> {
sqlx::query!(
"INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)
SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via
FROM usr WHERE workspace_id = $2
ON CONFLICT DO NOTHING",
target_workspace_id,
source_workspace_id,
)
.execute(&mut **tx)
.await?;
Ok(())
}
async fn clone_resource_types(
tx: &mut Transaction<'_, Postgres>,
source_workspace_id: &str,
@@ -4972,7 +5048,27 @@ async fn create_workspace_fork_branch(
return Err(Error::PermissionDenied(msg));
}
validate_fork_workspace_id(&nw.id)?;
// Two-phase create for git-synced workspaces: this endpoint only creates the git branch(es) and
// validates up front; it does NOT create the workspace row. The caller follows up with
// `create_workspace_fork`, which inserts the row and applies the dev designation + prod lock +
// member copy. So the dev/lock/copy_members fields here are validated only — they are acted on by
// that second call. Validating early lets a bad request fail before any branch is created.
if nw.is_dev_workspace {
validate_dev_workspace_id(&nw.id)?;
ensure_dev_parent_is_root(&db, &w_id).await?;
// Reject before creating any git branch if the parent already has a dev workspace,
// otherwise the deferred branch-creation job leaves a dangling branch on the synced repos.
ensure_no_existing_dev_workspace(&db, &w_id).await?;
// Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking the
// parent mutates its protection rules), so require admin of the parent regardless of the lock
// flags — mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker
// could claim the dev slot. Enforced in this first phase too so the request fails before any
// git branch is created rather than leaving dangling branches.
require_admin(authed.is_admin, &authed.username)?;
} else {
validate_fork_workspace_id(&nw.id)?;
}
validate_workspace_name(&nw.name)?;
// Fail before creating any git branch so a name conflict doesn't leave a
// dangling branch on the synced repos.
@@ -5117,7 +5213,12 @@ async fn create_workspace_fork(
)));
}
validate_fork_workspace_id(&nw.id)?;
if nw.is_dev_workspace {
validate_dev_workspace_id(&nw.id)?;
} else {
validate_fork_workspace_id(&nw.id)?;
}
validate_workspace_name(&nw.name)?;
// Check the id conflict before the CE workspace-count limit so that
// re-using a taken (possibly archived) fork id reports the actual
// conflict instead of a misleading "maximum number of workspaces" error.
@@ -5143,18 +5244,29 @@ async fn create_workspace_fork(
return Err(Error::PermissionDenied(msg));
}
if nw.is_dev_workspace {
ensure_dev_parent_is_root(&db, &parent_workspace_id).await?;
// Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking prod
// mutates its protection rules), so require admin of the parent regardless of the lock flags —
// mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker could
// claim the dev slot (and, without member copy, prod admins might not even see it to detach).
require_admin(authed.is_admin, &authed.username)?;
ensure_no_existing_dev_workspace(&db, &parent_workspace_id).await?;
}
let mut tx: Transaction<'_, Postgres> = db.begin().await?;
let forked_id = nw.id;
sqlx::query!(
"INSERT INTO workspace
(id, name, owner, parent_workspace_id)
VALUES ($1, $2, $3, $4)",
(id, name, owner, parent_workspace_id, is_dev_workspace)
VALUES ($1, $2, $3, $4, $5)",
forked_id,
nw.name,
authed.email,
parent_workspace_id,
nw.is_dev_workspace,
)
.execute(&mut *tx)
.await?;
@@ -5169,11 +5281,24 @@ async fn create_workspace_fork(
.execute(&mut *tx)
.await?;
// Optionally bring the parent's members into the fork (a shared dev env). Dev-only: it's part of the
// dev-workspace feature (and the frontend only offers it there), so the backend enforces it rather
// than trusting the client — copying the parent's whole team into an ordinary throwaway fork isn't
// intended. Dev creation is already admin-gated, so this is transitively admin-only too. Done before
// the explicit creator insert below so the creator (a parent member) is copied with full metadata
// (operator/role/is_service_account/added_via), not the bare row the insert alone would leave.
if nw.copy_members && nw.is_dev_workspace {
copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id).await?;
}
// Ensure the creator is a member of the fork even without copy_members (or if they aren't a parent
// member). No-op when copy_members already brought their full row.
sqlx::query!(
"INSERT INTO usr
(workspace_id, email, username, is_admin)
SELECT $1, email, username, is_admin FROM usr
WHERE workspace_id = $3 AND email = $2
ON CONFLICT DO NOTHING
",
forked_id,
authed.email,
@@ -5196,6 +5321,18 @@ async fn create_workspace_fork(
apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?;
}
// Lock the parent ("prod") so edits are funneled through this dev workspace.
let locked_prod = nw.is_dev_workspace && (nw.lock_prod_deploy || nw.lock_prod_forking);
if locked_prod {
lock_prod_workspace(
&mut tx,
&parent_workspace_id,
nw.lock_prod_deploy,
nw.lock_prod_forking,
)
.await?;
}
audit_log(
&mut *tx,
&authed,
@@ -5208,9 +5345,235 @@ async fn create_workspace_fork(
.await?;
tx.commit().await?;
if locked_prod {
windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id);
}
Ok(format!("Created forked workspace {}", &forked_id))
}
#[derive(Deserialize)]
struct AttachDevWorkspace {
dev_workspace_id: String,
#[serde(default)]
lock_prod_deploy: bool,
#[serde(default)]
lock_prod_forking: bool,
}
#[derive(Deserialize)]
struct DetachDevWorkspace {
dev_workspace_id: String,
}
/// Pair an existing standalone workspace to this workspace ("prod") as its dev workspace, without
/// cloning any data (both already exist). Sets the dev's parent + deploy_to to prod and, optionally,
/// locks prod against direct deployment.
async fn attach_dev_workspace(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(prod_w_id): Path<String>,
Json(req): Json<AttachDevWorkspace>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
if *CLOUD_HOSTED {
return Err(Error::BadRequest(
"Dev workspaces are not available on app.windmill.dev".to_string(),
));
}
let dev_w_id = req.dev_workspace_id;
if dev_w_id == prod_w_id {
return Err(Error::BadRequest(
"A workspace cannot be its own dev workspace".to_string(),
));
}
// The id is interpolated into a `wm-fork/<branch>/<id>` branch name like any fork.
validate_dev_workspace_id(&dev_w_id)?;
let dev = sqlx::query!(
r#"SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1"#,
&dev_w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("Workspace {} not found", dev_w_id)))?;
if dev.deleted {
return Err(Error::BadRequest(format!(
"Workspace {} is archived",
dev_w_id
)));
}
// A candidate that already belongs to a DIFFERENT parent can't be attached. A candidate already
// parented to this prod is allowed: it's the recovery path after renaming a dev workspace (the
// rename keeps the parent but drops the dev flag), and re-designating an existing fork of this
// prod as its dev.
if dev
.parent_workspace_id
.as_deref()
.is_some_and(|p| p != prod_w_id)
{
return Err(Error::BadRequest(format!(
"Workspace {} is already a fork or dev workspace of another workspace",
dev_w_id
)));
}
// The candidate can't itself be a prod with its own dev workspace (no nested dev chains).
ensure_no_existing_dev_workspace(&db, &dev_w_id).await?;
// Prod must be a root workspace, otherwise attaching could form a parent<->child cycle (e.g.
// attaching A as the dev of B when B is already the dev of A), which breaks hierarchy traversal.
let prod_has_parent = sqlx::query_scalar!(
r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#,
&prod_w_id
)
.fetch_optional(&db)
.await?
.ok_or_else(|| Error::NotFound(format!("Workspace {} not found", prod_w_id)))?;
if prod_has_parent {
return Err(Error::BadRequest(format!(
"Workspace {} is itself a fork or dev workspace and cannot be a prod workspace",
prod_w_id
)));
}
// The caller must be admin of the dev workspace too (or a superadmin).
let is_admin_of_dev = sqlx::query_scalar!(
"SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2",
&dev_w_id,
&authed.email
)
.fetch_optional(&db)
.await?
.unwrap_or(false);
if !is_admin_of_dev && !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? {
return Err(Error::PermissionDenied(format!(
"Attaching workspace '{dev_w_id}' as a dev requires being an admin of it (or a superadmin)"
)));
}
ensure_no_existing_dev_workspace(&db, &prod_w_id).await?;
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2",
&prod_w_id,
&dev_w_id
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2",
&prod_w_id,
&dev_w_id
)
.execute(&mut *tx)
.await?;
if req.lock_prod_deploy || req.lock_prod_forking {
lock_prod_workspace(
&mut tx,
&prod_w_id,
req.lock_prod_deploy,
req.lock_prod_forking,
)
.await?;
}
audit_log(
&mut *tx,
&authed,
"workspaces.attach_dev_workspace",
ActionKind::Update,
&prod_w_id,
Some(&dev_w_id),
None,
)
.await?;
tx.commit().await?;
// The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping
// so per-workspace job tags route to the prod family immediately rather than after the TTL.
windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id);
if req.lock_prod_deploy || req.lock_prod_forking {
windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id);
}
Ok(format!(
"Attached {} as dev workspace of {}",
dev_w_id, prod_w_id
))
}
/// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the
/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork).
async fn detach_dev_workspace(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(prod_w_id): Path<String>,
Json(req): Json<DetachDevWorkspace>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let dev_w_id = req.dev_workspace_id;
let is_dev_of_prod = sqlx::query_scalar!(
r#"SELECT EXISTS(
SELECT 1 FROM workspace
WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace
)"#,
&dev_w_id,
&prod_w_id
)
.fetch_one(&db)
.await?
.unwrap_or(false);
if !is_dev_of_prod {
return Err(Error::BadRequest(format!(
"{} is not the dev workspace of {}",
dev_w_id, prod_w_id
)));
}
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE workspace SET is_dev_workspace = false WHERE id = $1",
&dev_w_id
)
.execute(&mut *tx)
.await?;
// Only one dev per prod, so detaching it means prod no longer has a dev: drop the lock rule.
sqlx::query!(
"DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2",
&prod_w_id,
DEV_WORKSPACE_LOCK_RULE_NAME
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed,
"workspaces.detach_dev_workspace",
ActionKind::Update,
&prod_w_id,
Some(&dev_w_id),
None,
)
.await?;
tx.commit().await?;
windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id);
Ok(format!(
"Detached dev workspace {} from {}",
dev_w_id, prod_w_id
))
}
async fn edit_workspace(
authed: ApiAuthed,
Extension(db): Extension<DB>,
@@ -5250,6 +5613,10 @@ pub(crate) async fn archive_workspace_impl(
db: &DB,
w_id: &str,
username: &str,
// When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the
// prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the
// archive — a later failure can't strand a half-archived dev that's still flagged/locked.
dev_lock_parent: Option<&str>,
) -> Result<(usize, usize, usize)> {
// Step 1: Disable all schedules and clear their queued jobs
let mut tx = db.begin().await?;
@@ -5291,6 +5658,30 @@ pub(crate) async fn archive_workspace_impl(
.execute(&mut *tx)
.await?;
if let Some(prod) = dev_lock_parent {
// Dissolve the dev pairing atomically with the archive: clear the canonical-dev flag (so the
// archived row no longer occupies the parent's one-dev slot), and drop the prod's lock unless a
// replacement dev already holds it (NOT EXISTS sees the just-cleared flag within this tx, so the
// row being archived doesn't count).
sqlx::query!(
"UPDATE workspace SET is_dev_workspace = false WHERE id = $1",
w_id
)
.execute(&mut *tx)
.await?;
sqlx::query!(
"DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2
AND NOT EXISTS (
SELECT 1 FROM workspace
WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false
)",
prod,
DEV_WORKSPACE_LOCK_RULE_NAME
)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
// Step 2: Get all remaining queued jobs for this workspace (non-schedule jobs)
@@ -5334,8 +5725,39 @@ async fn archive_workspace(
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
// If this is an attached dev workspace, archiving it leaves the prod with no active dev (the
// unique index and user_workspaces both ignore deleted=true), so clear the prod's
// dev_workspace_lock too. Gate it on prod-admin since it removes prod's protection rule (mirrors
// detach/delete) — a dev-admin who isn't a prod-admin must not be able to unlock prod this way.
let dev_lock_parent: Option<String> = sqlx::query_scalar!(
"SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace",
&w_id
)
.fetch_optional(&db)
.await?
.flatten();
if let Some(ref prod) = dev_lock_parent {
let is_prod_admin = sqlx::query_scalar!(
"SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2",
prod,
&authed.email
)
.fetch_optional(&db)
.await?
.unwrap_or(false);
if !is_prod_admin
&& !windmill_common::auth::is_super_admin_email(&db, &authed.email).await?
{
return Err(Error::PermissionDenied(format!(
"Archiving dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)"
)));
}
}
// The dev pairing teardown (clear is_dev + drop the prod lock) runs inside archive_workspace_impl's
// transaction, atomically with `deleted = true`.
let (schedules_count, canceled_count, deleted_tokens_count) =
archive_workspace_impl(&db, &w_id, &authed.username).await?;
archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?;
// Audit log
let mut tx = db.begin().await?;
@@ -5370,6 +5792,10 @@ async fn archive_workspace(
.await?;
tx.commit().await?;
if let Some(prod) = dev_lock_parent {
windmill_common::workspaces::invalidate_protection_rules_cache(&prod);
}
Ok(format!(
"Archived workspace {}, disabled {} schedules, canceled {} jobs and deleted {} tokens",
&w_id, schedules_count, canceled_count, deleted_tokens_count
@@ -6211,6 +6637,119 @@ async fn list_protection_rules(
))
}
/// Insert or replace a protection ruleset within an existing transaction. Unlike the
/// `create_protection_rule` handler (which rejects an existing name), this upserts, so it is safe to
/// call programmatically when designating a dev/prod pair. Callers MUST invalidate the
/// protection-rules cache (`invalidate_protection_rules_cache`) after the transaction commits.
async fn upsert_protection_rule(
tx: &mut Transaction<'_, Postgres>,
w_id: &str,
name: &str,
rules: ProtectionRules,
bypass_groups: &[String],
bypass_users: &[String],
) -> Result<()> {
sqlx::query!(
r#"
INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (workspace_id, name)
DO UPDATE SET rules = EXCLUDED.rules,
bypass_groups = EXCLUDED.bypass_groups,
bypass_users = EXCLUDED.bypass_users
"#,
w_id,
name,
rules.bits(),
bypass_groups,
bypass_users,
)
.execute(&mut **tx)
.await?;
Ok(())
}
/// Lock a prod workspace by applying the reserved dev-workspace lock rule with the selected
/// restrictions (block direct deployment and/or ad-hoc forking). Non-admins are then funneled
/// through the one dev workspace; admins bypass the rules (their existing escape hatch).
async fn lock_prod_workspace(
tx: &mut Transaction<'_, Postgres>,
prod_w_id: &str,
block_deploy: bool,
block_forking: bool,
) -> Result<()> {
let mut rules = Vec::new();
if block_deploy {
rules.push(ProtectionRuleKind::DisableDirectDeployment);
}
if block_forking {
rules.push(ProtectionRuleKind::DisableWorkspaceForking);
}
if rules.is_empty() {
return Ok(());
}
upsert_protection_rule(
tx,
prod_w_id,
DEV_WORKSPACE_LOCK_RULE_NAME,
ProtectionRules::from(&rules),
&[],
&[],
)
.await
}
/// Error out if `parent_w_id` already has an active (non-archived) dev workspace. Mirrors the
/// partial unique index `workspace_canonical_dev_idx` with a friendly message.
async fn ensure_no_existing_dev_workspace(db: &DB, parent_w_id: &str) -> Result<()> {
let existing = sqlx::query_scalar!(
"SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false",
parent_w_id
)
.fetch_optional(db)
.await?;
if let Some(existing) = existing {
return Err(Error::BadRequest(format!(
"Workspace '{}' already has a dev workspace ('{}'). Detach it before creating another.",
parent_w_id, existing
)));
}
Ok(())
}
/// A dev workspace pairs with a root prod workspace; nesting dev workspaces (a dev of a dev) isn't
/// supported and would muddle the prod<->dev relationship.
async fn ensure_dev_parent_is_root(db: &DB, parent_w_id: &str) -> Result<()> {
let parent_is_fork = sqlx::query_scalar!(
r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#,
parent_w_id
)
.fetch_optional(db)
.await?
.unwrap_or(false);
if parent_is_fork {
return Err(Error::BadRequest(format!(
"Cannot create a dev workspace of '{}' because it is itself a fork or dev workspace.",
parent_w_id
)));
}
Ok(())
}
/// `dev_workspace_lock` is owned by the dev-workspace feature (attach/detach/archive/delete create and
/// remove it by name). Reserve it from the public protection-rule API so a user-managed rule can't
/// collide: otherwise the feature's name-based cleanup would clobber the user's rule, or a manual edit
/// could weaken the feature's lock.
fn reject_reserved_rule_name(name: &str) -> Result<()> {
if name == DEV_WORKSPACE_LOCK_RULE_NAME {
return Err(Error::BadRequest(format!(
"'{}' is a reserved protection-rule name managed by the dev workspace feature",
DEV_WORKSPACE_LOCK_RULE_NAME
)));
}
Ok(())
}
/// Create a new protection rule
async fn create_protection_rule(
authed: ApiAuthed,
@@ -6219,6 +6758,7 @@ async fn create_protection_rule(
Json(req): Json<CreateProtectionRuleRequest>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
reject_reserved_rule_name(&req.name)?;
let mut tx = db.begin().await?;
@@ -6293,6 +6833,7 @@ async fn update_protection_rule(
Json(req): Json<UpdateProtectionRuleRequest>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
reject_reserved_rule_name(&rule_name)?;
let mut tx = db.begin().await?;
@@ -6367,6 +6908,7 @@ async fn delete_protection_rule(
Path((w_id, rule_name)): Path<(String, String)>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
reject_reserved_rule_name(&rule_name)?;
let mut tx = db.begin().await?;
@@ -6511,10 +7053,22 @@ async fn compare_workspaces(
}));
}
// Honor ws_specific at read time: a workspace-specific resource/variable keeps its own value per
// environment, so it must never appear in the normal diff (the per-item compare suppresses it,
// but a cached `has_changes=true` row is trusted without re-running that compare, so filter those
// here too). Seeding the initial copy onto a side that lacks it is a separate explicit action
// (the "Create in <other>" button on the Workspace-specific list), not part of the diff. The row
// is left intact, so unpinning resurfaces it without a re-tally.
let diff_items = sqlx::query_as!(
WorkspaceDiffRow,
"SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff
WHERE source_workspace_id = $1 AND fork_workspace_id = $2",
WHERE source_workspace_id = $1 AND fork_workspace_id = $2
AND NOT EXISTS (
SELECT 1 FROM ws_specific ws
WHERE ws.path = workspace_diff.path
AND ws.item_kind = workspace_diff.kind
AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id)
)",
source_workspace_id,
fork_workspace_id,
)
@@ -7242,7 +7796,6 @@ async fn compare_two_resources(
.fetch_optional(db)
.await?;
// If either side is ws_specific, consider unchanged
let source_ws_specific = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)",
source_workspace_id,
@@ -7261,6 +7814,9 @@ async fn compare_two_resources(
.await?
.unwrap_or(false);
// A workspace-specific resource keeps its own value per environment, so it never appears in the
// diff (in either direction). Seeding the initial copy onto a side that lacks it is a separate
// explicit action ("Create in <other>"), not a diff entry.
if source_ws_specific || target_ws_specific {
return Ok(ItemComparison {
has_changes: false,
@@ -7317,7 +7873,8 @@ async fn compare_two_variables(
.fetch_one(db)
.await?;
// If either side is ws_specific, consider unchanged
// A workspace-specific variable keeps its own value per environment, so it never appears in the
// diff. Seeding the initial copy onto a side that lacks it is a separate explicit action.
if presence.src_ws || presence.tgt_ws {
return Ok(ItemComparison {
has_changes: false,
@@ -7831,6 +8388,19 @@ async fn list_ws_specific(
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
// RLS gates membership/folder access, but a scoped API token must also be held to its read
// scopes — mirror the resource/variable list endpoints, which filter with these predicates so a
// token lacking `resources:read:*` / `variables:read:*` can't enumerate pinned paths it can't read.
let resource_allowed = build_scope_path_predicate(&authed, "resources", "read");
let variable_allowed = build_scope_path_predicate(&authed, "variables", "read");
let items = items
.into_iter()
.filter(|it| match it.item_kind.as_str() {
"resource" => resource_allowed(&it.path),
"variable" => variable_allowed(&it.path),
_ => false,
})
.collect::<Vec<_>>();
Ok(Json(items))
}
@@ -7853,6 +8423,17 @@ async fn list_ws_specific_versions(
)));
}
// A scoped API token must hold the read scope for this path, like the resource/variable read
// endpoints. Without the scope, report no versions rather than leaking the path's history.
let domain = if q.kind == "resource" {
"resources"
} else {
"variables"
};
if !build_scope_path_predicate(&authed, domain, "read")(&q.path) {
return Ok(Json(vec![]));
}
let versions: Vec<String> = sqlx::query_scalar!(
r#"SELECT ws AS "ws!" FROM list_ws_specific_versions($1, $2, $3, $4)"#,
&w_id,
@@ -7865,3 +8446,155 @@ async fn list_ws_specific_versions(
Ok(Json(versions))
}
#[derive(Deserialize)]
struct SetWsSpecificBody {
item_kind: String,
path: String,
value: bool,
}
/// Mark (or unmark) a single resource/variable as workspace-specific. Pinning
/// excludes it from the deploy diff so each environment keeps its own value
/// (see `compare_two_resources`/`compare_two_variables`). Set per-workspace, so
/// the compare page calls this once per side to flag both environments.
async fn set_ws_specific(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(body): Json<SetWsSpecificBody>,
) -> Result<String> {
if body.item_kind != "resource" && body.item_kind != "variable" {
return Err(Error::BadRequest(format!(
"Invalid kind '{}'. Must be 'resource' or 'variable'",
body.item_kind
)));
}
// Reject a malformed path with a 400 before the auth check, which indexes the leading segments and
// would otherwise panic (500) on a path like `u` with no segment. Accept all three shared path
// shapes Windmill uses — `u/<user>`, `f/<folder>`, `g/<group>` (e.g. seeded `g/all/...` resources).
let segs: Vec<&str> = body.path.split('/').collect();
if segs.len() < 2 || !matches!(segs[0], "u" | "f" | "g") || segs[1].is_empty() {
return Err(Error::BadRequest(format!(
"Invalid {} path: {}",
body.item_kind, body.path
)));
}
// Authorize like the resource/variable editors' own ws_specific toggle:
// actual write access to the item + token scope + the workspace deploy rules.
// `require_owner_of_path` is the real write gate (the resource editor uses it);
// `check_scopes` only constrains scoped tokens (it is a no-op for session/cookie
// logins). Together: a non-admin who can edit the item may pin it, while a
// read-only member is rejected and a locked workspace still blocks non-deployers.
// `require_is_writer` matches the resource/variable editors' write semantics (owner, folder
// writer, or item writer via extra_perms) — not owner-only.
let writer_query = if body.item_kind == "resource" {
"SELECT extra_perms FROM resource WHERE path = $1 AND workspace_id = $2"
} else {
"SELECT extra_perms FROM variable WHERE path = $1 AND workspace_id = $2"
};
require_is_writer(
&authed,
&body.path,
&w_id,
db.clone(),
writer_query,
&body.item_kind,
)
.await?;
check_scopes(&authed, || {
format!("{}s:write:{}", body.item_kind, body.path)
})?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
&authed.username,
&authed.groups,
authed.is_admin,
&db,
)
.await?
{
return Err(Error::PermissionDenied(msg));
}
let mut tx = user_db.begin(&authed).await?;
if body.value {
// Existence guard keeps a dangling marker from being created for a
// path absent in this workspace.
if body.item_kind == "resource" {
sqlx::query!(
"INSERT INTO ws_specific (workspace_id, item_kind, path)
SELECT $1::varchar, 'resource', $2::varchar
WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar)
ON CONFLICT DO NOTHING",
w_id,
body.path,
)
.execute(&mut *tx)
.await?;
// A resource owns its `$var:` secrets, so pin those too.
windmill_store::resources::mark_linked_variables_ws_specific(
&mut tx, &authed, &w_id, &body.path,
)
.await?;
} else {
sqlx::query!(
"INSERT INTO ws_specific (workspace_id, item_kind, path)
SELECT $1::varchar, 'variable', $2::varchar
WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar)
ON CONFLICT DO NOTHING",
w_id,
body.path,
)
.execute(&mut *tx)
.await?;
}
} else {
// Unmark only this item; linked variables stay flagged (they may be
// referenced by other resources) — mirrors the resource-form toggle.
sqlx::query!(
"DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3",
w_id,
body.item_kind,
body.path,
)
.execute(&mut *tx)
.await?;
// While pinned, the item's cached workspace_diff verdict was never recomputed (the compare
// read filter excludes it), so it may now be stale in either direction. Mark it NULL so the
// next compare re-evaluates from scratch — and the now-shared item reappears (or is dropped)
// correctly instead of being stuck on its pre-pin verdict.
sqlx::query!(
"UPDATE workspace_diff SET has_changes = NULL
WHERE path = $2 AND kind = $3
AND ($1 IN (source_workspace_id, fork_workspace_id))",
w_id,
body.path,
body.item_kind,
)
.execute(&mut *tx)
.await?;
}
let value_str = body.value.to_string();
audit_log(
&mut *tx,
&authed,
&format!("{}s.set_ws_specific", body.item_kind),
ActionKind::Update,
&w_id,
Some(&body.path),
Some([("value", value_str.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(format!(
"Set workspace-specific={} for {} {}",
body.value, body.item_kind, body.path
))
}
backend/windmill-api-workspaces/src/workspaces_extra.rs
+153 -18
View File
@@ -1,7 +1,6 @@
use std::collections::HashMap;
use windmill_api_auth::{require_super_admin, ApiAuthed};
use windmill_common::workspaces::WM_FORK_PREFIX;
use windmill_common::DB;
use crate::workspaces::{
@@ -26,7 +25,7 @@ use windmill_common::{
db::UserDB,
error::{Error, Result},
utils::require_admin,
workspaces::DataTable,
workspaces::{DataTable, DEV_WORKSPACE_LOCK_RULE_NAME, WM_FORK_PREFIX},
};
use windmill_queue::schedule::{get_schedule_opt, push_scheduled_job};
@@ -65,22 +64,42 @@ pub(crate) async fn change_workspace_id(
old_id, rw.new_id
);
// Create new workspace with new id and name. A fork that keeps a wm-fork-
// id must carry its parent_workspace_id over, otherwise it becomes a
// parentless "fork of nothing" with no source to compare or merge against.
// A non-fork target id means the workspace is being promoted out of a fork,
// so the parent pointer is intentionally cleared.
// Create new workspace with new id and name. Fork lineage AND the dev designation are preserved
// from the source row, not inferred from the new id's prefix: a prefix-less fork (a dev or
// detached-dev workspace) would otherwise be silently promoted to a root workspace, and a dev
// would lose its flag — leaving its prod locked with no canonical dev. Promoting out of a fork is
// a separate, explicit action — a rename never does it implicitly.
info!("Creating new workspace row");
let new_is_fork = rw.new_id.starts_with(WM_FORK_PREFIX);
let old = sqlx::query!(
r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!", is_dev_workspace
FROM workspace WHERE id = $1"#,
&old_id
)
.fetch_optional(&mut *tx)
.await?;
let new_is_fork = old.as_ref().map(|o| o.has_parent).unwrap_or(false);
let new_is_dev = new_is_fork && old.as_ref().map(|o| o.is_dev_workspace).unwrap_or(false);
// Neutralize the old row's dev flag BEFORE inserting the new one: the move-and-archive archives
// the old row only later, so without this the new dev row and the not-yet-archived old dev row
// would momentarily both be active under the same parent and trip the one-dev-per-parent index.
if new_is_dev {
sqlx::query!(
"UPDATE workspace SET is_dev_workspace = false WHERE id = $1",
&old_id
)
.execute(&mut *tx)
.await?;
}
sqlx::query!(
"INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id)
"INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace)
SELECT $1, $2, owner, false, premium,
CASE WHEN $4 THEN parent_workspace_id ELSE NULL END
CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5
FROM workspace WHERE id = $3",
&rw.new_id,
&rw.new_name,
&old_id,
new_is_fork
new_is_fork,
new_is_dev
)
.execute(&mut *tx)
.await?;
@@ -360,8 +379,18 @@ pub(crate) async fn change_workspace_id(
// must follow the renamed parent to the new id, otherwise it is left
// pointing at the soft-deleted old shell (whose data has moved here).
info!("Re-parenting child forks to the new workspace id");
let reparented_children: Vec<String> = sqlx::query_scalar!(
"UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id",
&rw.new_id,
&old_id
)
.fetch_all(&mut *tx)
.await?;
// A dev/fork's `deploy_to` points at the prod root, so it must follow the rename too — otherwise
// the child re-parents to the new id but still deploys to the soft-deleted old shell.
sqlx::query!(
"UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2",
"UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2",
&rw.new_id,
&old_id
)
@@ -696,11 +725,18 @@ pub(crate) async fn change_workspace_id(
tx.commit().await?;
// The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent
// routing cache so jobs route under the renamed root rather than the old (archived) one until the
// 300s TTL would otherwise expire.
for child in &reparented_children {
windmill_queue::tags::invalidate_fork_parent_cache(child);
}
// Archive old workspace: disable schedules, cancel remaining jobs, set deleted=true
// Note: schedules were already moved to new workspace, so this will find 0 schedules
info!("Archiving old workspace");
let (_schedules_count, canceled_count, _deleted_tokens_count) =
archive_workspace_impl(&db, &old_id, &authed.username).await?;
archive_workspace_impl(&db, &old_id, &authed.username, None).await?;
info!(
"Workspace id change completed: moved {} to {}, archived old workspace",
@@ -734,15 +770,65 @@ pub(crate) async fn delete_workspace(
_ => Ok(w_id),
}?;
if dwq.only_delete_forks.unwrap_or(false) && !w_id.starts_with(WM_FORK_PREFIX) {
let is_fork = workspace_is_fork(&db, &w_id).await?;
if dwq.only_delete_forks.unwrap_or(false) && !is_fork {
return Err(Error::BadRequest(
"Cannot delete this workspace because it is not a workspace fork.".to_string(),
));
}
let mut tx = db.begin().await?;
if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) {
require_super_admin(&db, &authed.email).await?;
if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?)
&& !is_super_admin_email(&db, &authed.email).await?
{
return Err(Error::PermissionDenied(
"Deleting this workspace requires being the fork's owner or a superadmin".to_string(),
));
}
// Don't hard-delete a workspace that still has a dev workspace paired to it: the FK is
// ON DELETE SET NULL, which would orphan the (prefix-less) dev into a parentless, non-fork row
// its owner could no longer self-delete. Require detaching/deleting the dev first. Ordinary
// forks have no such guard — they keep their prefix and stay owner-deletable when orphaned.
// Archived devs (deleted = true) are included: they keep is_dev_workspace = true, so SET NULL on
// their parent would violate the `is_dev ⇒ has parent` CHECK and fail the whole delete with a 500.
if let Some(dev_id) = sqlx::query_scalar!(
"SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace",
&w_id
)
.fetch_optional(&mut *tx)
.await?
{
return Err(Error::BadRequest(format!(
"Cannot delete workspace '{}' because it has a dev workspace ('{}'). Detach or delete the dev workspace first.",
w_id, dev_id
)));
}
// Deleting an attached dev workspace removes the parent prod's dev_workspace_lock (below), so it
// must be a prod-admin action, not just the dev's own owner (dev ownership can diverge from
// prod's) — mirrors detach_dev_workspace, which is prod-admin gated.
if let Some(prod) = sqlx::query_scalar!(
"SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace",
&w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten()
{
let is_prod_admin = sqlx::query_scalar!(
"SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2",
&prod,
&authed.email
)
.fetch_optional(&mut *tx)
.await?
.unwrap_or(false);
if !is_prod_admin && !is_super_admin_email(&db, &authed.email).await? {
return Err(Error::PermissionDenied(format!(
"Deleting dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)"
)));
}
}
sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id)
@@ -920,10 +1006,31 @@ pub(crate) async fn delete_workspace(
.execute(&mut *tx)
.await?;
// If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop
// the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod
// stays locked against direct deploy/forking with no dev workspace left to make changes in.
let dev_lock_parent: Option<String> = sqlx::query_scalar!(
"SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace",
&w_id
)
.fetch_optional(&mut *tx)
.await?
.flatten();
sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id)
.execute(&mut *tx)
.await?;
if let Some(ref parent) = dev_lock_parent {
sqlx::query!(
"DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2",
parent,
DEV_WORKSPACE_LOCK_RULE_NAME
)
.execute(&mut *tx)
.await?;
}
// Record under the instance-level "admins" workspace. The per-workspace audit
// rows are deleted along with the workspace, so this instance-level entry is the
// only durable, superadmin-discoverable record of who deleted the workspace.
@@ -939,6 +1046,10 @@ pub(crate) async fn delete_workspace(
.await?;
tx.commit().await?;
if let Some(parent) = dev_lock_parent {
windmill_common::workspaces::invalidate_protection_rules_cache(&parent);
}
Ok(format!("Deleted workspace {}", &w_id))
}
@@ -957,9 +1068,15 @@ pub async fn drop_forked_datatable_databases(
Json(req): Json<DropForkedDatatableDatabasesRequest>,
) -> Result<Json<Vec<String>>> {
// Same permission check as delete_workspace: fork owner or super admin
let is_fork = workspace_is_fork(&db, &w_id).await?;
let mut tx = db.begin().await?;
if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) {
require_super_admin(&db, &authed.email).await?;
if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?)
&& !is_super_admin_email(&db, &authed.email).await?
{
return Err(Error::PermissionDenied(
"Dropping forked datatable databases requires being the fork's owner or a superadmin"
.to_string(),
));
}
tx.commit().await?;
@@ -1103,3 +1220,21 @@ async fn is_workspace_owner(
.await?;
Ok(owner.map(|o| o == authed.email).unwrap_or(false))
}
/// Whether a workspace is a fork or dev workspace. Both forks and dev workspaces set
/// `parent_workspace_id`, but a `wm-fork-` workspace can outlive its parent (the FK is
/// `ON DELETE SET NULL`), so also treat the prefix as fork-ness — otherwise an orphaned fork would
/// lose owner-self-delete. Used to gate owner-self-delete, which is permitted for forks/dev
/// workspaces but requires superadmin otherwise.
async fn workspace_is_fork(db: &DB, w_id: &str) -> Result<bool> {
if w_id.starts_with(WM_FORK_PREFIX) {
return Ok(true);
}
Ok(sqlx::query_scalar!(
r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or(false))
}
backend/windmill-api/openapi.yaml
+242 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.742.0
version: 1.744.0
title: Windmill API
contact:
@@ -1184,6 +1184,89 @@ paths:
schema:
type: string
/w/{workspace}/workspaces/attach_dev_workspace:
post:
summary: attach an existing workspace as the dev workspace of this (prod) workspace
operationId: attachDevWorkspace
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
dev_workspace_id:
type: string
lock_prod_deploy:
type: boolean
lock_prod_forking:
type: boolean
required:
- dev_workspace_id
responses:
"200":
description: dev workspace attached
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/detach_dev_workspace:
post:
summary: detach the dev workspace from this (prod) workspace
operationId: detachDevWorkspace
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
dev_workspace_id:
type: string
required:
- dev_workspace_id
responses:
"200":
description: dev workspace detached
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/get_dev_workspace:
get:
summary: this workspace's active canonical dev workspace, if any
operationId: getDevWorkspace
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: the dev workspace, or null if none
content:
application/json:
schema:
nullable: true
type: object
properties:
id:
type: string
name:
type: string
required:
- id
- name
/workspaces/exists:
post:
summary: exists workspace
@@ -6238,6 +6321,42 @@ paths:
items:
type: string
/w/{workspace}/workspaces/set_ws_specific:
post:
summary: mark or unmark an item as workspace-specific
operationId: setWsSpecific
tags:
- workspace
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
item_kind:
type: string
enum:
- resource
- variable
path:
type: string
value:
type: boolean
required:
- item_kind
- path
- value
responses:
"200":
description: workspace-specific flag updated
content:
text/plain:
schema:
type: string
/w/{workspace}/workspaces/public_app_rate_limit:
post:
summary: Set public app rate limit for this workspace
@@ -21355,6 +21474,54 @@ paths:
format: int64
description: Number of pipeline-member scripts in the folder
/w/{workspace}/assets/partitions:
get:
summary: List materialized partitions for a ducklake asset
operationId: listAssetPartitions
tags:
- asset
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: path
in: query
required: true
description: The materialized ducklake asset path (`<ducklake>/<table>`)
schema:
type: string
responses:
"200":
description: per-partition materialization status
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/MaterializedPartition"
/w/{workspace}/assets/asset_schemas:
get:
summary: List captured output-schema versions for a ducklake asset
operationId: listAssetSchemas
tags:
- asset
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: path
in: query
required: true
description: The materialized ducklake asset path (`<ducklake>/<table>`)
schema:
type: string
responses:
"200":
description: captured schema versions, newest first
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/AssetSchemaVersion"
/w/{workspace}/volumes/list:
get:
summary: List all volumes in the workspace
@@ -27756,6 +27923,8 @@ components:
parent_workspace_id:
type: string
nullable: true
is_dev_workspace:
type: boolean
created_by:
type: string
nullable: true
@@ -27766,6 +27935,7 @@ components:
- name
- username
- color
- is_dev_workspace
- disabled
required:
- email
@@ -27807,6 +27977,18 @@ components:
new_dbname:
type: string
description: "New database name for the fork"
is_dev_workspace:
type: boolean
description: "Create the fork as a persistent dev workspace (id not required to carry the wm-fork- prefix; at most one per parent)"
lock_prod_deploy:
type: boolean
description: "When creating a dev workspace, lock the parent (prod) against direct deployment"
lock_prod_forking:
type: boolean
description: "When creating a dev workspace, prevent forking the parent (prod)"
copy_members:
type: boolean
description: "Copy the parent's members (users + group memberships) into the fork so the team can work in it"
required:
- id
- name
@@ -29491,6 +29673,65 @@ components:
- ducklake
- datatable
- volume
MaterializedPartition:
type: object
required: [asset_kind, asset_path, partition, status, materialized_at]
properties:
asset_kind:
$ref: "#/components/schemas/AssetKind"
asset_path:
type: string
partition:
type: string
status:
type: string
enum: [running, materialized, failed]
snapshot_id:
type: integer
format: int64
nullable: true
row_count:
type: integer
format: int64
nullable: true
job_id:
type: string
format: uuid
nullable: true
materialized_at:
type: string
format: date-time
error:
type: string
nullable: true
AssetSchemaVersion:
type: object
required: [version, columns, captured_at]
properties:
version:
type: integer
format: int64
columns:
type: array
items:
type: object
required: [name, type]
properties:
name:
type: string
type:
type: string
snapshot_id:
type: integer
format: int64
nullable: true
job_id:
type: string
format: uuid
nullable: true
captured_at:
type: string
format: date-time
Asset:
type: object
properties:
backend/windmill-api/src/mcp/utils.rs
+60 -3
View File
@@ -382,12 +382,18 @@ pub fn build_query_string(
.map(|value| {
// Use the original name for the query parameter key
let original_name = get_original_name(param_name, query_field_renames);
let value_str = value.to_string();
let str_val = value_str.trim_matches('"');
// For string values, use the raw content: to_string() would JSON-encode
// it, and stripping the outer quotes leaves inner quotes backslash-escaped
// (e.g. `{\"k\":\"v\"}`), which breaks downstream JSON parsing of params
// like `args`/`result`. Non-string values keep their JSON serialization.
let str_val = value
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| value.to_string());
format!(
"{}={}",
urlencoding::encode(&original_name),
urlencoding::encode(str_val)
urlencoding::encode(&str_val)
)
})
})
@@ -624,4 +630,55 @@ mod tests {
.expect("legitimate path should substitute");
assert_eq!(result, "/w/dev/scripts/get/p/u/alice/my_script");
}
fn single_query_schema(param: &str) -> Option<Value> {
Some(json!({
"type": "object",
"properties": { param: { "type": "string" } }
}))
}
#[test]
fn build_query_string_preserves_json_string_content() {
// A string param carrying JSON (e.g. the `args` filter on listJobs) must be
// emitted as its raw content so the backend can `serde_json::from_str` it.
let mut args = serde_json::Map::new();
args.insert("args".to_string(), json!("{\"key\":\"val\"}"));
let qs = build_query_string(&args, &single_query_schema("args"), &None);
// No backslash escaping: %5C must not appear; the encoded braces/quotes are exact.
assert_eq!(qs, "?args=%7B%22key%22%3A%22val%22%7D");
assert!(
!qs.contains("%5C"),
"must not contain backslash escapes: {qs}"
);
}
#[test]
fn build_query_string_keeps_non_string_serialization() {
let mut args = serde_json::Map::new();
args.insert("per_page".to_string(), json!(42));
assert_eq!(
build_query_string(&args, &single_query_schema("per_page"), &None),
"?per_page=42"
);
let mut args = serde_json::Map::new();
args.insert("running".to_string(), json!(true));
assert_eq!(
build_query_string(&args, &single_query_schema("running"), &None),
"?running=true"
);
}
#[test]
fn build_query_string_encodes_plain_string() {
let mut args = serde_json::Map::new();
args.insert("path".to_string(), json!("u/alice/my script"));
assert_eq!(
build_query_string(&args, &single_query_schema("path"), &None),
"?path=u%2Falice%2Fmy%20script"
);
}
}
backend/windmill-common/src/ee_oss.rs
+9
View File
@@ -20,6 +20,7 @@ lazy_static::lazy_static! {
pub static ref LICENSE_KEY: arc_swap::ArcSwap<String> = arc_swap::ArcSwap::from_pointee("".to_string());
pub static ref LICENSE_OFFLINE_METADATA: arc_swap::ArcSwap<Option<OfflineMetadata>> = arc_swap::ArcSwap::from_pointee(None);
pub static ref LICENSE_OFFLINE_OVER_CU_CAP: AtomicBool = AtomicBool::new(false);
pub static ref LICENSE_OFFLINE_OVER_SEAT_CAP: AtomicBool = AtomicBool::new(false);
pub static ref LICENSE_OFFLINE_LAST_STATUS: arc_swap::ArcSwap<Option<OfflineCapStatus>> = arc_swap::ArcSwap::from_pointee(None);
pub static ref LICENSE_OFFLINE_LAST_CHECKED_AT: arc_swap::ArcSwap<Option<chrono::DateTime<chrono::Utc>>> = arc_swap::ArcSwap::from_pointee(None);
}
@@ -62,6 +63,14 @@ pub async fn check_seat_cap_for_new_user(
Ok(None)
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn check_seat_cap_for_reactivation(
_db: &DB,
_email: &str,
) -> anyhow::Result<Option<String>> {
Ok(None)
}
#[cfg(all(feature = "enterprise", not(feature = "private")))]
pub async fn compute_instance_hash(_db: &DB) -> anyhow::Result<Option<String>> {
// Implementation is not open source
backend/windmill-common/src/lib.rs
+238 -32
View File
@@ -564,6 +564,132 @@ mod validate_dbname_tests {
}
}
#[cfg(test)]
mod pg_tls_tests {
use super::PgDatabase;
// A syntactically valid (self-signed) certificate, used only to exercise the
// "root certificate supplied" branch — its contents are never validated here.
const VALID_PEM: &str = "-----BEGIN CERTIFICATE-----\n\
MIIDETCCAfmgAwIBAgIUX/yHsMoWBljFzJr5Xh7V2I6ykMEwDQYJKoZIhvcNAQEL\n\
BQAwGDEWMBQGA1UEAwwNd2luZG1pbGwtdGVzdDAeFw0yNjA2MjkwOTUwNTlaFw0z\n\
NjA2MjYwOTUwNTlaMBgxFjAUBgNVBAMMDXdpbmRtaWxsLXRlc3QwggEiMA0GCSqG\n\
SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvF2hMw8adQGG6EnDk8GsOIoHT+kLN1W0F\n\
yYFwH1wGVmzVP1YNfUts8aQfMtl/ZjW7SQlvKeK+18id4fVNYvZpbFhj66IsKMOU\n\
MnJHcC6X/IAdhANyhM1fcrS6YupanAKOhLPk4HYRD5tGI4Y1vzTnQKGffIZ0bof7\n\
3GtCiJLv8wrJKszeoKPtdFazdW+CYePbFq3Owc7HMo8CwA7A5TsgcowELhCfYwZv\n\
Pn/9v+NDHQO0jJclH7qK221RkbqZGD+nPJ4rUm7oRi0vfApBQZ0FFJZjiki/Kg2+\n\
RACb6Ud/LOeRBerKQHbN8KeYnGafCaIC4s/XytVwxAz+kgK1qyl7AgMBAAGjUzBR\n\
MB0GA1UdDgQWBBRo2Jby4SZlrwMNbhA4bswZcBNRyjAfBgNVHSMEGDAWgBRo2Jby\n\
4SZlrwMNbhA4bswZcBNRyjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\n\
A4IBAQBlED+FQW3GB3Wa1NdVN252vihuFNnbq81yvhf4T7dfAxwkxI9jiM+ZWCw2\n\
g59FbLupj8Rwun5gE2H/9M8ZunISdlwaMH5nyDJlbRjttPfY1cEoyGEY+UXIslfg\n\
BoiI5rOtz9R2qurxEic1VtEVfXhEuWwCG86vCBDdHrL/qqqUJEx/P8qyC7uVc8XC\n\
uclnJVL7x1ax0jTmEPur9K+DQn2ws01mzpq2QwSunibpDL5D5xM1oYekv0tQFEkT\n\
ta9ELulniZau8zUAtwqwecxodzl+KO8NYj0a9PGgAM64dMqkRtRA8P4UP350Nag3\n\
+hOq1qpWD7yPVyycx/KCilICOKVf\n\
-----END CERTIFICATE-----\n";
fn pg(sslmode: Option<&str>, root_cert: Option<&str>) -> PgDatabase {
PgDatabase {
host: "db.example.com".to_string(),
user: Some("u".to_string()),
password: Some("p".to_string()),
port: Some(5432),
sslmode: sslmode.map(|s| s.to_string()),
dbname: "mydb".to_string(),
root_certificate_pem: root_cert.map(|s| s.to_string()),
accept_invalid_certs: None,
use_iam_auth: None,
region: None,
}
}
/// Whether the connector enforces certificate verification for the given config.
fn verifies(
sslmode: Option<&str>,
root_cert: Option<&str>,
accept_invalid_certs: Option<bool>,
) -> bool {
let mut builder = native_tls::TlsConnector::builder();
PgDatabase::configure_pg_tls_verification(
&mut builder,
sslmode,
root_cert,
accept_invalid_certs,
)
.unwrap()
}
#[test]
fn verify_modes_enforce_verification_when_explicitly_requested() {
// accept_invalid_certs=Some(false) is what newly created resources carry: it
// verifies even with no custom cert (against the OS trust store).
assert!(verifies(Some("verify-full"), None, Some(false)));
assert!(verifies(Some("verify-ca"), None, Some(false)));
assert!(verifies(Some("verify-full"), Some(""), Some(false)));
assert!(verifies(Some("verify-full"), Some(VALID_PEM), Some(false)));
assert!(verifies(Some("verify-ca"), Some(VALID_PEM), Some(false)));
}
#[test]
fn verify_modes_unset_fall_back_to_legacy_behavior() {
// Unset (None): verify iff a root cert is present — preserves the behavior of
// resources that predate the flag (incl. git-synced), so upgrades don't break.
assert!(!verifies(Some("verify-full"), None, None));
assert!(!verifies(Some("verify-ca"), None, None));
assert!(!verifies(Some("verify-full"), Some(""), None));
assert!(verifies(Some("verify-full"), Some(VALID_PEM), None));
assert!(verifies(Some("verify-ca"), Some(VALID_PEM), None));
}
#[test]
fn accept_invalid_certs_true_disables_verification_for_verify_modes() {
assert!(!verifies(Some("verify-full"), Some(VALID_PEM), Some(true)));
assert!(!verifies(Some("verify-ca"), None, Some(true)));
}
#[test]
fn accept_invalid_certs_is_ignored_outside_verify_modes() {
// require never consults the flag: it verifies iff a cert is present, and
// encrypts-without-verifying otherwise, regardless of accept_invalid_certs.
assert!(!verifies(Some("require"), None, Some(false)));
assert!(!verifies(Some("require"), None, Some(true)));
assert!(!verifies(None, None, Some(true)));
assert!(verifies(Some("require"), Some(VALID_PEM), Some(true)));
assert!(verifies(Some("require"), Some(VALID_PEM), None));
}
#[test]
fn invalid_pem_is_rejected() {
let mut builder = native_tls::TlsConnector::builder();
let err = PgDatabase::configure_pg_tls_verification(
&mut builder,
Some("verify-full"),
Some("not a certificate"),
Some(false),
);
assert!(err.is_err());
}
#[test]
fn to_uri_collapses_verify_modes_for_tokio_postgres() {
// to_uri() feeds tokio-postgres, which only parses disable/prefer/require;
// verify-* therefore map to require there (verification is connector-driven).
for mode in ["require", "verify-ca", "verify-full"] {
assert!(
pg(Some(mode), None).to_uri().contains("sslmode=require"),
"{mode} should map to sslmode=require in to_uri()"
);
}
assert!(pg(Some("disable"), None)
.to_uri()
.contains("sslmode=disable"));
assert!(pg(Some("allow"), None).to_uri().contains("sslmode=prefer"));
assert!(pg(None, None).to_uri().contains("sslmode=prefer"));
}
}
#[derive(Serialize, Debug)]
pub struct PrepareQueryColumnInfo {
pub name: String,
@@ -588,6 +714,12 @@ pub struct PgDatabase {
pub sslmode: Option<String>,
pub dbname: String,
pub root_certificate_pem: Option<String>,
/// Only meaningful for sslmode verify-ca/verify-full. `Some(true)` accepts any
/// server certificate (no chain or hostname check); `Some(false)` enforces
/// verification. `None` falls back to legacy behavior — verify only when a root
/// certificate is present — so resources that predate this flag (including
/// git-synced ones, whose source never sets it) keep working unchanged.
pub accept_invalid_certs: Option<bool>,
pub use_iam_auth: Option<bool>,
pub region: Option<String>,
}
@@ -676,10 +808,94 @@ impl PgDatabase {
}
}
/// True when sslmode requests verification (verify-ca/verify-full) but the
/// effective configuration disables it, so the server's identity is not
/// checked. Mirrors the verify-* decision in `configure_pg_tls_verification`.
pub fn verify_mode_skips_verification(&self) -> bool {
matches!(
self.sslmode.as_deref(),
Some("verify-ca") | Some("verify-full")
) && self.accept_invalid_certs.unwrap_or(
self.root_certificate_pem
.as_deref()
.unwrap_or("")
.is_empty(),
)
}
/// Configure certificate and hostname verification on a native-tls connector
/// according to the requested Postgres `sslmode`. The crates.io tokio-postgres
/// build only parses disable/prefer/require, so verify-ca and verify-full are
/// enforced here, on the connector, rather than through the connection URI.
///
/// verify-full — verify the certificate chain AND that it matches the host.
/// verify-ca — verify the chain only; libpq does not check the hostname.
/// require / other — encrypt without verifying identity, unless a root
/// certificate is supplied (then verify the chain).
///
/// `accept_invalid_certs` only applies to verify-ca/verify-full: `Some(true)`
/// accepts any certificate, `Some(false)` enforces verification, and `None`
/// falls back to the legacy behavior — verify only when a root certificate is
/// present — so resources predating the flag (including git-synced ones, whose
/// source never sets it) keep working unchanged. Verification uses the OS trust
/// store plus any supplied root certificate. Returns false when the connector
/// was set to accept any certificate, so callers can surface that an unverified
/// connection is being made.
fn configure_pg_tls_verification(
builder: &mut native_tls::TlsConnectorBuilder,
sslmode: Option<&str>,
root_certificate_pem: Option<&str>,
accept_invalid_certs: Option<bool>,
) -> Result<bool, error::Error> {
use native_tls::Certificate;
let custom_root = match root_certificate_pem {
Some(pem) if !pem.is_empty() => Some(
Certificate::from_pem(pem.as_bytes())
.map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?,
),
_ => None,
};
match sslmode {
Some("verify-full") | Some("verify-ca") => {
// Unset falls back to the legacy behavior: verify iff a cert is present.
if accept_invalid_certs.unwrap_or(custom_root.is_none()) {
builder
.danger_accept_invalid_certs(true)
.danger_accept_invalid_hostnames(true);
return Ok(false);
}
if let Some(cert) = custom_root {
builder.add_root_certificate(cert);
}
if sslmode == Some("verify-ca") {
// verify-ca verifies the chain but, per libpq, not the hostname.
builder.danger_accept_invalid_hostnames(true);
}
Ok(true)
}
_ => {
// "require": accept_invalid_certs does not apply. Encrypt but do not
// verify identity, unless an explicit root certificate was supplied
// (then verify the chain).
if let Some(cert) = custom_root {
builder.add_root_certificate(cert);
Ok(true)
} else {
builder
.danger_accept_invalid_certs(true)
.danger_accept_invalid_hostnames(true);
Ok(false)
}
}
}
}
async fn connect_inner(
&self,
) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> {
use native_tls::{Certificate, TlsConnector};
use native_tls::TlsConnector;
use postgres_native_tls::MakeTlsConnector;
use tokio_postgres::tls::NoTls;
let ssl_mode_is_require = matches!(
@@ -690,21 +906,17 @@ impl PgDatabase {
if ssl_mode_is_require {
tracing::info!("Creating new connection");
let mut connector = TlsConnector::builder();
if let Some(root_certificate_pem) = &self.root_certificate_pem {
if !root_certificate_pem.is_empty() {
connector.add_root_certificate(
Certificate::from_pem(root_certificate_pem.as_bytes()).map_err(|e| {
error::Error::BadConfig(format!("Invalid Certs: {e:#}"))
})?,
);
} else {
connector.danger_accept_invalid_certs(true);
connector.danger_accept_invalid_hostnames(true);
}
} else {
connector
.danger_accept_invalid_certs(true)
.danger_accept_invalid_hostnames(true);
Self::configure_pg_tls_verification(
&mut connector,
self.sslmode.as_deref(),
self.root_certificate_pem.as_deref(),
self.accept_invalid_certs,
)?;
if self.verify_mode_skips_verification() {
tracing::warn!(
"Postgres connection with sslmode={} is not verifying the server certificate (accept_invalid_certs is set, or no root certificate is configured and the resource predates that flag). Set accept_invalid_certs=false or provide root_certificate_pem to enforce verification.",
self.sslmode.as_deref().unwrap_or("")
);
}
let (client, connection) = tokio::time::timeout(
@@ -759,23 +971,16 @@ impl PgDatabase {
error::Error::InternalErr(format!("IAM token generation failed: {e:#}"))
})?;
// RDS IAM auth requires SSL
// RDS IAM auth requires SSL.
let mut connector = TlsConnector::builder();
if let Some(root_certificate_pem) = &self.root_certificate_pem {
if !root_certificate_pem.is_empty() {
connector.add_root_certificate(
native_tls::Certificate::from_pem(root_certificate_pem.as_bytes())
.map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?,
);
} else {
connector.danger_accept_invalid_certs(true);
connector.danger_accept_invalid_hostnames(true);
}
} else {
tracing::warn!("IAM RDS auth without root certificate: TLS certificate verification is disabled. Consider providing root_certificate_pem for production use.");
connector
.danger_accept_invalid_certs(true)
.danger_accept_invalid_hostnames(true);
let verified = Self::configure_pg_tls_verification(
&mut connector,
self.sslmode.as_deref(),
self.root_certificate_pem.as_deref(),
self.accept_invalid_certs,
)?;
if !verified {
tracing::warn!("IAM RDS auth without certificate verification: TLS certificate verification is disabled. Provide root_certificate_pem (and set sslmode=verify-full) to enforce verification.");
}
tracing::info!("Creating new IAM RDS connection to {}", &self.host);
@@ -840,6 +1045,7 @@ impl PgDatabase {
dbname,
sslmode,
root_certificate_pem: None,
accept_invalid_certs: None,
use_iam_auth: None,
region: None,
})
backend/windmill-common/src/workspaces.rs
+63 -3
View File
@@ -184,7 +184,37 @@ pub const WM_FORK_PREFIX: &str = "wm-fork-";
/// layer because the actual branch creation runs in a deferred git-sync worker job — without
/// this check, the API returns 200 and the failure only surfaces later in the worker.
pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> {
if !id.starts_with(WM_FORK_PREFIX) {
validate_workspace_branch_id(id, true)
}
/// Like [`validate_fork_workspace_id`] but does not require the `wm-fork-` prefix. Used for dev
/// workspaces, whose id is an ordinary (prefix-less) workspace id but must still be git-branch-safe
/// because it is interpolated into a `wm-fork/<original_branch>/<id>` branch name like any fork.
pub fn validate_dev_workspace_id(id: &str) -> error::Result<()> {
validate_workspace_branch_id(id, false)
}
/// The `workspace.name` column is `character varying(50)`, so a name longer than 50 characters
/// triggers a raw `value too long for type character varying(50)` SQL error on insert. Validate
/// up front to return a clear message instead.
pub fn validate_workspace_name(name: &str) -> error::Result<()> {
if name.chars().count() > 50 {
return Err(Error::BadRequest(format!(
"Workspace name is too long ({} chars). Maximum length is 50 characters.",
name.chars().count()
)));
}
Ok(())
}
fn validate_workspace_branch_id(id: &str, require_fork_prefix: bool) -> error::Result<()> {
if id.is_empty() {
return Err(Error::BadRequest(
"Workspace id cannot be empty".to_string(),
));
}
if require_fork_prefix && !id.starts_with(WM_FORK_PREFIX) {
return Err(Error::BadRequest(format!(
"The id `{}` is invalid for a forked workspace. It should be prefixed by {}",
id, WM_FORK_PREFIX
@@ -193,8 +223,9 @@ pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> {
if id.len() > 50 {
return Err(Error::BadRequest(format!(
"Fork workspace id `{}` is too long ({} chars). Maximum length is 50 characters (including the '{}' prefix).",
id, id.len(), WM_FORK_PREFIX
"Workspace id `{}` is too long ({} chars). Maximum length is 50 characters.",
id,
id.len()
)));
}
@@ -558,6 +589,11 @@ pub enum RuleCheckResult {
/// - User is not in bypass lists
///
/// Returns `Err` if the rule is not found
/// Reserved protection-rule name applied to a prod workspace paired with a dev workspace. It carries
/// `DisableDirectDeployment` + `DisableWorkspaceForking` and is auto-managed by the dev-workspace
/// feature (applied on pairing, removed on detach).
pub const DEV_WORKSPACE_LOCK_RULE_NAME: &str = "dev_workspace_lock";
pub async fn check_user_against_rule(
workspace_id: &str,
rule: &ProtectionRuleKind,
@@ -915,6 +951,30 @@ mod tests {
assert!(validate_fork_workspace_id("wm-fork-foo/bar.lock").is_err());
}
#[test]
fn test_validate_dev_workspace_id_accepts_prefixless_valid() {
// Dev workspaces use ordinary, prefix-less ids but must stay git-branch-safe.
validate_dev_workspace_id("dev").unwrap();
validate_dev_workspace_id("my-dev-workspace").unwrap();
validate_dev_workspace_id("staging.42").unwrap();
// The fork prefix is allowed but not required.
validate_dev_workspace_id("wm-fork-dev").unwrap();
}
#[test]
fn test_validate_dev_workspace_id_rejects_empty_and_git_unsafe() {
assert!(validate_dev_workspace_id("").is_err());
assert!(validate_dev_workspace_id("dev workspace").is_err());
assert!(validate_dev_workspace_id("dev..staging").is_err());
assert!(validate_dev_workspace_id("dev/.x").is_err());
assert!(validate_dev_workspace_id("dev.lock").is_err());
}
#[test]
fn test_validate_fork_workspace_id_rejects_empty() {
assert!(validate_fork_workspace_id("").is_err());
}
#[test]
fn test_validate_fork_workspace_id_rejects_too_long() {
let long_id = format!("wm-fork-{}", "a".repeat(43));
backend/windmill-object-store/src/lib.rs
+57 -1
View File
@@ -494,6 +494,23 @@ fn build_azure_blob_client(
return Ok(Arc::new(store));
}
/// Whether a GCS `service_account_key` carries no static credentials, in which case the client
/// should fall back to the instance's ambient credentials (GKE Workload Identity / metadata server)
/// instead of being handed an unparseable key. Besides an empty/whitespace string, the settings UI
/// stores "no key" as an empty JSON object `{}` (and `serde_json` may yield `null`), so treat those
/// as absent too. Shared with the connectivity-test SSRF guard so both agree on what "no key" means.
pub fn gcs_service_account_key_is_blank(service_account_key: &str) -> bool {
let trimmed = service_account_key.trim();
if trimmed.is_empty() {
return true;
}
match serde_json::from_str::<serde_json::Value>(trimmed) {
Ok(serde_json::Value::Null) => true,
Ok(serde_json::Value::Object(map)) => map.is_empty(),
_ => false,
}
}
#[cfg(feature = "parquet")]
async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result<Arc<dyn ObjectStore>> {
let gcs_resource = gcs_resource_ref.clone();
@@ -509,7 +526,12 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result<Arc<d
)
.with_bucket_name(gcs_resource.bucket);
store_builder = store_builder.with_service_account_key(gcs_resource.service_account_key);
// A blank key means no static credentials: let the builder fall back to the metadata server
// (InstanceCredentialProvider) so GKE Workload Identity / ambient credentials work. Passing a
// blank/`{}` key to `with_service_account_key` would instead fail to parse.
if !gcs_service_account_key_is_blank(&gcs_resource.service_account_key) {
store_builder = store_builder.with_service_account_key(gcs_resource.service_account_key);
}
let store = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store_builder.build()))
.map_err(|panic_info| {
@@ -1703,6 +1725,40 @@ mod tests {
.contains("GCS is not supported"));
}
#[test]
fn test_gcs_service_account_key_is_blank() {
for blank in ["", " ", "\n\t", "{}", " {} ", "null"] {
assert!(
gcs_service_account_key_is_blank(blank),
"{blank:?} should be treated as no key"
);
}
for present in ["{\"client_email\":\"x@y.z\"}", "not json"] {
assert!(
!gcs_service_account_key_is_blank(present),
"{present:?} should be treated as a key"
);
}
}
#[cfg(feature = "parquet")]
#[tokio::test]
async fn test_build_gcs_client_blank_key_uses_instance_credentials() {
// A blank service account key must not be passed to `with_service_account_key`
// (which would fail to parse): the builder should fall back to instance credentials
// (GKE Workload Identity / metadata server) and construct successfully. `{}` is the
// settings UI's representation of "no key".
for key in ["", " ", "{}"] {
let resource =
GcsResource { bucket: "bucket".to_string(), service_account_key: key.to_string() };
assert!(
build_gcs_client(&resource).await.is_ok(),
"blank key {:?} should build via instance credentials",
key
);
}
}
#[test]
fn test_duckdb_connection_settings_filesystem_unsupported() {
let resource = ObjectStoreResource::Filesystem(FilesystemSettings {
backend/windmill-queue/src/tags.rs
+28 -17
View File
@@ -8,13 +8,21 @@ const FORK_PARENT_CACHE_TTL_SECS: u64 = 300;
lazy_static::lazy_static! {
// Cache of fork workspace id -> (parent_workspace_id, cached_at).
// `parent_workspace_id` is essentially immutable once a fork is created, so a multi-minute TTL
// is safe. `None` means the lookup found no parent (or the DB call failed); we still cache it
// briefly so that forks missing a parent do not hammer the DB.
// `parent_workspace_id` is stable for the lifetime of a fork EXCEPT across attach/detach of a
// dev workspace, which set/keep it; those paths call `invalidate_fork_parent_cache` so routing
// doesn't lag. `None` means the lookup found no parent (or the DB call failed); we still cache
// it briefly so that forks missing a parent do not hammer the DB.
static ref FORK_PARENT_CACHE: quick_cache::sync::Cache<String, (Option<String>, std::time::Instant)> =
quick_cache::sync::Cache::new(500);
}
/// Drop the cached fork->parent mapping for a workspace. Call after mutating `parent_workspace_id`
/// (attaching/detaching a dev workspace) so per-workspace job tags resolve to the new parent
/// immediately instead of after the cache TTL.
pub fn invalidate_fork_parent_cache(workspace_id: &str) {
FORK_PARENT_CACHE.remove(workspace_id);
}
/// Returns `Some(effective_workspace_tag_id)` if jobs of `workspace_id` should use workspace-
/// specific tags, where `effective_workspace_tag_id` is the string embedded in the tag. For forks,
/// this is always the parent workspace id, optionally suffixed with `-fork` (controlled by the
@@ -26,16 +34,14 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool<Postgres>) -> Optio
return None;
}
let is_fork = workspace_id.starts_with(WM_FORK_PREFIX);
// For forks, always resolve to the parent workspace id; regular workspaces avoid the lookup.
let effective_ws_id: String = if is_fork {
lookup_fork_parent(workspace_id, db)
.await
.unwrap_or_else(|| workspace_id.to_string()) // no parent found -> fall back to fork's own id
} else {
workspace_id.to_string()
};
// Resolve to the parent workspace id when the workspace is a fork or dev workspace (both set
// parent_workspace_id). The lookup caches its `None` result, so non-forks stay cheap after warmup
// (and the common case is already short-circuited by the global toggle above).
let parent = lookup_fork_parent(workspace_id, db).await;
// A `wm-fork-` workspace can outlive its parent (the FK is `ON DELETE SET NULL`), so keep
// treating the prefix as fork-ness for the `-fork` suffix even when the parent link is gone.
let is_fork = parent.is_some() || workspace_id.starts_with(WM_FORK_PREFIX);
let effective_ws_id: String = parent.unwrap_or_else(|| workspace_id.to_string());
// Whitelist check is against the resolved (parent) id so that including a parent in the
// whitelist transparently covers all of its forks.
@@ -62,8 +68,10 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool<Postgres>) -> Optio
})
}
/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set (or the
/// DB lookup failed). Backed by a short-TTL cache to avoid a DB round-trip per job push.
/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set. Backed by a
/// short-TTL cache to avoid a DB round-trip per job push. A transient DB error returns `None` for
/// this call but is NOT cached, so the next push retries instead of misrouting a (prefix-less) dev
/// workspace's jobs for the whole TTL.
async fn lookup_fork_parent(fork_id: &str, db: &Pool<Postgres>) -> Option<String> {
if let Some((parent, cached_at)) = FORK_PARENT_CACHE.get(fork_id) {
if cached_at.elapsed().as_secs() < FORK_PARENT_CACHE_TTL_SECS {
@@ -78,8 +86,11 @@ async fn lookup_fork_parent(fork_id: &str, db: &Pool<Postgres>) -> Option<String
.fetch_optional(db)
.await
{
Ok(Some(Some(parent))) => Some(parent),
_ => None,
Ok(opt) => opt.flatten(),
Err(e) => {
tracing::warn!("failed to look up fork parent for {fork_id}: {e:#}");
return None;
}
};
FORK_PARENT_CACHE.insert(
backend/windmill-store/src/resources.rs
+48 -16
View File
@@ -1088,21 +1088,48 @@ async fn create_resource(
.execute(&db)
.await?;
}
sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, description, resource_type, created_by, edited_at, labels)
VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)
DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels",
w_id,
resource.path,
raw_json as sqlx::types::Json<&RawValue>,
resource.description,
resource.resource_type,
authed.username,
resource.labels.as_deref() as Option<&[String]>
)
.execute(&mut *tx)
.await?;
if update_if_exists {
sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, description, resource_type, created_by, edited_at, labels)
VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)
DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels",
w_id,
resource.path,
raw_json as sqlx::types::Json<&RawValue>,
resource.description,
resource.resource_type,
authed.username,
resource.labels.as_deref() as Option<&[String]>
)
.execute(&mut *tx)
.await?;
} else {
// Create-only (the default): DO NOTHING + a row-count guard, so a path that appears between
// check_path_conflict above and this insert is rejected rather than overwritten. A plain
// DO UPDATE here would clobber a concurrently-created resource, breaking create-only callers
// (e.g. Compare & Deploy "Create in <other>").
let inserted = sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, description, resource_type, created_by, edited_at, labels)
VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING",
w_id,
resource.path,
raw_json as sqlx::types::Json<&RawValue>,
resource.description,
resource.resource_type,
authed.username,
resource.labels.as_deref() as Option<&[String]>
)
.execute(&mut *tx)
.await?;
if inserted.rows_affected() == 0 {
return Err(Error::BadRequest(format!(
"Resource {} already exists",
resource.path
)));
}
}
// Mirror update_resource: Some(true) inserts, Some(false) clears (only
// meaningful on the upsert path, since a pure create has no existing row),
@@ -1385,7 +1412,12 @@ fn collect_var_refs(value: &serde_json::Value, out: &mut Vec<String>) {
}
}
async fn mark_linked_variables_ws_specific(
/// Marks every variable referenced by the resource at `resource_path` as workspace-specific.
///
/// AUTH CONTRACT: this mutates `ws_specific` and does NOT check authorization itself. The caller
/// MUST verify that `authed` has write access to the resource at `resource_path` in `w_id` (e.g. via
/// `require_owner_of_path`) before calling it.
pub async fn mark_linked_variables_ws_specific(
tx: &mut Transaction<'_, Postgres>,
authed: &ApiAuthed,
w_id: &str,
backend/windmill-worker/src/ansible_executor.rs
+410
View File
@@ -938,6 +938,143 @@ remote_tmp={job_dir}/.ansible/tmp
Ok(())
}
/// Read a colon-separated path list (e.g. `roles_path`, `collections_path`) from
/// the `[defaults]` section of an ansible.cfg. Returns the raw entries as written,
/// unresolved. Deliberately minimal: no inline-comment or continuation handling,
/// which ansible's configparser also does not apply to these values. `key` and `=`
/// or `:` as the delimiter are both matched (Python configparser accepts either),
/// with the value itself split on `:` (`os.pathsep`).
fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option<Vec<String>> {
let mut in_defaults = false;
for line in content.lines() {
let trimmed = line.trim();
if trimmed.starts_with('[') && trimmed.ends_with(']') {
in_defaults = trimmed[1..trimmed.len() - 1]
.trim()
.eq_ignore_ascii_case("defaults");
continue;
}
if !in_defaults || trimmed.starts_with('#') || trimmed.starts_with(';') {
continue;
}
// configparser delimits key/value on the first `=` or `:`, whichever
// comes first; the remaining `:` in the value are path separators.
let sep = trimmed.find('=').into_iter().chain(trimmed.find(':')).min();
if let Some(sep) = sep {
let (k, rest) = trimmed.split_at(sep);
let v = &rest[1..];
if k.trim().eq_ignore_ascii_case(key) {
let entries: Vec<String> = v
.split(':')
.map(|s| s.trim())
.filter(|s| !s.is_empty())
.map(|s| s.to_string())
.collect();
return (!entries.is_empty()).then_some(entries);
}
}
}
None
}
/// Prepend Windmill's dependency install dir to the repo cfg's declared path list.
/// Relative entries from the repo cfg are resolved against `cfg_dir` to match how
/// ansible resolves them relative to the config file's own directory.
fn resolve_and_prepend_path(
base: String,
repo_entries: Option<Vec<String>>,
cfg_dir: &str,
) -> String {
let mut paths = vec![base];
if let Some(entries) = repo_entries {
for e in entries {
if e.starts_with('/') || e.starts_with('~') {
paths.push(e);
} else {
paths.push(format!("{cfg_dir}/{e}"));
}
}
}
paths.join(":")
}
/// Build the environment overrides applied when delegating to a git repo that
/// ships its own ansible.cfg. See the call site for the layering rationale.
async fn build_ansible_cfg_override_envs(
cfg_path: &str,
job_dir: &str,
vault_password_file_exists: bool,
reqs: Option<&AnsibleRequirements>,
) -> error::Result<Vec<(String, String)>> {
let mut envs = vec![
("ANSIBLE_CONFIG".to_string(), cfg_path.to_string()),
// Runtime-bound: reference the ephemeral job dir, cannot be set statically.
("ANSIBLE_HOME".to_string(), format!("{job_dir}/.ansible")),
(
"ANSIBLE_LOCAL_TEMP".to_string(),
format!("{job_dir}/.ansible/tmp"),
),
(
"ANSIBLE_REMOTE_TEMP".to_string(),
format!("{job_dir}/.ansible/tmp"),
),
];
// Vault: Windmill manages the secret, so its config wins over the repo cfg.
if vault_password_file_exists {
envs.push((
"ANSIBLE_VAULT_PASSWORD_FILE".to_string(),
format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}"),
));
}
if let Some(vault_ids) = reqs.map(|r| &r.vault_id).filter(|v| !v.is_empty()) {
// Defense in depth: entries are validated at parse time, but re-check
// here since they are interpolated raw into the env value.
for vault_id in vault_ids {
validate_vault_id(vault_id)?;
}
envs.push((
"ANSIBLE_VAULT_IDENTITY_LIST".to_string(),
vault_ids.join(","),
));
}
// Dependency search paths: additive. Windmill installs galaxy roles into
// `{job_dir}/roles` and collections into `{job_dir}`; prepend those to the
// repo cfg's declared paths so both Windmill-installed and repo deps resolve.
let cfg_dir = std::path::Path::new(cfg_path)
.parent()
.and_then(|p| p.to_str())
.unwrap_or(job_dir);
let cfg_content = tokio::fs::read_to_string(cfg_path).await.map_err(|e| {
windmill_common::error::Error::internal_err(format!(
"Failed to read delegated ansible.cfg at `{cfg_path}`: {e}"
))
})?;
envs.push((
"ANSIBLE_ROLES_PATH".to_string(),
resolve_and_prepend_path(
format!("{job_dir}/roles"),
parse_ansible_cfg_path_list(&cfg_content, "roles_path"),
cfg_dir,
),
));
envs.push((
"ANSIBLE_COLLECTIONS_PATH".to_string(),
resolve_and_prepend_path(
job_dir.to_string(),
// Also probe the deprecated plural ini alias; env vars replace (not
// merge) the cfg value, so a repo using it would otherwise be dropped.
parse_ansible_cfg_path_list(&cfg_content, "collections_path")
.or_else(|| parse_ansible_cfg_path_list(&cfg_content, "collections_paths")),
cfg_dir,
),
));
Ok(envs)
}
pub async fn get_git_ssh_cmd(
reqs: &AnsibleRequirements,
job_dir: &str,
@@ -1136,6 +1273,9 @@ pub async fn handle_ansible_job(
let mut nsjail_extra_mounts = vec![];
let mut playbook_override = None;
// Absolute path of a repo-provided `ansible.cfg` to use as the effective
// config, set when `delegate_to_git_repo.ansible_cfg` is provided.
let mut ansible_config_override: Option<String> = None;
if let Some(r) = reqs.as_ref() {
nsjail_extra_mounts = create_file_resources(
@@ -1301,6 +1441,23 @@ pub async fn handle_ansible_job(
inventories.push(format!("{}/{}", &repo.target_path, inv));
}
if let Some(cfg_rel) = delegated_git_repo.ansible_cfg.as_ref() {
let cfg_rel = interpolate_template(
cfg_rel,
interpolated_args.as_ref(),
"delegate_to_git_repo.ansible_cfg",
)?;
validate_relative_path(&cfg_rel, "delegate_to_git_repo.ansible_cfg")?;
let cfg_path = format!("{}/{}/{}", job_dir, &repo.target_path, cfg_rel);
if !tokio::fs::try_exists(&cfg_path).await.unwrap_or(false) {
return Err(windmill_common::error::Error::BadRequest(format!(
"delegate_to_git_repo.ansible_cfg: no ansible.cfg found in the cloned repo at `{}/{}`",
&repo.target_path, cfg_rel
)));
}
ansible_config_override = Some(cfg_path);
}
if delegated_git_repo.install_requirements {
install_requirements_from_cloned_repo(
&repo.target_path,
@@ -1450,6 +1607,32 @@ pub async fn handle_ansible_job(
create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?;
// When the run delegates to a git repo that ships its own ansible.cfg, that
// file becomes the effective config (ansible loads exactly one config file and
// does not merge). These env vars layer Windmill's runtime-bound settings back
// on top — env vars outrank ansible.cfg. Only applied on the non-sandboxed
// path: git-repo delegation clones into `job_dir` which the nsjail profile does
// not mount, so it already requires DISABLE_NSJAIL.
let ansible_env_overrides = match ansible_config_override.as_ref() {
Some(cfg_path) => {
if is_sandboxing_enabled() {
tracing::warn!(
"delegate_to_git_repo.ansible_cfg is set but sandboxing is enabled; \
git-repo delegation requires DISABLE_NSJAIL, the ansible.cfg override \
will not take effect"
);
}
build_ansible_cfg_override_envs(
cfg_path,
job_dir,
vault_password_file_exists,
reqs.as_ref(),
)
.await?
}
None => vec![],
};
let mut reserved_variables =
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
let additional_python_paths_folders = additional_python_paths.join(":");
@@ -1560,6 +1743,7 @@ fi
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("HOME", HOME_ENV.as_str())
.envs(ansible_env_overrides)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped());
@@ -1856,4 +2040,230 @@ mod tests {
assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err());
assert!(!dir.path().join("ansible.cfg").exists());
}
#[test]
fn test_parse_ansible_cfg_path_list() {
let cfg = "\
[defaults]
roles_path = roles:extra/roles
collections_path=/opt/collections
host_key_checking = False
[inventory]
roles_path = ignored/section
";
assert_eq!(
parse_ansible_cfg_path_list(cfg, "roles_path"),
Some(vec!["roles".to_string(), "extra/roles".to_string()])
);
assert_eq!(
parse_ansible_cfg_path_list(cfg, "collections_path"),
Some(vec!["/opt/collections".to_string()])
);
// Keys only in another section are not picked up.
assert_eq!(parse_ansible_cfg_path_list(cfg, "library"), None);
}
#[test]
fn test_parse_ansible_cfg_path_list_ignores_comments() {
let cfg = "\
[defaults]
# roles_path = commented
; roles_path = also_commented
";
assert_eq!(parse_ansible_cfg_path_list(cfg, "roles_path"), None);
}
#[test]
fn test_parse_ansible_cfg_path_list_colon_delimiter() {
// configparser accepts `:` as a key/value delimiter, and the value can
// itself be a `:`-separated list.
let cfg = "\
[defaults]
roles_path: my_roles
collections_path : a/col:b/col
";
assert_eq!(
parse_ansible_cfg_path_list(cfg, "roles_path"),
Some(vec!["my_roles".to_string()])
);
assert_eq!(
parse_ansible_cfg_path_list(cfg, "collections_path"),
Some(vec!["a/col".to_string(), "b/col".to_string()])
);
}
#[test]
fn test_resolve_and_prepend_path() {
// No repo entries: only Windmill's install dir.
assert_eq!(
resolve_and_prepend_path("/job/roles".to_string(), None, "/job/repo"),
"/job/roles"
);
// Relative repo entries resolve against the cfg dir; absolute/~ kept as-is.
assert_eq!(
resolve_and_prepend_path(
"/job/roles".to_string(),
Some(vec![
"roles".to_string(),
"/abs/roles".to_string(),
"~/r".to_string()
]),
"/job/repo/config"
),
"/job/roles:/job/repo/config/roles:/abs/roles:~/r"
);
}
fn ansible_playbook_available() -> bool {
std::process::Command::new("ansible-playbook")
.arg("--version")
.output()
.is_ok()
}
/// End-to-end: with a delegated repo that ships its own `ansible.cfg` pointing
/// `roles_path` at an in-repo directory, the override env vars must make the
/// real `ansible-playbook` resolve a role it otherwise cannot. Requires the
/// `ansible-playbook` binary; self-skips when absent (e.g. standard CI). Run on
/// a worker devbox with `cargo test -p windmill-worker --features python`.
#[tokio::test]
async fn test_ansible_cfg_override_resolves_repo_roles_e2e() {
if !ansible_playbook_available() {
eprintln!(
"SKIP test_ansible_cfg_override_resolves_repo_roles_e2e: ansible-playbook not found on PATH"
);
return;
}
let dir = tempfile::tempdir().unwrap();
let job_dir = dir.path().to_str().unwrap();
let repo = dir.path().join(DELEGATE_GIT_REPO_TARGET);
let role_tasks = repo.join("my_roles/greet/tasks");
std::fs::create_dir_all(&role_tasks).unwrap();
std::fs::write(
repo.join("ansible.cfg"),
"[defaults]\nroles_path = my_roles\n",
)
.unwrap();
std::fs::write(
role_tasks.join("main.yml"),
"- debug:\n msg: \"hello from greet role\"\n",
)
.unwrap();
let play = "- hosts: localhost\n connection: local\n gather_facts: false\n roles:\n - greet\n";
std::fs::write(repo.join("play.yml"), play).unwrap();
// Windmill's own generated cfg (the negative-control config that exists today).
create_ansible_cfg(None, job_dir, false).unwrap();
let playbook = format!("{DELEGATE_GIT_REPO_TARGET}/play.yml");
let run = |envs: Vec<(String, String)>| {
std::process::Command::new("ansible-playbook")
.arg(&playbook)
.current_dir(job_dir)
.envs(envs)
.output()
.unwrap()
};
// Negative control: today's behavior (Windmill cfg via cwd, no override) —
// the role lives in the repo subdir and is not found.
let cfg_path = repo.join("ansible.cfg");
let before = run(vec![]);
assert!(
!before.status.success(),
"without the override the repo role must NOT resolve; stdout={}",
String::from_utf8_lossy(&before.stdout)
);
// With the override: ANSIBLE_CONFIG points at the repo cfg and roles_path
// is honored, so the role runs.
let envs =
build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None)
.await
.unwrap();
let after = run(envs);
let stdout = String::from_utf8_lossy(&after.stdout);
assert!(
after.status.success() && stdout.contains("hello from greet role"),
"with the override the repo role must resolve; status={:?} stdout={stdout} stderr={}",
after.status,
String::from_utf8_lossy(&after.stderr)
);
}
#[tokio::test]
async fn test_build_ansible_cfg_override_envs() {
let dir = tempfile::tempdir().unwrap();
let job_dir = dir.path().to_str().unwrap();
let repo_dir = dir.path().join("delegate_git_repository");
std::fs::create_dir_all(&repo_dir).unwrap();
let cfg_path = repo_dir.join("ansible.cfg");
std::fs::write(&cfg_path, "[defaults]\nroles_path = my_roles\n").unwrap();
let cfg_path = cfg_path.to_str().unwrap();
let reqs = AnsibleRequirements {
vault_id: vec!["dev@vault_pass.txt".to_string()],
..Default::default()
};
let envs = build_ansible_cfg_override_envs(cfg_path, job_dir, true, Some(&reqs))
.await
.unwrap();
let map: std::collections::HashMap<_, _> = envs.into_iter().collect();
assert_eq!(
map.get("ANSIBLE_CONFIG").map(|s| s.as_str()),
Some(cfg_path)
);
assert_eq!(
map.get("ANSIBLE_HOME"),
Some(&format!("{job_dir}/.ansible"))
);
assert_eq!(
map.get("ANSIBLE_VAULT_PASSWORD_FILE"),
Some(&format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}"))
);
assert_eq!(
map.get("ANSIBLE_VAULT_IDENTITY_LIST").map(|s| s.as_str()),
Some("dev@vault_pass.txt")
);
// Windmill's `{job_dir}/roles` is prepended to the repo cfg's own path,
// which is resolved against the cfg directory.
assert_eq!(
map.get("ANSIBLE_ROLES_PATH"),
Some(&format!(
"{job_dir}/roles:{}/my_roles",
repo_dir.to_str().unwrap()
))
);
// No collections_path in the repo cfg → only Windmill's job dir.
assert_eq!(
map.get("ANSIBLE_COLLECTIONS_PATH").map(|s| s.as_str()),
Some(job_dir)
);
}
#[tokio::test]
async fn test_build_ansible_cfg_override_envs_collections_paths_alias() {
let dir = tempfile::tempdir().unwrap();
let job_dir = dir.path().to_str().unwrap();
let repo_dir = dir.path().join("delegate_git_repository");
std::fs::create_dir_all(&repo_dir).unwrap();
let cfg_path = repo_dir.join("ansible.cfg");
// Deprecated plural alias must still be picked up so the repo's collections
// are not silently dropped when the env override replaces the cfg value.
std::fs::write(&cfg_path, "[defaults]\ncollections_paths = my_cols\n").unwrap();
let envs =
build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None)
.await
.unwrap();
let map: std::collections::HashMap<_, _> = envs.into_iter().collect();
assert_eq!(
map.get("ANSIBLE_COLLECTIONS_PATH"),
Some(&format!("{job_dir}:{}/my_cols", repo_dir.to_str().unwrap()))
);
}
}
backend/windmill-worker/src/duckdb_executor.rs
+97 -1
View File
@@ -195,7 +195,21 @@ fn build_materialized_query(
for s in plan.setup.iter_mut() {
*s = substitute(s);
}
let strategy = if m.append {
let strategy = if m.scd2 {
// SCD2 needs a natural key to identify an entity across versions, and its
// diff/close/open shape has no partition-scoped form in v1.
let key = m.unique_key.clone().ok_or_else(|| {
Error::ExecutionErr(
"materialize scd2: requires a natural key — add `key=<col>`".to_string(),
)
})?;
if partitioned {
return Err(Error::ExecutionErr(
"materialize scd2: `// partitioned` is not supported with scd2 in v1".to_string(),
));
}
MaterializeStrategy::Scd2 { key, track: m.track.clone(), close_deleted: m.close_deleted }
} else if m.append {
MaterializeStrategy::Append
} else if let Some(uk) = m.unique_key {
MaterializeStrategy::Merge { unique_key: uk }
@@ -1520,6 +1534,88 @@ mod tests {
assert!(!rewritten.contains("-- $file"));
}
// SCD2 managed mode wraps the SELECT into the diff → close-old → open-new
// shape (unit-covered in the parser's codegen tests); here we pin the
// executor-level wiring: the natural key flows through and the wrap is
// generated (not the manual track-only path).
#[test]
fn materialize_scd2_wraps_with_history() {
// Primary spelling: `key=<col> history` on a merge.
let script = "-- materialize ducklake://main/dim key=id history track=name\n\
SELECT id, name FROM dl.src";
let (rewritten, _) =
build_materialized_query(script, None, &std::collections::HashMap::new())
.expect("materialize builds")
.expect("materialize present");
let rewritten = rewritten.expect("scd2 is managed — must rewrite");
assert!(
rewritten.contains("valid_from"),
"adds SCD2 columns:\n{rewritten}"
);
assert!(rewritten.contains("is_current"));
assert!(
rewritten.contains("_wm_scd2_changed"),
"captures changed keys"
);
assert!(
rewritten.contains("UPDATE _wm_target.dim SET valid_to"),
"closes prior version"
);
assert!(
rewritten.contains("CREATE VIEW IF NOT EXISTS _wm_target.dim_current"),
"emits the consumer-convenience current view"
);
// default is soft-delete — no deleted-key set without `deletes=close`
assert!(!rewritten.contains("_wm_scd2_deleted"));
assert!(!rewritten.contains("MERGE INTO"));
}
#[test]
fn materialize_scd2_deletes_close_wires_through() {
let script = "-- materialize ducklake://main/dim key=id history deletes=close\n\
SELECT id, name FROM dl.src";
let (rewritten, _) =
build_materialized_query(script, None, &std::collections::HashMap::new())
.expect("materialize builds")
.expect("materialize present");
let rewritten = rewritten.expect("scd2 is managed — must rewrite");
assert!(
rewritten.contains("_wm_scd2_deleted"),
"deletes=close adds the vanished-key set + close:\n{rewritten}"
);
}
#[test]
fn materialize_scd2_requires_key() {
let script = "-- materialize scd2 ducklake://main/dim\nSELECT id, name FROM dl.src";
let err = match build_materialized_query(script, None, &std::collections::HashMap::new()) {
Err(e) => e,
Ok(_) => panic!("scd2 without key must error"),
};
assert!(
format!("{err}").contains("requires a natural key"),
"got: {err}"
);
}
#[test]
fn materialize_scd2_rejects_partitioned() {
let script = "-- pipeline\n-- partitioned daily\n\
-- materialize scd2 ducklake://main/dim key=id\nSELECT id, name FROM dl.src";
let err = match build_materialized_query(
script,
Some("2026-07-01"),
&std::collections::HashMap::new(),
) {
Err(e) => e,
Ok(_) => panic!("partitioned + scd2 must error"),
};
assert!(
format!("{err}").contains("not supported with scd2"),
"got: {err}"
);
}
// Tests for parse_attach_db_resource function
#[test]
fn test_parse_attach_db_resource_postgres_res_prefix() {
backend/windmill-worker/src/pg_executor.rs
+37 -2
View File
@@ -615,6 +615,25 @@ pub async fn do_postgresql(
return Err(Error::BadRequest("Missing database argument".to_string()));
};
// Surface in the job logs (not just the worker logs) when a verify-ca/verify-full
// resource is connecting without actually verifying the server certificate, so the
// person running the query can see and fix the misconfiguration.
if database.verify_mode_skips_verification() {
windmill_queue::append_logs(
&job.id,
&job.workspace_id,
format!(
"warning: sslmode={} but the server's TLS certificate is not being verified \
(accept_invalid_certs is enabled, or no root certificate is configured). Set \
accept_invalid_certs to false or provide root_certificate_pem to verify the \
server identity.\n",
database.sslmode.as_deref().unwrap_or("")
),
conn,
)
.await;
}
let annotations = windmill_common::worker::SqlAnnotations::parse(query);
let collection_strategy = if annotations.raw_output || annotations.return_last_result {
// raw_output emits a single envelope from the last statement, so the
@@ -629,10 +648,26 @@ pub async fn do_postgresql(
// Include use_iam_auth in cache key to distinguish IAM vs non-IAM connections to the same host.
// The cache key is static (doesn't include the token), which is correct because PostgreSQL
// connections remain valid after initial auth — fresh tokens are generated on cache miss.
//
// to_uri() collapses require/verify-ca/verify-full to the same string, so the TLS verification
// inputs are folded into the key separately. Without this a connection established under a
// weaker sslmode (or a different root cert) could be reused for a stricter request, undoing
// the verification configured in PgDatabase::configure_pg_tls_verification.
let tls_disc = {
use std::hash::{Hash, Hasher};
let mut h = std::collections::hash_map::DefaultHasher::new();
database.sslmode.hash(&mut h);
database.root_certificate_pem.hash(&mut h);
database.accept_invalid_certs.hash(&mut h);
h.finish()
};
// to_uri() already ends with `?sslmode=...`, so append further key segments
// with `&` to keep database_string a well-formed URI (it is only ever a cache
// key, but a malformed one would mislead anyone who later logs or parses it).
let database_string = if use_iam_auth {
format!("{}?iam=true", database.to_uri())
format!("{}&iam=true&tls={tls_disc:x}", database.to_uri())
} else {
database.to_uri()
format!("{}&tls={tls_disc:x}", database.to_uri())
};
let database_string_clone = database_string.clone();
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.742.0";
export const VERSION = "v1.744.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/bun.lock
+3
View File
@@ -19,6 +19,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.740.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
@@ -288,6 +289,8 @@
"utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="],
"windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.740.0", "", {}, "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw=="],
"windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="],
"windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.510.1", "", {}, "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ=="],
cli/package-lock.json
Generated
+6
View File
@@ -20,6 +20,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.740.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
@@ -1409,6 +1410,11 @@
"node": ">= 4"
}
},
"node_modules/windmill-parser-wasm-asset": {
"version": "1.740.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.740.0.tgz",
"integrity": "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw=="
},
"node_modules/windmill-parser-wasm-csharp": {
"version": "1.510.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-csharp/-/windmill-parser-wasm-csharp-1.510.1.tgz",
cli/package.json
+1
View File
@@ -28,6 +28,7 @@
"pg-gateway": "0.3.0-beta.4",
"svelte": "^5.45.2",
"tar-stream": "^3.1.7",
"windmill-parser-wasm-asset": "1.740.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
cli/src/commands/pipeline/boundedCascade.ts
+61 -7
View File
@@ -40,10 +40,14 @@ export const isScriptNode = (id: string): boolean => id.startsWith(SCRIPT_PREFIX
export const scriptPathOf = (id: string): string => id.slice(SCRIPT_PREFIX.length);
const assetNodeId = (kind: string, path: string): string => `${kind}:${path}`;
// Native trigger kinds that fan out per-event — never bounded-run starts.
// `webhook` / `data_upload` have no trigger row in the graph payload, so a root
// whose only entry is one of those reads as a manual root.
const EVENT_TRIGGER_KINDS = new Set([
// Native trigger kinds whose scripts must NOT be auto-run by the CLI cascade:
// event triggers fan out per external event, and `webhook`/`data_upload` are UI
// entrypoints that need caller-supplied input (a request body / an uploaded
// S3Object) — previewing any of them with empty args runs the wrong thing.
// The deployed graph omits `webhook`/`data_upload` rows, but the local graph
// emits them, so a `// on data_upload` script would otherwise read as a manual
// root and get auto-run without its upload argument.
const NON_AUTORUN_TRIGGER_KINDS = new Set([
"kafka",
"mqtt",
"nats",
@@ -51,6 +55,8 @@ const EVENT_TRIGGER_KINDS = new Set([
"sqs",
"gcp",
"email",
"webhook",
"data_upload",
]);
/** Resolve an asset URI (`datatable://x`, `s3://b/k`, …) to its node id. */
@@ -121,6 +127,38 @@ function closure(adj: Map<string, Set<string>>, start: string): Set<string> {
export const descendants = (dag: LineageDag, n: string): Set<string> => closure(dag.down, n);
export const ancestors = (dag: LineageDag, n: string): Set<string> => closure(dag.up, n);
/**
* Nodes reachable from `starts` over the lineage DAG, treating `barriers` as cut
* points: a barrier node is neither included NOR traversed through. So a node
* reachable ONLY via a barrier is excluded, while one also reachable via another
* path stays. Used for whole-pipeline runs to keep event handlers AND their
* event-only downstream closure out (subtracting only the handler would leave a
* consumer whose producer was skipped, which topoOrder would then run with
* missing/stale inputs).
*/
export function reachableCutting(
dag: LineageDag,
starts: Iterable<string>,
barriers: Set<string>,
): Set<string> {
const seen = new Set<string>();
const queue: string[] = [];
for (const s of starts) {
if (barriers.has(s) || seen.has(s)) continue;
seen.add(s);
queue.push(s);
}
while (queue.length > 0) {
const n = queue.shift()!;
for (const next of dag.down.get(n) ?? []) {
if (barriers.has(next) || seen.has(next)) continue;
seen.add(next);
queue.push(next);
}
}
return seen;
}
export type BoundedResult = {
nodes: Set<string>;
reachableEnds: string[];
@@ -151,19 +189,35 @@ export function boundedSet(dag: LineageDag, start: string, ends: string[]): Boun
export function validStarts(g: BCGraph): Set<string> {
const subscribers = new Set<string>();
const scheduleScripts = new Set<string>();
const eventScripts = new Set<string>();
const nonAutorunScripts = new Set<string>();
for (const t of g.triggers ?? []) {
if (t.runnable_kind !== "script") continue;
if (t.trigger_kind === "asset") subscribers.add(t.runnable_path);
else if (t.trigger_kind === "schedule") scheduleScripts.add(t.runnable_path);
else if (EVENT_TRIGGER_KINDS.has(t.trigger_kind)) eventScripts.add(t.runnable_path);
else if (NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) nonAutorunScripts.add(t.runnable_path);
}
const out = new Set<string>();
for (const r of g.runnables ?? []) {
if (r.usage_kind !== "script") continue;
const p = r.path;
if (scheduleScripts.has(p)) out.add(scriptNodeId(p));
else if (!subscribers.has(p) && !eventScripts.has(p)) out.add(scriptNodeId(p));
else if (!subscribers.has(p) && !nonAutorunScripts.has(p)) out.add(scriptNodeId(p));
}
return out;
}
/**
* Script node ids that carry a trigger requiring caller-supplied input or
* per-event fanout (kafka/mqtt/nats/postgres/sqs/gcp/email/webhook/data_upload).
* These can't be run with empty args, so a whole-pipeline run must exclude them
* even when they're a lineage descendant of a valid start (not just a root).
*/
export function nonAutorunTriggerScripts(g: BCGraph): Set<string> {
const out = new Set<string>();
for (const t of g.triggers ?? []) {
if (t.runnable_kind === "script" && NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) {
out.add(scriptNodeId(t.runnable_path));
}
}
return out;
}
cli/src/commands/pipeline/dev.ts
+290
View File
@@ -0,0 +1,290 @@
// `wmill pipeline dev [folder]` — live-preview a data pipeline from local files.
//
// The pipeline analog of `wmill dev` / `wmill app dev`: watch a folder of
// `// pipeline` scripts, rebuild the asset graph from the working tree on every
// save, and push it over a WebSocket to the `/pipeline_dev` page, which renders
// the same graph editor the UI uses and runs the cascade via preview (no deploy).
// Editing stays in the user's own editor; the page live-reloads.
import { Command } from "@cliffy/command";
import { colors } from "@cliffy/ansi/colors";
import * as http from "node:http";
import * as fs from "node:fs";
import * as path from "node:path";
import { createHash, randomBytes } from "node:crypto";
import process from "node:process";
import * as open from "open";
import { WebSocket, WebSocketServer } from "ws";
import * as log from "../../core/log.ts";
import { GlobalOptions } from "../../types.ts";
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace } from "../../core/context.ts";
import {
mergeConfigWithConfigFile,
type SyncOptions,
} from "../../core/conf.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
import { resolveBindPort } from "../../utils/port-probe.ts";
import { getConfigDirPath } from "../../../windmill-utils-internal/src/config/config.ts";
import { buildLocalPipelineGraph, workspaceRoot } from "./localGraph.ts";
const PORT = 3201;
// Bind loopback only: each WS frame ships the folder's full script source
// (`scripts[].content` + `temp_script_refs`) with no auth, so it must not be
// reachable from the LAN. The webview connects via `ws://localhost`, and an SSH
// `-L` / devbox port-forward targets 127.0.0.1 on the host, so both still work.
const LISTEN_HOST = "127.0.0.1";
interface PipelineDevOpts extends GlobalOptions, SyncOptions {
port?: number;
open?: boolean;
defaultTs?: "bun" | "deno";
frontend?: string;
}
// The WS token gates access to local source. Persist it under the user-private
// config dir (0600) so a `pipeline dev` restart reuses it — an already-open
// `/pipeline_dev` page auto-reconnecting with the token from its URL then
// survives the restart, instead of every upgrade being rejected by
// `verifyClient` until the freshly printed URL is reopened. Scoped by
// remote+workspace+root+folder+port (NOT port alone): a same-session restart
// reconnects, but any *different* session — another folder, workspace, remote, or
// local checkout on the same port — gets a different token, so a stale tab can't
// reconnect and receive another session's source. Falls back to an ephemeral
// token if the config dir can't be read/written.
async function stableWsToken(
remote: string,
workspaceId: string,
root: string,
folder: string,
port: number,
): Promise<string> {
// Hash the tuple (NUL-delimited so no value can spoof the boundary) → a
// collision-resistant, filesystem-safe key. A plain sanitized join would let
// different values collide (`a/b` and `a_b` → same name), which would reuse a
// token across sessions and reintroduce the cross-session leak.
const key = createHash("sha256")
.update(`${remote}\0${workspaceId}\0${root}\0${folder}\0${port}`)
.digest("hex")
.slice(0, 32);
let tokenFile: string | undefined;
try {
tokenFile = path.join(await getConfigDirPath(), `pipeline-dev-${key}.token`);
const existing = fs.readFileSync(tokenFile, "utf-8").trim();
if (existing) return existing;
} catch {
// no reusable token file yet (or config dir unavailable) — mint a fresh one
}
const token = randomBytes(24).toString("base64url");
if (tokenFile) {
try {
fs.writeFileSync(tokenFile, token, { mode: 0o600 });
} catch {
// best-effort persistence; fall back to the in-memory token
}
}
return token;
}
// Resolve the target folder: explicit arg, else auto-detect when cwd sits inside
// `f/<folder>/…` (the supported `cd f/my_pipeline && wmill pipeline dev` form).
function resolveFolder(root: string, folderArg?: string): string | undefined {
if (folderArg) return folderArg.replace(/^f\//, "").replace(/\/$/, "");
const rel = path.relative(root, process.cwd()).replaceAll("\\", "/");
if (rel === "" || rel.startsWith("..")) return undefined;
const segs = rel.split("/");
if (segs[0] === "f" && segs[1]) return segs[1];
return undefined;
}
async function dev(opts: PipelineDevOpts, folderArg?: string) {
const root = workspaceRoot();
const folder = resolveFolder(root, folderArg);
if (!folder) {
log.error(
colors.red(
"Could not determine the pipeline folder. Pass it explicitly " +
"(`wmill pipeline dev <folder>`) or run from inside an `f/<folder>` directory.",
),
);
process.exit(1);
}
const folderDir = path.join(root, "f", folder);
if (!fs.existsSync(folderDir)) {
log.error(colors.red(`Folder not found on disk: ${folderDir}`));
process.exit(1);
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const merged = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(merged);
// Resolve relative imports from local (not-yet-deployed) content for previews.
// Snapshot at startup like `wmill dev`; restart to refresh. Degrades to
// undefined on older backends.
let tempScriptRefs: Record<string, string> | undefined;
try {
const { buildPreviewTempScriptRefs } = await import(
"../generate-metadata/generate-metadata.ts"
);
tempScriptRefs = await buildPreviewTempScriptRefs(
workspace,
merged,
codebases,
{ kind: "all" },
);
} catch {
// best-effort
}
const EMPTY_GRAPH = { runnables: [], assets: [], edges: [], triggers: [] };
async function buildBundle() {
const { graph, scripts } = await buildLocalPipelineGraph({
root,
folder: folder!,
defaultTs: merged.defaultTs,
});
return {
type: "pipeline" as const,
folder,
graph,
scripts,
temp_script_refs: tempScriptRefs,
};
}
// Don't let a transient build error (e.g. a half-written file) abort startup —
// serve an empty graph and recover on the next save.
let current: Awaited<ReturnType<typeof buildBundle>>;
try {
current = await buildBundle();
} catch (e: any) {
log.error(colors.red(`Initial graph build failed: ${e.message}`));
current = { type: "pipeline", folder, graph: EMPTY_GRAPH, scripts: [], temp_script_refs: tempScriptRefs };
}
log.info(
colors.blue(
`Watching f/${folder} — ${current.scripts.length} pipeline script(s)`,
),
);
const clients = new Set<WebSocket>();
function broadcast() {
const msg = JSON.stringify(current);
for (const ws of clients) {
if (ws.readyState === WebSocket.OPEN) ws.send(msg);
}
}
// Debounced rebuild on any change under the folder.
let timer: ReturnType<typeof setTimeout> | undefined;
const watcher = fs.watch(folderDir, { recursive: true });
watcher.on("change", (_ev, filename) => {
if (filename && filename.toString().endsWith(".lock")) return;
if (timer) clearTimeout(timer);
timer = setTimeout(async () => {
timer = undefined;
try {
current = await buildBundle();
log.info(colors.cyan(`↻ rebuilt graph (${current.scripts.length} scripts)`));
broadcast();
} catch (e: any) {
log.error(colors.red(`Failed to rebuild pipeline graph: ${e.message}`));
}
}, 150);
});
watcher.on("error", (e) => log.error(colors.red(`Watcher error: ${e.message}`)));
const port = await resolveBindPort(opts.port ?? PORT, "wmill pipeline dev", {
info: (m) => log.info(m),
warn: (m) => log.warn(m),
});
const server = http.createServer((_req, res) => {
res.writeHead(200);
res.end();
});
// Loopback bind keeps the LAN out, but any browser tab can still open a
// `ws://localhost:<port>` connection — and each frame ships the folder's full
// script source. Gate the upgrade on an unguessable per-session token (carried
// in the dev-page URL) so a stray page on the predictable dev port can't
// exfiltrate the source. base64url → safe as a query value.
// Stable across restarts (scoped to remote+workspace+root+folder+port) so an
// already-open page reconnects after a CLI restart (see stableWsToken), not
// just after a transient WS drop.
const wsToken = await stableWsToken(
workspace.remote,
workspace.workspaceId,
root,
folder,
port,
);
const wss = new WebSocketServer({
server,
verifyClient: (info) => {
try {
const u = new URL(info.req.url ?? "", "http://localhost");
return u.searchParams.get("token") === wsToken;
} catch {
return false;
}
},
});
wss.on("connection", (ws: WebSocket) => {
clients.add(ws);
// Push the current bundle immediately so the page renders without waiting
// for the first file change.
try {
ws.send(JSON.stringify(current));
} catch {
// ignore
}
ws.on("close", () => clients.delete(ws));
ws.on("error", () => clients.delete(ws));
});
// The `/pipeline_dev` page is served by the frontend, not the backend. By
// default we open it on the workspace remote, but that 404s on a remote whose
// deployed frontend predates this route — `--frontend` points the page at a
// locally-run frontend (`REMOTE=<remote> npm run dev`) while the API/token
// still target the remote. Normalize to a single trailing slash either way.
const pageBase = (opts.frontend ?? workspace.remote).replace(/\/?$/, "/");
const url =
`${pageBase}pipeline_dev?workspace=${workspace.workspaceId}` +
`&wm_token=${workspace.token}&folder=${encodeURIComponent(folder)}&port=${port}` +
`&ws_token=${encodeURIComponent(wsToken)}`;
server.listen(port, LISTEN_HOST, () => {
log.info(colors.green.bold(`🚀 Pipeline dev server on ws://localhost:${port}/ws`));
log.info(colors.gray(`Open: ${url}`));
if (opts.open !== false) {
open.default(url).catch((e: any) =>
log.warn(colors.yellow(`Failed to open browser: ${e.message}`)),
);
}
});
process.on("SIGINT", () => {
log.info(colors.yellow("\n🛑 Shutting down…"));
watcher.close();
for (const ws of clients) ws.close();
server.close();
process.exit(0);
});
}
const command = new Command()
.description(
"Live-preview a data pipeline from local files: watch an `f/<folder>` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).",
)
.arguments("[folder:string]")
.option("--port <port:number>", "Port for the dev WebSocket server.")
.option("--no-open", "Do not open the browser automatically.")
.option(
"--frontend <origin:string>",
"Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.",
)
.action(dev as any);
export default command;

Some files were not shown because too many files have changed in this diff Show More