From 5d7fb6deca3e02e89d77e5d3856483beb8b8bfeb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 04:23:44 +0000 Subject: [PATCH 01/34] =?UTF-8?q?feat(pipelines):=20asset=20freshness=20?= =?UTF-8?q?=E2=80=94=20fresh/stale=20badge=20(CE)=20+=20watchdog=20(EE)=20?= =?UTF-8?q?(#9909)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): passive asset freshness tracking on the graph Co-Authored-By: Claude Fable 5 * chore(pipelines): drop dead freshness-enforcement stub, document query ordering Co-Authored-By: Claude Fable 5 * feat(pipelines): freshness watchdog (EE) — auto re-run stale producers Co-Authored-By: Claude Fable 5 * fix(pipelines): watchdog review fixes — archived workspaces, badge kind parity, scan index Co-Authored-By: Claude Fable 5 * fix(pipelines): CI review — no singlestepflow in freshness, +N parity, completion-time fallback Co-Authored-By: Claude Fable 5 * fix(pipelines): CI review — history completedAt, freshness/asset trigger UI metadata Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 6f5fe0f7f56696fbef5a8349da38496c32e71666 This commit updates the EE repository reference after PR #643 was merged in windmill-ee-private. Previous ee-repo-ref: 1f13380354bf591ae25a2c20d36917534bcc5459 New ee-repo-ref: 6f5fe0f7f56696fbef5a8349da38496c32e71666 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...bc7e9ee85289ff5c52af43166928337c257e7.json | 3 +- ...91f3afd87eeea5e4e8d935b1b86eb02f061e7.json | 15 + ...adbb2860ce52d37a56b2861de1215260ecff8.json | 6 +- ...cb3adcf0175d0a71013f2497397c1c8ccc619.json | 3 +- ...66b49254036ca93f7a6415ceb4e316ed64eed.json | 29 ++ ...de8b6fcebe83002c74647666e3a6f5b8490dd.json | 23 ++ ...5ff3ae182ff7c0631d4fa44b67b11a133498f.json | 17 + ...c23caf312ec377dd91e5307f00d3fb8ec325d.json | 3 +- ...07667fc1b5e802dc2465ffd13178a042e4f89.json | 24 ++ ...f6d740a1720f7746c01f9ea693c70271ff55c.json | 17 + ...04af5a7eb488c76f783e690af0c2153b1d6a8.json | 3 +- ...6017386cf2767c953e0d152e7983a00218c6f.json | 15 + ...34128c3a6ade024d6662379857905a7447a2a.json | 32 ++ ...a995a8a555e4a9c2d1387dd8cee9c81efd40b.json | 15 + ...fbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json | 3 +- ...e33244fe8ab9a803248fc23f334034d24aad4.json | 3 +- ...5bd601b7dcabdf70e85543455cb4fc3986d74.json | 23 ++ ...bb6a8f961c119516e4c33dc928cd3b4f4edbc.json | 3 +- ...39a26f649c960a4ee8d0a584884a236129d43.json | 15 + ...77429c51e6166c414bcc5ce2e97fac25bcd77.json | 3 +- ...9746147aa9b8d0a81642a7b7cb4738a0cad66.json | 3 +- ...ad8bc7a2d226a9f22418782bcc66e46d0f706.json | 15 + ...e1ea2fe96a1b94f5a92953fb1c1642d15c016.json | 3 +- ...120e5b0f629f9f04e08a2982df33ff23ac7a9.json | 3 +- ...9352d4d1e3cb3a8a08f1c9c03055a1cab1235.json | 3 +- ...a088b5c0c1ad601146ff1c57d4a3ddc30556a.json | 34 ++ ...dba01f2f3a02bb36a8d7c20e1990dc81e0472.json | 22 ++ backend/ee-repo-ref.txt | 2 +- ...65613_pipeline_freshness_watchdog.down.sql | 5 + ...3165613_pipeline_freshness_watchdog.up.sql | 35 +++ .../windmill-parser/src/asset_parser.rs | 12 +- backend/src/monitor.rs | 31 ++ backend/tests/freshness_watchdog.rs | 294 ++++++++++++++++++ backend/windmill-api-assets/src/lib.rs | 57 ++++ backend/windmill-api-scripts/src/scripts.rs | 10 - backend/windmill-api/openapi.yaml | 1 + backend/windmill-common/src/assets.rs | 4 + .../src/pipeline_advanced_oss.rs | 16 +- .../src/freshness_watchdog_oss.rs | 9 + backend/windmill-queue/src/lib.rs | 7 + backend/windmill-types/src/jobs.rs | 4 + .../assets/AssetGraph/AssetGraphCanvas.svelte | 1 + .../assets/AssetGraph/RunnableNode.svelte | 64 +++- .../AssetGraph/activeRunnables.svelte.ts | 49 ++- .../parsePipelineAnnotations.test.ts | 38 +++ .../AssetGraph/parsePipelineAnnotations.ts | 22 ++ .../AssetGraph/pipelineHistory.svelte.ts | 10 +- .../lib/components/assets/AssetGraph/types.ts | 5 + frontend/src/lib/components/triggers/utils.ts | 30 +- .../(logged)/pipeline/[folder]/+page.svelte | 13 +- 50 files changed, 998 insertions(+), 59 deletions(-) create mode 100644 backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json create mode 100644 backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json create mode 100644 backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json create mode 100644 backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json create mode 100644 backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json create mode 100644 backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json create mode 100644 backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json create mode 100644 backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json create mode 100644 backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json create mode 100644 backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json create mode 100644 backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json create mode 100644 backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json create mode 100644 backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json create mode 100644 backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json create mode 100644 backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql create mode 100644 backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql create mode 100644 backend/tests/freshness_watchdog.rs create mode 100644 backend/windmill-queue/src/freshness_watchdog_oss.rs diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index a779aa0e95..6efb66005d 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json new file mode 100644 index 0000000000..906c710fdf --- /dev/null +++ b/backend/.sqlx/query-0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0e22295e40bfc173d85f0a962d491f3afd87eeea5e4e8d935b1b86eb02f061e7" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index f9bc97d29d..704883d4f1 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -39,7 +39,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } @@ -77,7 +78,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 3568d1723e..f5ee767768 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -128,7 +128,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json new file mode 100644 index 0000000000..60dbee76ab --- /dev/null +++ b/backend/.sqlx/query-3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT p.path AS \"path!\",\n (SELECT c.completed_at\n FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = p.path\n AND j.parent_job IS NULL\n -- No 'singlestepflow': flows may share a script's path, and\n -- a same-path flow run must not read as the script being\n -- fresh (false-fresh). Script retries land as native\n -- 'script' jobs; only the rare flow-wrapper fallback is\n -- missed, which errs stale. Kept in lockstep with the\n -- freshness watchdog's queries (freshness_watchdog_ee).\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n ORDER BY j.created_at DESC\n LIMIT 1) AS last_success_at\n FROM unnest($2::text[]) AS p(path)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "last_success_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3ea82f16050318a0b3b87c12beb66b49254036ca93f7a6415ceb4e316ed64eed" +} diff --git a/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json new file mode 100644 index 0000000000..5b3e70acbf --- /dev/null +++ b/backend/.sqlx/query-45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job_queue q\n JOIN v2_job j ON j.id = q.id\n WHERE q.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND (q.running = true OR q.scheduled_for <= now())\n ) AS \"in_flight!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "in_flight!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "45a539e61a3c35098fb6d59c3d5de8b6fcebe83002c74647666e3a6f5b8490dd" +} diff --git a/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json new file mode 100644 index 0000000000..6dc5fdb9a6 --- /dev/null +++ b/backend/.sqlx/query-45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE pipeline_freshness_state\n SET attempts = attempts + 1,\n last_push_at = now(),\n next_attempt_at = now()\n + (LEAST($3::bigint, $4::bigint * (1::bigint << LEAST(attempts + 1, 20)))::text\n || ' seconds')::interval\n WHERE workspace_id = $1 AND script_path = $2 AND next_attempt_at <= now()\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "45c73bd8742601753ad947c88b65ff3ae182ff7c0631d4fa44b67b11a133498f" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 7d950f6d8f..7474818bf3 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json new file mode 100644 index 0000000000..580666f07d --- /dev/null +++ b/backend/.sqlx/query-572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 FROM v2_job j\n JOIN v2_job_completed c ON c.id = j.id\n WHERE j.workspace_id = $1\n AND j.runnable_path = $2\n AND j.parent_job IS NULL\n AND j.kind IN ('script', 'preview')\n AND c.status = 'success'\n AND c.completed_at > now() - ($3::bigint::text || ' seconds')::interval\n ) AS \"fresh!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "fresh!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "572b80f7b16fd5d1a05f8fe9bbb07667fc1b5e802dc2465ffd13178a042e4f89" +} diff --git a/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json new file mode 100644 index 0000000000..18e0936d3a --- /dev/null +++ b/backend/.sqlx/query-5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO pipeline_freshness_state\n (workspace_id, script_path, attempts, last_push_at, next_attempt_at)\n VALUES ($1, $2, 1, now(),\n now() + (LEAST($3::bigint, $4::bigint * 2)::text || ' seconds')::interval)\n ON CONFLICT (workspace_id, script_path) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5e0569092c559a1b19f636fae68f6d740a1720f7746c01f9ea693c70271ff55c" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index 08ebe6bba5..fcc16e9a7c 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -161,7 +161,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json new file mode 100644 index 0000000000..4e8c7acee2 --- /dev/null +++ b/backend/.sqlx/query-6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second'\n WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "6d81570849980f316b08487a03f6017386cf2767c953e0d152e7983a00218c6f" +} diff --git a/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json new file mode 100644 index 0000000000..5c8ab6fb5b --- /dev/null +++ b/backend/.sqlx/query-7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT ON (workspace_id, path)\n workspace_id AS \"workspace_id!\", path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND content ILIKE '%freshness%'\n -- Workspace archival stops all execution but leaves script rows\n -- intact for unarchival; without this the watchdog would keep\n -- resurrecting runs in a workspace the admin shut down.\n AND EXISTS (SELECT 1 FROM workspace w\n WHERE w.id = script.workspace_id AND w.deleted = false)\n ORDER BY workspace_id, path, created_at DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "content!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "7cd0b27f4b4a20b05ac876861b934128c3a6ade024d6662379857905a7447a2a" +} diff --git a/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json new file mode 100644 index 0000000000..3f6a4a8136 --- /dev/null +++ b/backend/.sqlx/query-81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM pipeline_freshness_state s\n WHERE NOT EXISTS (\n SELECT 1 FROM unnest($1::text[], $2::text[]) AS w(workspace_id, script_path)\n WHERE w.workspace_id = s.workspace_id AND w.script_path = s.script_path\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "81d1c56c369c5328d947f1ad21da995a8a555e4a9c2d1387dd8cee9c81efd40b" +} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index ad9e57801e..ab01730c02 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -35,7 +35,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index 9a21f228ea..405904604a 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json new file mode 100644 index 0000000000..d5b10b13b9 --- /dev/null +++ b/backend/.sqlx/query-b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "attempts", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b10e21ec9edf2d22ff956fdcd085bd601b7dcabdf70e85543455cb4fc3986d74" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index 3efa843923..092d15e592 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -166,7 +166,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json new file mode 100644 index 0000000000..ce77275faf --- /dev/null +++ b/backend/.sqlx/query-bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bc6f566bc958923045dc3e1ff1639a26f649c960a4ee8d0a584884a236129d43" +} diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index f6ff25a4bf..419ab26383 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -80,7 +80,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index 8dc66064dc..b9d33a6b5f 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json new file mode 100644 index 0000000000..cbf96dad52 --- /dev/null +++ b/backend/.sqlx/query-d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "d1a1f64cb6b14c561c53c44910cad8bc7a2d226a9f22418782bcc66e46d0f706" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index d5365ffe94..d97c02d26b 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -111,7 +111,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index 33a5534b42..052d83fcd9 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -251,7 +251,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 470c651020..a35373a959 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -191,7 +191,8 @@ "ci_test", "github", "azure", - "asset" + "asset", + "freshness" ] } } diff --git a/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json new file mode 100644 index 0000000000..8c80c14ea4 --- /dev/null +++ b/backend/.sqlx/query-e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", created_by AS \"created_by!\",\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'freshness'\n ORDER BY created_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false, + true + ] + }, + "hash": "e50bce4bba01c90171973862bcfa088b5c0c1ad601146ff1c57d4a3ddc30556a" +} diff --git a/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json new file mode 100644 index 0000000000..8257722c5c --- /dev/null +++ b/backend/.sqlx/query-f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) AS \"count!\" FROM pipeline_freshness_state WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f26cde14890cc754e04341264c4dba01f2f3a02bb36a8d7c20e1990dc81e0472" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 857e48ba09..0cd164d1ce 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -81d8a449effdc540a6e8810668dd5d4aea0c485a +6f5fe0f7f56696fbef5a8349da38496c32e71666 diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql new file mode 100644 index 0000000000..aed2f9d32a --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.down.sql @@ -0,0 +1,5 @@ +-- Postgres has no ALTER TYPE ... DROP VALUE for enums. The 'freshness' value +-- stays even on rollback, consistent with prior job_trigger_kind additions +-- (see 20260510174213_asset_trigger_dispatch). +DROP INDEX IF EXISTS idx_script_pipeline_freshness_scan; +DROP TABLE IF EXISTS pipeline_freshness_state; diff --git a/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql new file mode 100644 index 0000000000..acf9545c96 --- /dev/null +++ b/backend/migrations/20260703165613_pipeline_freshness_watchdog.up.sql @@ -0,0 +1,35 @@ +-- Attribution for runs pushed by the pipeline freshness watchdog (the EE +-- background loop that re-runs a `// freshness`-annotated producer whose +-- output aged past its window). +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'freshness'; + +-- Per-(workspace, script) watchdog state: exponential-backoff bookkeeping so +-- a persistently failing producer isn't re-pushed on every scan tick, and an +-- atomic claim so concurrent servers can't double-push in the same tick +-- (claim = the UPDATE/INSERT that advances next_attempt_at; only the winner +-- pushes). Rows exist only while a script is stale — observing it fresh (or +-- its annotation gone) deletes the row, resetting the backoff. +CREATE TABLE pipeline_freshness_state ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + script_path VARCHAR(510) NOT NULL, + attempts INTEGER NOT NULL DEFAULT 1, + last_push_at TIMESTAMPTZ NOT NULL DEFAULT now(), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, script_path) +); + +-- Written only by the server monitor loop on the raw (non-RLS) pool, but +-- granted like every other app table so a future user-transaction reader +-- doesn't hit the recurring missing-GRANT class of bug. +GRANT ALL ON pipeline_freshness_state TO windmill_user; +GRANT ALL ON pipeline_freshness_state TO windmill_admin; + +-- The watchdog's ~60s candidate scan (latest deployed pipeline members) +-- filters on this exact predicate and orders by (workspace_id, path, +-- created_at DESC); without a matching partial index it seq-scans the whole +-- script-version heap on every tick, on instances that mostly have zero +-- pipeline scripts. (idx_script_pipeline_path is text_pattern_ops for +-- prefix LIKE — it can't serve this ordering.) +CREATE INDEX idx_script_pipeline_freshness_scan + ON script (workspace_id, path, created_at DESC) + WHERE auto_kind = 'pipeline' AND archived = false AND deleted = false; diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index c22359f6b0..299e3a70f9 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -77,10 +77,12 @@ pub struct ParseAssetsOutput { #[serde(skip_serializing_if = "Option::is_none", default)] pub partition: Option, // `// freshness ` — SLA stating outputs must be at most - // `duration` old. Active backstop: when no other trigger has fired the - // script within the window, a watchdog re-runs it. Distinct from - // schedule (which is producer cadence); freshness is consumer SLA and - // applies regardless of which trigger last fired. + // `duration` old. Drives passive monitoring in CE (the asset graph + // colors the node's badge fresh/stale against its last successful run) + // and the enterprise watchdog (windmill-queue `freshness_watchdog`), + // which re-runs a stale unpartitioned producer. Distinct from schedule + // (which is producer cadence); freshness is consumer SLA and applies + // regardless of which trigger last fired. #[serde(skip_serializing_if = "Option::is_none", default)] pub freshness: Option, // `// trigger all` → AND join barrier; default (`any`) = OR (current @@ -637,7 +639,7 @@ fn parse_kv_opts(s: &str) -> BTreeMap { // - `on ` → asset / native trigger edge (including // the marker-only `on schedule` form) // - `partitioned [opts]` → partition declaration -// - `freshness ` → SLA / active backstop +// - `freshness ` → SLA window (badge + EE watchdog) // - `tag ` → worker-tag override (annotation wins // over UI-set value at deploy) // - `retry []` → cascade-only retry policy diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 817887eaad..60b4024c0a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -178,6 +178,14 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(false); + // Ops kill switch for the pipeline freshness watchdog (a background + // pusher — being able to stop it without a redeploy matters more than + // for read-only monitors). + pub static ref DISABLE_FRESHNESS_WATCHDOG: bool = std::env::var("DISABLE_FRESHNESS_WATCHDOG") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + pub static ref WORKERS_NAMES: Arc>> = Arc::new(RwLock::new(Vec::new())); static ref QUEUE_COUNT_TAGS: Arc>> = Arc::new(RwLock::new(Vec::new())); @@ -2994,6 +3002,28 @@ pub async fn monitor_db( } }; + // run every ~60s (2 iterations * 30s). Enterprise feature: the active + // `// freshness` backstop lives in windmill-queue's `freshness_watchdog` + // (`private`); OSS gets a no-op stub. Runtime-gated on an Enterprise + // license like the audit export above. Safe on concurrent servers — the + // watchdog claims per-script state rows atomically before pushing. + let pipeline_freshness_watchdog_f = async { + if server_mode + && !*DISABLE_FRESHNESS_WATCHDOG + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(2) + { + if let Some(db) = conn.as_sql() { + if matches!( + windmill_common::ee_oss::get_license_plan().await, + windmill_common::ee_oss::LicensePlan::Enterprise + ) { + windmill_queue::freshness_watchdog::tick(db).await; + } + } + } + }; + join!( expired_items_f, zombie_jobs_f, @@ -3021,6 +3051,7 @@ pub async fn monitor_db( manage_audit_partitions_f, export_audit_logs_to_object_store_f, cleanup_scheduled_job_deletions_f, + pipeline_freshness_watchdog_f, ); } diff --git a/backend/tests/freshness_watchdog.rs b/backend/tests/freshness_watchdog.rs new file mode 100644 index 0000000000..17851d0a1e --- /dev/null +++ b/backend/tests/freshness_watchdog.rs @@ -0,0 +1,294 @@ +//! End-to-end tests for the pipeline freshness watchdog (Enterprise). +//! +//! `windmill_queue::freshness_watchdog::tick` is called directly against +//! seeded `script` / `v2_job(_completed)` rows — no worker or API server is +//! needed, since the watchdog's job ends at the push (the pushed job sitting +//! in `v2_job_queue` is itself part of the assertions). Covers: staleness on +//! never-ran and aged-out members, the fresh short-circuit + state reset, +//! the in-flight suppression, the backoff claim, and the skip rules +//! (partitioned, malformed window, non-pipeline scripts). + +#![cfg(feature = "private")] + +use sqlx::{Pool, Postgres}; +use windmill_queue::freshness_watchdog::tick; +use windmill_test_utils::initialize_tracing; + +const WS: &str = "test-workspace"; +const PATH: &str = "u/test-user/freshness_producer"; + +/// Seed a deployed pipeline-member script. Mirrors the deploy path's output: +/// `auto_kind = 'pipeline'`, empty (non-NULL) lock so run-by-path resolution +/// treats it as deployed, hash derived from path+content for uniqueness. +async fn seed_pipeline_script( + db: &Pool, + path: &str, + content: &str, +) -> anyhow::Result<()> { + let mut h = 0i64; + for b in path.bytes().chain(content.bytes()) { + h = h.wrapping_mul(31).wrapping_add(b as i64); + } + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, tag, lock, auto_kind) + VALUES ($1, $2, $3, '', '', $4, 'test-user', 'bash'::script_lang, 'bash', '', 'pipeline') + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(h) + .bind(path) + .bind(content) + .execute(db) + .await?; + // Process-global deployed-script caches are keyed by (workspace, path) / + // (workspace, hash) and would leak between #[sqlx::test] isolated DBs + // that reuse both — resolve everything from this test's own DB. + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) +} + +/// Seed a completed root run of `path` that finished `age_s` seconds ago. +async fn seed_completed_run( + db: &Pool, + path: &str, + age_s: i64, + success: bool, +) -> anyhow::Result<()> { + let id = uuid::Uuid::new_v4(); + sqlx::query( + r#"INSERT INTO v2_job (id, workspace_id, runnable_path, kind, created_at, + created_by, permissioned_as, permissioned_as_email, tag) + VALUES ($1, $2, $3, 'script'::job_kind, + now() - ($4::bigint::text || ' seconds')::interval, + 'test-user', 'u/test-user', 'test@windmill.dev', 'bash')"#, + ) + .bind(id) + .bind(WS) + .bind(path) + .bind(age_s) + .execute(db) + .await?; + sqlx::query( + r#"INSERT INTO v2_job_completed (id, workspace_id, duration_ms, status, started_at, completed_at) + VALUES ($1, $2, 0, CASE WHEN $3 THEN 'success'::job_status ELSE 'failure'::job_status END, + now() - ($4::bigint::text || ' seconds')::interval, + now() - ($4::bigint::text || ' seconds')::interval)"#, + ) + .bind(id) + .bind(WS) + .bind(success) + .bind(age_s) + .execute(db) + .await?; + Ok(()) +} + +/// Jobs the watchdog pushed: (path, created_by, args) rows attributed to +/// `trigger_kind = 'freshness'`. +async fn fetch_pushed( + db: &Pool, +) -> anyhow::Result)>> { + let rows = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", created_by AS "created_by!", + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'freshness' + ORDER BY created_at"#, + WS, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.runnable_path, r.created_by, r.args.map(|a| a.0))) + .collect()) +} + +async fn state_row(db: &Pool, path: &str) -> anyhow::Result> { + Ok(sqlx::query_scalar!( + "SELECT attempts FROM pipeline_freshness_state WHERE workspace_id = $1 AND script_path = $2", + WS, + path, + ) + .fetch_optional(db) + .await?) +} + +#[sqlx::test(fixtures("base"))] +async fn never_ran_member_is_pushed_once(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + + tick(&db).await; + + let pushed = fetch_pushed(&db).await?; + assert_eq!(pushed.len(), 1, "one watchdog push expected"); + let (path, created_by, args) = &pushed[0]; + assert_eq!(path, PATH); + assert_eq!(created_by, &format!("freshness-{PATH}")); + let args = args.as_ref().expect("args recorded"); + assert_eq!( + args.get("_wmill_skip_asset_dispatch"), + Some(&serde_json::json!(true)), + "watchdog runs must not re-fire the cascade" + ); + assert_eq!( + args.pointer("/trigger/kind"), + Some(&serde_json::json!("freshness")) + ); + assert_eq!(state_row(&db, PATH).await?, Some(1), "claim row recorded"); + + // Second tick: the pushed job is queued-and-due, so the in-flight guard + // suppresses a duplicate regardless of backoff. + tick(&db).await; + assert_eq!( + fetch_pushed(&db).await?.len(), + 1, + "no duplicate while queued" + ); + + // Simulate the queued job vanishing without a completion: the backoff + // claim (next_attempt_at in the future) now carries the suppression. + sqlx::query!("DELETE FROM v2_job_queue WHERE workspace_id = $1", WS) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 1, "backoff holds the retry"); + + // Force the backoff window open: the watchdog retries and escalates. + sqlx::query!( + "UPDATE pipeline_freshness_state SET next_attempt_at = now() - interval '1 second' + WHERE workspace_id = $1 AND script_path = $2", + WS, + PATH, + ) + .execute(&db) + .await?; + tick(&db).await; + assert_eq!(fetch_pushed(&db).await?.len(), 2, "due retry pushed"); + assert_eq!(state_row(&db, PATH).await?, Some(2), "attempts escalated"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn fresh_member_is_skipped_and_state_reset(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + seed_completed_run(&db, PATH, 10, true).await?; + // Leftover backoff row from an earlier staleness episode. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path, attempts) VALUES ($1, $2, 3)", + WS, + PATH, + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "fresh member not pushed" + ); + assert_eq!(state_row(&db, PATH).await?, None, "backoff reset on fresh"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn aged_out_member_is_pushed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 1h\necho hi\n").await?; + // Old success outside the window + a recent failure: still stale. + seed_completed_run(&db, PATH, 7200, true).await?; + seed_completed_run(&db, PATH, 60, false).await?; + + tick(&db).await; + + assert_eq!(fetch_pushed(&db).await?.len(), 1, "aged-out member pushed"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn partitioned_malformed_and_plain_members_are_skipped( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + // Partitioned: freshness means partition-gap detection, out of scope. + seed_pipeline_script( + &db, + "u/test-user/partitioned", + "# pipeline\n# partitioned daily\n# freshness 1h\necho hi\n", + ) + .await?; + // Malformed window: fails safe to unwatched. + seed_pipeline_script( + &db, + "u/test-user/malformed", + "# pipeline\n# freshness soonish\necho hi\n", + ) + .await?; + // Freshness only in prose (parser must reject; ILIKE prefilter passes). + seed_pipeline_script( + &db, + "u/test-user/prose", + "# pipeline\n# ensure freshness of data below\necho hi\n", + ) + .await?; + + tick(&db).await; + + assert!(fetch_pushed(&db).await?.is_empty(), "no member is watched"); + let rows = sqlx::query_scalar!( + r#"SELECT COUNT(*) AS "count!" FROM pipeline_freshness_state WHERE workspace_id = $1"#, + WS, + ) + .fetch_one(&db) + .await?; + assert_eq!(rows, 0, "no state rows for unwatched members"); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn archived_workspace_is_not_resurrected(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // Workspace archival stops all execution but keeps script rows for + // unarchival — the watchdog must not keep pushing runs there. + seed_pipeline_script(&db, PATH, "# pipeline\n# freshness 30s\necho hi\n").await?; + sqlx::query!("UPDATE workspace SET deleted = true WHERE id = $1", WS) + .execute(&db) + .await?; + + tick(&db).await; + + assert!( + fetch_pushed(&db).await?.is_empty(), + "no pushes into an archived workspace" + ); + assert_eq!( + state_row(&db, PATH).await?, + None, + "no state bookkeeping either" + ); + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn state_of_unwatched_member_is_cleaned_up(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + // A stale-bookkeeping row whose script no longer declares freshness + // (e.g. annotation removed and redeployed) must not survive the sweep. + sqlx::query!( + "INSERT INTO pipeline_freshness_state (workspace_id, script_path) VALUES ($1, $2)", + WS, + "u/test-user/gone", + ) + .execute(&db) + .await?; + + tick(&db).await; + + assert_eq!(state_row(&db, "u/test-user/gone").await?, None); + Ok(()) +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index b4b0583863..2eb24cee6c 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -632,6 +632,14 @@ struct GraphRunnableNode { partition_kind: Option, #[serde(skip_serializing_if = "Option::is_none", default)] freshness: Option, + // Completion time of the most recently started successful run of this + // pipeline member. The canvas checks it against the `// freshness` window + // to color the badge fresh/stale. The badge itself is passive; on EE the + // freshness watchdog (windmill-queue) separately re-runs stale + // unpartitioned producers. Absent when no successful run is visible to + // the caller (job RLS applies). + #[serde(skip_serializing_if = "Option::is_none", default)] + last_success_at: Option>, #[serde(skip_serializing_if = "Option::is_none", default)] tag: Option, #[serde(skip_serializing_if = "Option::is_none", default)] @@ -920,6 +928,46 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // Newest successful completed run per pipeline member, for the passive + // freshness status on the canvas. Correlated per-path lookup walks + // ix_job_root_job_index_by_path_2 newest-first until the first success, + // so cost is bounded by the member count, not run history. "Newest" is + // by created_at (the index order), not completed_at: with overlapping + // runs of one path this can pick an earlier completion, erring toward + // stale — never toward false-fresh. Inside the user tx so job-visibility + // RLS applies — a caller who can't see the runs gets no timestamp rather + // than leaked completion times. + let member_paths: Vec = pipeline_member_paths + .iter() + .map(|r| r.path.clone()) + .collect(); + let last_success_rows = sqlx::query!( + r#" + SELECT p.path AS "path!", + (SELECT c.completed_at + FROM v2_job j + JOIN v2_job_completed c ON c.id = j.id + WHERE j.workspace_id = $1 + AND j.runnable_path = p.path + AND j.parent_job IS NULL + -- No 'singlestepflow': flows may share a script's path, and + -- a same-path flow run must not read as the script being + -- fresh (false-fresh). Script retries land as native + -- 'script' jobs; only the rare flow-wrapper fallback is + -- missed, which errs stale. Kept in lockstep with the + -- freshness watchdog's queries (freshness_watchdog_ee). + AND j.kind IN ('script', 'preview') + AND c.status = 'success' + ORDER BY j.created_at DESC + LIMIT 1) AS last_success_at + FROM unnest($2::text[]) AS p(path) + "#, + &w_id, + &member_paths, + ) + .fetch_all(&mut *tx) + .await?; + // Existing scripts / flows in the workspace. Used to filter out // orphan trigger rows whose `script_path` no longer resolves — those // would otherwise be added to `runnable_set` below and surface as @@ -1010,6 +1058,11 @@ async fn asset_graph( (r.path.clone(), lineage) }) .collect(); + let last_success_by_path: std::collections::HashMap> = + last_success_rows + .into_iter() + .filter_map(|r| r.last_success_at.map(|t| (r.path, t))) + .collect(); let pipeline_member_script_paths: std::collections::HashSet = pipeline_member_paths.into_iter().map(|r| r.path).collect(); let existing_script_paths: std::collections::HashSet = @@ -1215,6 +1268,10 @@ async fn asset_graph( freshness: ann .and_then(|a| a.freshness.as_ref()) .map(|f| f.duration.clone()), + last_success_at: (usage_kind == AssetUsageKind::Script) + .then(|| last_success_by_path.get(&path)) + .flatten() + .copied(), tag: ann.and_then(|a| a.tag.clone()), retry: ann.and_then(|a| a.retry.clone()), data_tests: ann.map(|a| a.data_tests.clone()).unwrap_or_default(), diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 4f9de4f416..49a4091f07 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1252,16 +1252,6 @@ async fn create_script_internal<'c>( // membership; parsed writes tell us what is produced (we don't record // them in auto_kind itself). let pipeline_annotations = parse_pipeline_annotations(&ns.content); - // `// freshness` is parsed but enforcement is a not-yet-implemented - // enterprise feature (skeleton in windmill_common::pipeline_advanced). - // Surface a clear TODO at deploy rather than silently accepting an - // annotation that does nothing. - if pipeline_annotations.freshness.is_some() { - tracing::warn!( - "{}", - windmill_common::pipeline_advanced::freshness_enforcement_todo() - ); - } // `// materialize` materializes a `ducklake:///` target from a // DuckDB script. These two constraints hold for *both* modes: a non-DuckLake // target would otherwise deploy, register a producer in the asset graph, then diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 01ae7229e6..fbad5fffd4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -25592,6 +25592,7 @@ components: - google - github - asset + - freshness TriggerMode: description: job trigger mode diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 0757ef6894..722ad29747 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -352,6 +352,10 @@ mod debounce_duration_tests { assert_eq!(parse_duration_secs("5m"), Some(300)); assert_eq!(parse_duration_secs("2h"), Some(7200)); assert_eq!(parse_duration_secs(" 1d "), Some(86400)); + // Explicit plus sign comes free with i64 parsing; the TS mirror + // (parseDurationSecs) matches it — keep the two in lockstep. + assert_eq!(parse_duration_secs("+5m"), Some(300)); + assert_eq!(parse_duration_secs("+45"), Some(45)); } #[test] diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs index 009e783bd5..15bb4b127a 100644 --- a/backend/windmill-common/src/pipeline_advanced_oss.rs +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -1,16 +1,12 @@ -//! OSS fallback: pipeline freshness/SLA enforcement and partition backfills -//! are enterprise features; their implementations live in windmill-ee-private -//! (see `pipeline_advanced_ee`). In the public build the entry points report -//! that the enterprise edition is required. +//! OSS fallback: pipeline partition backfills are an enterprise feature; +//! their implementations live in windmill-ee-private (see +//! `pipeline_advanced_ee`). In the public build the entry points report that +//! the enterprise edition is required. (Freshness lives elsewhere: the +//! fresh/stale badge is CE in the assets API, the active watchdog is +//! windmill-queue's `freshness_watchdog`.) use crate::error::Error; -pub fn freshness_enforcement_todo() -> Error { - Error::internal_err( - "Pipeline freshness/SLA enforcement requires the enterprise edition".to_string(), - ) -} - pub fn backfill_todo() -> Error { Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) } diff --git a/backend/windmill-queue/src/freshness_watchdog_oss.rs b/backend/windmill-queue/src/freshness_watchdog_oss.rs new file mode 100644 index 0000000000..319fb25f62 --- /dev/null +++ b/backend/windmill-queue/src/freshness_watchdog_oss.rs @@ -0,0 +1,9 @@ +//! OSS fallback for the pipeline freshness watchdog. The active backstop — +//! re-running a `// freshness`-annotated producer whose output aged past its +//! window — is an enterprise feature (see `freshness_watchdog_ee`). In the +//! public build the tick is a no-op; CE keeps the passive fresh/stale badge +//! on the asset graph. + +use windmill_common::DB; + +pub async fn tick(_db: &DB) {} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 6f689c7026..59ac7f1fa8 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -14,6 +14,13 @@ pub mod cascade_oss; pub use cascade_ee as cascade; #[cfg(not(feature = "private"))] pub use cascade_oss as cascade; +#[cfg(feature = "private")] +pub mod freshness_watchdog_ee; +pub mod freshness_watchdog_oss; +#[cfg(feature = "private")] +pub use freshness_watchdog_ee as freshness_watchdog; +#[cfg(not(feature = "private"))] +pub use freshness_watchdog_oss as freshness_watchdog; pub mod jobs; #[cfg(feature = "private")] pub mod jobs_ee; diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 448e6f4d01..f9372b949d 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -46,6 +46,9 @@ pub enum JobTriggerKind { // A run dispatched because an upstream pipeline script wrote an asset // this runnable subscribes to via `// on s3://...` annotations. Asset, + // A run pushed by the pipeline freshness watchdog (EE) because the + // script's `// freshness` window elapsed without a successful run. + Freshness, } impl std::fmt::Display for JobTriggerKind { @@ -68,6 +71,7 @@ impl std::fmt::Display for JobTriggerKind { JobTriggerKind::Github => "github", JobTriggerKind::CiTest => "ci_test", JobTriggerKind::Asset => "asset", + JobTriggerKind::Freshness => "freshness", }; write!(f, "{}", kind) } diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 1c5cf2dde6..6352e6adc9 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -401,6 +401,7 @@ in_pipeline: r.in_pipeline ?? false, partition_kind: r.partition_kind, freshness: r.freshness, + last_success_at: r.last_success_at, tag: r.tag, retry: r.retry, macros: r.macros, diff --git a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte index 8b5e90170c..6bbf8f5656 100644 --- a/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte @@ -22,12 +22,13 @@ import { preventDefault, stopPropagation } from 'svelte/legacy' import type { GraphUsageKind } from './types' import type { RunnableRunState } from './activeRunnables.svelte' + import { parseDurationSecs } from './parsePipelineAnnotations' import { NODE } from '$lib/components/graph/util' import DropdownV2 from '$lib/components/DropdownV2.svelte' import Popover from '$lib/components/meltComponents/Popover.svelte' import type { Item } from '$lib/utils' import { workspaceStore } from '$lib/stores' - import { sendUserToast } from '$lib/utils' + import { sendUserToast, msToReadableTimeShort } from '$lib/utils' interface Props { data: { @@ -36,6 +37,9 @@ in_pipeline?: boolean partition_kind?: 'daily' | 'hourly' | 'weekly' | 'monthly' | 'dynamic' freshness?: string + // Completion time (ISO) of the newest successful run visible to + // the caller. With `freshness`, drives the fresh/stale chip state. + last_success_at?: string tag?: string retry?: { count: number; delay?: string } // Macros this script provides (deployed/drafted `// macros` library). @@ -122,6 +126,44 @@ } } + // Freshness verdict: newest successful run (server `last_success_at`, + // or a newer one the session poll observed) vs the `// freshness` + // window. No verdict (undefined) for drafts — no run history — and for + // unparseable windows; the chip then stays neutral like the other + // annotation chips. + let freshnessWindowS = $derived(data.freshness ? parseDurationSecs(data.freshness) : undefined) + // Ticks so a node crosses fresh→stale while the canvas stays open (the + // graph payload is static between refetches). Armed only when a verdict + // is rendered. + let nowMs = $state(Date.now()) + $effect(() => { + if (freshnessWindowS === undefined || data.unsaved) return + const id = setInterval(() => (nowMs = Date.now()), 30_000) + return () => clearInterval(id) + }) + let lastSuccessMs = $derived.by(() => { + const server = data.last_success_at ? new Date(data.last_success_at).getTime() : undefined + const polled = data.runState?.lastSuccessAt + ? new Date(data.runState.lastSuccessAt).getTime() + : undefined + if (server === undefined) return polled + return polled === undefined ? server : Math.max(server, polled) + }) + let freshnessState = $derived.by((): 'fresh' | 'stale' | undefined => { + if (freshnessWindowS === undefined || data.unsaved) return undefined + if (lastSuccessMs === undefined) return 'stale' + return nowMs - lastSuccessMs <= freshnessWindowS * 1000 ? 'fresh' : 'stale' + }) + let freshnessTooltip = $derived.by(() => { + const base = `// freshness ${data.freshness}` + if (freshnessState === undefined) return base + if (lastSuccessMs === undefined) return `${base} — stale: no successful run yet` + const ago = msToReadableTimeShort(Math.max(0, nowMs - lastSuccessMs)) + return freshnessState === 'fresh' + ? `${base} — fresh: last successful run ${ago} ago` + : `${base} — stale: last successful run ${ago} ago` + }) + // Cascade + bounded-run options live on the Run button's caret popover // (whenever there's a cascade OR a bounded-run start — see `hasCaret` // below), so the kebab menu stays focused on lifecycle actions only. @@ -164,8 +206,8 @@ + guidelines). Only the freshness chip (when it has a verdict) + and the run-state chip below use semantic colors. --> {#if data.partition_kind}
{data.partition_kind}
{/if} + {#if data.freshness}
{data.freshness} diff --git a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts index 917e21766a..f6682804c7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts @@ -1,8 +1,15 @@ import { JobService } from '$lib/gen' export type RunStatus = 'running' | 'success' | 'failure' -/** Per-runnable badge state: latest run status + runs observed this session. */ -export type RunnableRunState = { status: RunStatus; runs: number } +/** + * Per-runnable badge state: latest run status + runs observed this session. + * `lastSuccessAt` is the completion time (start + duration when the listing + * carries it, else start as a conservative lower bound) of the newest + * successful run seen by the poll — lets the freshness chip go green right + * after an in-session run, ahead of the next graph refetch (whose + * `last_success_at` would carry it). + */ +export type RunnableRunState = { status: RunStatus; runs: number; lastSuccessAt?: string } export type EventStatus = 'queued' | 'running' | 'success' | 'failure' /** One folder activity-log row (a job observed by the poll). */ @@ -14,6 +21,12 @@ export type PipelineEvent = { /** What started it, as far as the job listing reveals. */ source: 'schedule' | 'run' at: string + /** + * Completion time (start + duration) for completed rows. The freshness + * chip compares against completion — `at` is the start time and would + * read a long run as older than its output actually is. + */ + completedAt?: string /** * Queued jobs: when the job is due to start. A future value means a * scheduled run waiting for its cron tick, not pipeline activity. @@ -45,7 +58,8 @@ function statesEq(a: Map, b: Map() const countedJobIds = new Set() // Job ids we've observed in-flight at least once. The catch-up pulse is @@ -241,10 +255,25 @@ export function useActiveRunnableIds( const prev = completedHistory.get(id) const status: RunStatus = (j as any).success === true ? 'success' : 'failure' const ts = startedTs ?? new Date(pollStartedMs).toISOString() + // Freshness compares against COMPLETION time (that's + // when the output materialized — the server-side + // last_success_at is completed_at too). The listing + // only carries started_at, so add duration_ms; when + // absent, the start is a conservative lower bound + // (errs stale, never false-fresh). + const durationMs = (j as any).duration_ms + const doneTs = + typeof durationMs === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + durationMs).toISOString() + : ts completedHistory.set(id, { runs: (prev?.runs ?? 0) + 1, lastStatus: !prev || ts >= prev.lastTs ? status : prev.lastStatus, - lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs + lastTs: !prev || ts >= prev.lastTs ? ts : prev.lastTs, + lastSuccessTs: + status === 'success' && (!prev?.lastSuccessTs || doneTs >= prev.lastSuccessTs) + ? doneTs + : prev?.lastSuccessTs }) } } @@ -267,6 +296,10 @@ export function useActiveRunnableIds( : 'failure', source: (j as any).schedule_path ? 'schedule' : 'run', at: startedTs ?? new Date(pollStartedMs).toISOString(), + completedAt: + !isQueued && typeof (j as any).duration_ms === 'number' && startedTs + ? new Date(new Date(startedTs).getTime() + (j as any).duration_ms).toISOString() + : undefined, scheduledFor: isQueued ? ((j as any).scheduled_for as string | undefined) : undefined }) } @@ -289,7 +322,11 @@ export function useActiveRunnableIds( // previous badge state until a worker picks the job up. const snap = new Map() for (const [id, h] of completedHistory) { - snap.set(id, { status: runningThisTick.has(id) ? 'running' : h.lastStatus, runs: h.runs }) + snap.set(id, { + status: runningThisTick.has(id) ? 'running' : h.lastStatus, + runs: h.runs, + lastSuccessAt: h.lastSuccessTs + }) } for (const id of runningThisTick) { if (!snap.has(id)) snap.set(id, { status: 'running', runs: 0 }) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts index 315c7526f3..e9e23525ec 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { mergeColumnLineage, + parseDurationSecs, parsePipelineAnnotations, type ColumnLineage } from './parsePipelineAnnotations' @@ -188,3 +189,40 @@ describe('mergeColumnLineage', () => { expect(mergeColumnLineage([], annotated)).toEqual(annotated) }) }) + +// Mirror of the Rust `parse_duration_secs` tests (windmill-common assets.rs) +// — the freshness chip's staleness verdict depends on identical parsing. +describe('parseDurationSecs', () => { + it('parses suffixed durations', () => { + expect(parseDurationSecs('30s')).toBe(30) + expect(parseDurationSecs('5m')).toBe(300) + expect(parseDurationSecs('2h')).toBe(7200) + expect(parseDurationSecs('1d')).toBe(86400) + }) + + it('bare integer means seconds', () => { + expect(parseDurationSecs('45')).toBe(45) + }) + + it('tolerates surrounding whitespace', () => { + expect(parseDurationSecs(' 5 m ')).toBe(300) + }) + + it('accepts an explicit plus sign (Rust i64 parsing does)', () => { + expect(parseDurationSecs('+5m')).toBe(300) + expect(parseDurationSecs('+45')).toBe(45) + }) + + it('rejects malformed / non-positive input', () => { + expect(parseDurationSecs('')).toBeUndefined() + expect(parseDurationSecs('h')).toBeUndefined() + expect(parseDurationSecs('1.5h')).toBeUndefined() + expect(parseDurationSecs('-5m')).toBeUndefined() + expect(parseDurationSecs('0')).toBeUndefined() + expect(parseDurationSecs('fast')).toBeUndefined() + }) + + it('rejects values beyond i32 seconds (mirrors backend cap)', () => { + expect(parseDurationSecs('999999999d')).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts index f33a03fff5..964e48570a 100644 --- a/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts +++ b/frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts @@ -67,6 +67,28 @@ export type FreshnessSpec = { duration: string } +// Mirrors backend `parse_duration_secs` (windmill-common assets.rs): a bare +// integer means seconds, otherwise `` with an `s`/`m`/`h`/`d` suffix +// (e.g. `30s`, `5m`, `2h`, `1d`). Returns undefined for malformed or +// non-positive input so a typo'd `// freshness` window fails safe (the chip +// stays neutral instead of guessing a staleness verdict). +export function parseDurationSecs(s: string): number | undefined { + const t = s.trim() + if (!t) return undefined + const last = t[t.length - 1] + const mult = + last === 's' ? 1 : last === 'm' ? 60 : last === 'h' ? 3600 : last === 'd' ? 86400 : undefined + const num = (mult !== undefined ? t.slice(0, -1) : t).trim() + // `+?`: Rust's i64 parsing accepts an explicit plus sign (`+5m`), so the + // mirror must too — divergence here would leave the chip neutral for a + // window the deploy path and watchdog honor. + if (mult === undefined && !/^\+?\d+$/.test(t)) return undefined + if (!/^\+?\d+$/.test(num)) return undefined + const secs = Number(num) * (mult ?? 1) + if (!Number.isSafeInteger(secs) || secs <= 0 || secs > 2147483647) return undefined + return secs +} + // `// retry []` — see backend RetrySpec. Delay is kept as the // raw duration string and resolved to seconds at deploy. export type RetrySpec = { diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts index 50ad3e1d6e..d654085179 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts @@ -80,7 +80,15 @@ export function usePipelineHistory( kind: j.job_kind.startsWith('flow') ? 'flow' : 'script', status: j.success ? 'success' : 'failure', source: j.schedule_path ? 'schedule' : 'run', - at: j.started_at ?? j.created_at + at: j.started_at ?? j.created_at, + // Same completion-time derivation as the live poll — + // the freshness chip compares against completion, and + // `at` (start) would read a long run as older than its + // output actually is. + completedAt: + j.started_at != undefined + ? new Date(new Date(j.started_at).getTime() + j.duration_ms).toISOString() + : undefined }) } sawFullPage = rows.length === PER_PAGE diff --git a/frontend/src/lib/components/assets/AssetGraph/types.ts b/frontend/src/lib/components/assets/AssetGraph/types.ts index 0db259a389..fb7b5b1de7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/types.ts +++ b/frontend/src/lib/components/assets/AssetGraph/types.ts @@ -22,6 +22,11 @@ export interface AssetGraphRunnableNode { // Raw `// freshness ` value, e.g. "1h", "30m". Surfaced for // the badge; the runtime parses it as needed. freshness?: string + // Completion time (ISO) of the newest successful run of this pipeline + // member visible to the caller. The freshness chip compares it against + // the `// freshness` window to render fresh/stale. Absent = no + // successful run found (or none visible under job RLS). + last_success_at?: string // `// tag ` worker-tag override. Surfaced for the badge so users // can see which worker pool will pick this script up at a glance. tag?: string diff --git a/frontend/src/lib/components/triggers/utils.ts b/frontend/src/lib/components/triggers/utils.ts index 231de5cea4..3d17491791 100644 --- a/frontend/src/lib/components/triggers/utils.ts +++ b/frontend/src/lib/components/triggers/utils.ts @@ -1,4 +1,14 @@ -import { Webhook, Mail, Calendar, Route, Unplug, Database, Terminal } from 'lucide-svelte' +import { + Webhook, + Mail, + Calendar, + Route, + Unplug, + Database, + Terminal, + Timer, + Zap +} from 'lucide-svelte' import KafkaIcon from '$lib/components/icons/KafkaIcon.svelte' import NatsIcon from '$lib/components/icons/NatsIcon.svelte' import MqttIcon from '$lib/components/icons/MqttIcon.svelte' @@ -82,7 +92,8 @@ export const jobTriggerKinds: JobTriggerKind[] = [ 'azure', 'google', 'github', - 'asset' + 'asset', + 'freshness' ] export type Trigger = { @@ -118,7 +129,12 @@ export const triggerIconMap = { cli: Terminal, nextcloud: NextcloudIcon, google: GoogleIcon, - github: GithubIcon + github: GithubIcon, + // Job-attribution-only kinds (no trigger CRUD page): the pipeline asset + // cascade and the freshness watchdog. Needed so the Runs filter and job + // detail render these trigger kinds instead of a blank label / no icon. + asset: Zap, + freshness: Timer } export const triggerDisplayNamesMap = { @@ -139,8 +155,12 @@ export const triggerDisplayNamesMap = { cli: 'CLI', nextcloud: 'Nextcloud', google: 'Google', - github: 'GitHub' -} as const satisfies Record + github: 'GitHub', + asset: 'Asset cascade', + freshness: 'Freshness' + // `asset` / `freshness` are job-attribution-only (JobTriggerKind, not + // TriggerType) — hence the union in the satisfies below. +} as const satisfies Record /** * Converts a TriggerType to a CaptureTriggerKind when a mapping exists diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 4d41f7ff05..845a93505b 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -1337,8 +1337,17 @@ ? 'failure' : 'success' const cur = m.get(id) - if (cur) cur.runs += 1 - else m.set(id, { status, runs: 1 }) + // Freshness compares against completion; `at` (start) is the + // fallback lower bound for rows without a duration. + const successAt = e.status === 'success' ? (e.completedAt ?? e.at) : undefined + if (cur) { + cur.runs += 1 + // Newest-first, so the first success per id is the latest one — + // it feeds the freshness chip between graph refetches. + if (successAt && !cur.lastSuccessAt) cur.lastSuccessAt = successAt + } else { + m.set(id, { status, runs: 1, lastSuccessAt: successAt }) + } } return m }) From 42e11c6570b62ffaa86598438fa8ddf462c4035f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 08:40:00 +0000 Subject: [PATCH 02/34] =?UTF-8?q?feat(pipelines):=20schema=20contracts=20?= =?UTF-8?q?=E2=80=94=20save-time=20consumer=20checks=20vs=20captured=20sch?= =?UTF-8?q?emas=20(#9917)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): schema contracts — save-time consumer checks vs captured schemas Co-Authored-By: Claude Fable 5 * refactor: move schemaContractContext above schemaCanEvolve doc comment Co-Authored-By: Claude Fable 5 * fix: emit scd2/on_schema_change in CLI local graph, address review notes Co-Authored-By: Claude Fable 5 * fix: gate editor _current ignore-suppression on scd2, matching backend Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...368a6862e35350ae43383fb2335e2c82a9419.json | 29 + ...eb5a0665c90c4926f1afb689d6cd16bd5b722.json | 29 + ...3ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json | 41 ++ .../windmill-parser/src/asset_parser.rs | 67 ++ .../tests/fixtures/pipeline_annotations.json | 37 ++ .../tests/pipeline_annotations_parity.rs | 15 + backend/windmill-api-assets/src/lib.rs | 17 + backend/windmill-api-scripts/src/scripts.rs | 53 ++ backend/windmill-api/openapi.yaml | 77 +++ backend/windmill-common/src/assets.rs | 2 +- backend/windmill-common/src/lib.rs | 1 + .../windmill-common/src/schema_contracts.rs | 614 ++++++++++++++++++ cli/src/commands/pipeline/localGraph.ts | 33 +- docs/ducklake-materialization.md | 31 + docs/pipelines-vs-dbt.md | 44 +- frontend/src/lib/components/Editor.svelte | 91 ++- .../src/lib/components/ScriptBuilder.svelte | 6 + .../src/lib/components/ScriptEditor.svelte | 43 +- .../AssetGraph/AssetGraphDetailsPane.svelte | 12 +- .../AssetGraph/PipelineGraphEditor.svelte | 6 + .../parsePipelineAnnotations.parity.test.ts | 5 + .../AssetGraph/parsePipelineAnnotations.ts | 13 +- .../assets/AssetGraph/schemaContracts.test.ts | 276 ++++++++ .../assets/AssetGraph/schemaContracts.ts | 440 +++++++++++++ .../lib/components/assets/AssetGraph/types.ts | 11 +- .../(logged)/pipeline/[folder]/+page.svelte | 8 + 26 files changed, 1972 insertions(+), 29 deletions(-) create mode 100644 backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json create mode 100644 backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json create mode 100644 backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json create mode 100644 backend/windmill-common/src/schema_contracts.rs create mode 100644 frontend/src/lib/components/assets/AssetGraph/schemaContracts.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts diff --git a/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json new file mode 100644 index 0000000000..d05abd4f3a --- /dev/null +++ b/backend/.sqlx/query-36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT path AS \"asset_path!\", usage_path AS \"producer_path!\"\n FROM asset\n WHERE workspace_id = $1 AND kind = 'ducklake' AND path = ANY($2)\n AND usage_kind = 'script' AND usage_access_type IN ('w', 'rw')", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "producer_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "36da136cf9b9554702e3b3ec364368a6862e35350ae43383fb2335e2c82a9419" +} diff --git a/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json new file mode 100644 index 0000000000..b554ef8b55 --- /dev/null +++ b/backend/.sqlx/query-431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path, content\n FROM script\n WHERE workspace_id = $1 AND path = ANY($2)\n AND archived = false AND deleted = false\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "431a5eb2eb5a0b57e096cc78f68eb5a0665c90c4926f1afb689d6cd16bd5b722" +} diff --git a/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json new file mode 100644 index 0000000000..7cfc1f109f --- /dev/null +++ b/backend/.sqlx/query-c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (asset_path)\n asset_path, version, columns AS \"columns: Json>\", captured_at\n FROM materialized_asset_schema\n WHERE workspace_id = $1 AND asset_kind = 'ducklake' AND asset_path = ANY($2)\n ORDER BY asset_path, version DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "columns: Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 3, + "name": "captured_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "c6c186eb3a108e1f695bf5730c23ef4ab1ccfdc4ed33904e83e5130ac0bf5896" +} diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 299e3a70f9..3e6d433405 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -252,6 +252,8 @@ pub struct RetrySpec { // history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a // recognized alias for `history`. `deletes=close` (scd2 only) also closes a key // that disappears from the snapshot; default leaves absent keys current. +// `on_schema_change=ignore` suppresses downstream schema-contract warnings for +// the produced asset (save-time metadata only; default `warn`). #[derive(Serialize, Debug, PartialEq, Clone)] pub struct MaterializeSpec { pub target_kind: AssetKind, @@ -276,6 +278,32 @@ pub struct MaterializeSpec { // `hard_deletes=close`). Default (false) leaves absent keys current. #[serde(skip_serializing_if = "std::ops::Not::not", default)] pub close_deleted: bool, + // `on_schema_change=ignore` opts this producer's asset out of downstream + // schema-contract warnings (gap #2b): consumers referencing columns the + // captured schema no longer has warn by default (`warn`); `ignore` declares + // the schema deliberately unstable and suppresses those warnings. Save-time + // metadata only — the materialize write strategy is unaffected. + #[serde(skip_serializing_if = "OnSchemaChange::is_warn", default)] + pub on_schema_change: OnSchemaChange, +} + +// dbt's `on_schema_change` narrowed to the save-time contract check: `warn` +// (default) surfaces consumer warnings, `ignore` suppresses them. dbt's `fail` +// is deliberately not offered — saves are never hard-blocked (a deliberate +// upstream reshape must not fail every consumer save); a CI/CLI gate can layer +// it on later without touching the grammar. +#[derive(Serialize, Debug, PartialEq, Eq, Clone, Copy, Default)] +#[serde(rename_all = "lowercase")] +pub enum OnSchemaChange { + #[default] + Warn, + Ignore, +} + +impl OnSchemaChange { + pub fn is_warn(&self) -> bool { + matches!(self, OnSchemaChange::Warn) + } } // `// data_test …` — a data-quality assertion run against the @@ -917,6 +945,14 @@ fn parse_materialize_spec(s: &str) -> Option { // `deletes=close` (scd2 only) opts into hard-delete-close; any other value // (or absence) keeps the soft-delete default. let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false); + // `on_schema_change=ignore` suppresses downstream contract warnings; any + // other value (or absence) keeps the `warn` default, fail-safe like + // `deletes=` above. + let on_schema_change = if opts.get("on_schema_change").map(String::as_str) == Some("ignore") { + OnSchemaChange::Ignore + } else { + OnSchemaChange::Warn + }; Some(MaterializeSpec { target_kind, target_path: path.to_string(), @@ -926,6 +962,7 @@ fn parse_materialize_spec(s: &str) -> Option { scd2, track, close_deleted, + on_schema_change, }) } @@ -1623,6 +1660,36 @@ mod pipeline_annotation_tests { assert!(!out.materialize.expect("materialize").close_deleted); } + #[test] + fn materialize_on_schema_change_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/orders on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + // default is warn + let out = parse_pipeline_annotations("// materialize ducklake://a/orders"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // unknown value keeps the warn default (fail-safe, like `deletes=`); + // `fail` is deliberately unrecognized in v1 + let out = + parse_pipeline_annotations("// materialize ducklake://a/orders on_schema_change=fail"); + assert_eq!( + out.materialize.expect("materialize").on_schema_change, + OnSchemaChange::Warn + ); + // composes with other opts + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history on_schema_change=ignore", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.on_schema_change, OnSchemaChange::Ignore); + } + #[test] fn materialize_key_without_history_is_plain_merge() { let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id"); diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index f7977239c1..5ee2945d18 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -310,6 +310,43 @@ } } }, + { + "name": "materialize on_schema_change=ignore opt", + "code": "// pipeline\n// materialize ducklake://analytics/orders on_schema_change=ignore\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "on_schema_change": "ignore" + } + } + }, + { + "name": "materialize on_schema_change unknown value keeps warn default (fail unrecognized in v1)", + "code": "// materialize ducklake://analytics/orders key=id on_schema_change=fail\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/orders", + "unique_key": "id", + "on_schema_change": "warn" + } + } + }, { "name": "materialize key without history is plain merge (SCD1, not scd2)", "code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;", diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 2df4acd2e4..b51b33faf8 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -71,6 +71,13 @@ struct ExpectedMaterialize { track: Vec, #[serde(default)] close_deleted: bool, + // "warn" | "ignore"; absent === "warn" (the default). + #[serde(default = "default_on_schema_change")] + on_schema_change: String, +} + +fn default_on_schema_change() -> String { + "warn".to_string() } #[derive(Deserialize)] @@ -209,6 +216,14 @@ fn pipeline_annotation_fixtures_match() { m.close_deleted, e.close_deleted, "{ctx}: materialize close_deleted" ); + let osc = match m.on_schema_change { + windmill_parser::asset_parser::OnSchemaChange::Warn => "warn", + windmill_parser::asset_parser::OnSchemaChange::Ignore => "ignore", + }; + assert_eq!( + osc, e.on_schema_change, + "{ctx}: materialize on_schema_change" + ); } (got, want) => panic!( "{ctx}: materialize mismatch — got {:?}, want present={}", diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 2eb24cee6c..587307afbb 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -664,6 +664,13 @@ struct GraphRunnableNode { // `merge` / any partitioned write INSERTs into a fixed-schema table. #[serde(skip_serializing_if = "Option::is_none", default)] materialize_strategy: Option, + // `on_schema_change=ignore` on the managed materialize — the producer's + // opt-out from downstream schema-contract warnings. Threaded to the editor + // so its client-side contract mirror suppresses the same warnings the + // server check does. Only serialized when set to `ignore` (default `warn` + // is absent). Lockstep with TS `AssetGraphRunnableNode.materialize_on_schema_change`. + #[serde(skip_serializing_if = "Option::is_none", default)] + materialize_on_schema_change: Option, // Macros this script provides to the workspace registry (deployed // `// macros` library). Drives the library node state + details-pane // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. @@ -1289,8 +1296,12 @@ async fn asset_graph( } }), materialize_strategy: ann.and_then(|a| a.materialize.as_ref()).and_then(|m| { + // Precedence mirrors the runtime strategy derivation: + // scd2 (`history`) > append > merge (`key=`) > replace. if m.manual { None + } else if m.scd2 { + Some("scd2".to_string()) } else if m.append { Some("append".to_string()) } else if m.unique_key.is_some() { @@ -1299,6 +1310,12 @@ async fn asset_graph( Some("replace".to_string()) } }), + materialize_on_schema_change: ann + .and_then(|a| a.materialize.as_ref()) + .filter(|m| { + m.on_schema_change == windmill_common::assets::OnSchemaChange::Ignore + }) + .map(|_| "ignore".to_string()), macros: (usage_kind == AssetUsageKind::Script) .then(|| macros_by_provider.get(&path)) .flatten() diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 49a4091f07..e38d1925f0 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -141,6 +141,8 @@ pub fn workspaced_service() -> Router { // CI test results .route("/ci_test_results/{kind}/{*path}", get(get_ci_test_results)) .route("/ci_test_results_batch", post(get_ci_test_results_batch)) + // Save-time schema-contract check (pipelines gap #2b) + .route("/check_schema_contracts", post(check_schema_contracts)) } #[derive(Serialize, FromRow)] @@ -1305,6 +1307,14 @@ async fn create_script_internal<'c>( ns.path ); } + // `manual` materialize never captures a schema (no wrap codegen), so + // there is no contract for `on_schema_change` to mute downstream. + if m.manual && m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Ignore { + tracing::warn!( + "script {}: `on_schema_change=ignore` on a `manual` materialize is inert — manual mode captures no schema, so consumers have no contract to check", + ns.path + ); + } } // `// macros` — this script is a workspace macro library: its body is // CREATE [OR REPLACE] MACRO statements plus plain setup, registered into @@ -3966,3 +3976,46 @@ async fn get_ci_test_results_batch( Ok(Json(result_map)) } + +#[derive(Deserialize)] +struct CheckSchemaContractsRequest { + language: ScriptLang, + content: String, +} + +#[derive(Serialize)] +struct CheckSchemaContractsResponse { + warnings: Vec, +} + +// Save-time schema-contract check (pipelines gap #2b): validate the given +// script content's asset references (body column reads, `// column` lineage, +// `// data_test relationships`) against the latest captured producer schemas +// and return WARNINGS — never errors, and deploy never blocks on this. The +// frontend calls it right after a successful deploy (post-commit, so a +// self-produced target resolves to the fresh content) and the editor mirrors +// the same diff client-side; this endpoint is the authoritative check. Parsing +// uses the same server path as deploy (`effective_script_assets` + +// `parse_pipeline_annotations`) so the verdict matches what deployed. +async fn check_schema_contracts( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(req): Json, +) -> JsonResult { + let assets = crate::asset_inference::effective_script_assets(&req.language, &req.content, None) + .unwrap_or_default(); + let ann = parse_pipeline_annotations(&req.content); + let mut tx = user_db.begin(&authed).await?; + let warnings = windmill_common::schema_contracts::check_schema_contracts( + &mut tx, + &w_id, + &assets, + &ann.column_lineage, + &ann.data_tests, + ann.materialize.as_ref(), + ) + .await?; + tx.commit().await?; + Ok(Json(CheckSchemaContractsResponse { warnings })) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fbad5fffd4..1f836cfb2b 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9187,6 +9187,50 @@ paths: items: $ref: "#/components/schemas/CiTestResult" + /w/{workspace}/scripts/check_schema_contracts: + post: + summary: check a script's asset references against captured producer schemas + description: | + Save-time schema-contract check for data pipelines: validates the given + script content's asset references (body column reads, `// column` lineage, + `// data_test relationships`) against the latest captured producer schemas + and returns warnings. Warnings never block a save/deploy; an asset whose + producer declares `on_schema_change=ignore` is suppressed to a single + informational entry. + operationId: checkSchemaContracts + tags: + - script + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - language + - content + properties: + language: + $ref: "#/components/schemas/ScriptLang" + content: + type: string + responses: + "200": + description: contract warnings (empty when all references match) + content: + application/json: + schema: + type: object + required: + - warnings + properties: + warnings: + type: array + items: + $ref: "#/components/schemas/ContractWarning" + /w/{workspace}/scripts/raw_temp/store: post: summary: store raw script content temporarily for CLI lock generation @@ -29903,6 +29947,39 @@ components: kind: $ref: "#/components/schemas/AssetKind" required: [path, kind] + ContractWarning: + description: | + One save-time schema-contract warning: a consumer reference that does + not match the referenced asset's latest captured schema. + `schema_version`/`captured_at` identify the capture the check ran + against (as-of the producer's last run, not its latest save). + type: object + required: [kind, asset_path, message] + properties: + kind: + type: string + enum: + - missing_column + - missing_lineage_source + - missing_relationship_column + - relationship_type_mismatch + - suppressed + asset_path: + type: string + column: + type: string + expected_type: + type: string + found_type: + type: string + schema_version: + type: integer + format: int64 + captured_at: + type: string + format: date-time + message: + type: string Volume: type: object required: diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 722ad29747..63309a1400 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -6,7 +6,7 @@ use crate::{error, scripts::ScriptHash}; pub use windmill_parser::asset_parser::{ merge_column_lineage, parse_pipeline_annotations, ColumnLineage, ColumnRef, DataTest, - PartitionKind, PipelineAnnotations, RetrySpec, TriggerSpec, PARTITION_TOKEN, + OnSchemaChange, PartitionKind, PipelineAnnotations, RetrySpec, TriggerSpec, PARTITION_TOKEN, }; pub use windmill_types::assets::*; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index baa3e4b500..acf97734d8 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -65,6 +65,7 @@ pub mod materialization; pub mod min_version; pub mod notify_events; pub mod runtime_assets; +pub mod schema_contracts; pub mod workspace_dependencies; #[cfg(feature = "private")] diff --git a/backend/windmill-common/src/schema_contracts.rs b/backend/windmill-common/src/schema_contracts.rs new file mode 100644 index 0000000000..b605aae33b --- /dev/null +++ b/backend/windmill-common/src/schema_contracts.rs @@ -0,0 +1,614 @@ +//! Save-time schema-contract check (pipelines gap #2b): validate a consumer +//! script's asset references against the latest *captured* producer schema +//! (`materialized_asset_schema`, written post-materialize by #2a) and return +//! WARNINGS — never errors. A deliberate upstream reshape must not fail every +//! consumer save; blocking (`on_schema_change=fail`) is deliberately not +//! offered in v1. +//! +//! Ducklake-only: `// materialize` targets are ducklake-only in v1, so nothing +//! else has a captured schema to check against; an asset with no captured +//! schema produces no warnings (first deploy, datatable, external tables). +//! +//! The comparison itself (`diff_contract`) is pure so it can be unit-tested +//! and mirrored 1:1 by the editor-side TS check +//! (frontend/src/lib/components/assets/AssetGraph/schemaContracts.ts); the +//! async wrapper owns the DB reads (schemas + producer resolution) and runs on +//! the caller's RLS-scoped transaction. + +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::types::Json; +use sqlx::{Postgres, Transaction}; +use windmill_parser::asset_parser::{ + ColumnLineage, DataTest, MaterializeSpec, OnSchemaChange, PARTITION_TOKEN, +}; +use windmill_types::assets::{AssetKind, AssetWithAltAccessType}; + +use crate::error::Result; +use crate::materialization::SchemaColumn; + +/// Columns the materialize engine adds/manages; never part of the captured +/// schema, so consumer reads of them must not warn (`_wm_partition` is +/// filtered out of the DESCRIBE capture on purpose). +const RESERVED_COLUMNS: &[&str] = &["_wm_partition"]; + +#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ContractWarningKind { + /// A column the body reads/writes is absent from the captured schema. + MissingColumn, + /// A `// column … <- .
` source column is absent. + MissingLineageSource, + /// A `// data_test relationships … -> .` ref column is absent. + MissingRelationshipColumn, + /// Relationship join columns have different captured types (may still + /// coerce at run time — phrased as "differs", not "will fail"). + RelationshipTypeMismatch, + /// Warnings for this asset were suppressed by the producer's + /// `on_schema_change=ignore` (one informational entry per asset). + Suppressed, +} + +/// One save-time contract warning. `schema_version`/`captured_at` identify the +/// capture the check ran against, so a stale-capture warning is +/// self-explaining (the schema is as-of the producer's last run, not its +/// latest save). +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct ContractWarning { + pub kind: ContractWarningKind, + /// Normalized ducklake asset path (`/
`). + pub asset_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub column: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub found_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub schema_version: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub captured_at: Option>, + pub message: String, +} + +/// The latest captured schema of one asset, as loaded by the wrapper. +#[derive(Debug, Clone)] +pub struct CapturedSchema { + pub columns: Vec, + pub version: i64, + pub captured_at: DateTime, +} + +impl CapturedSchema { + /// Case-insensitive column lookup — DuckDB matches unquoted identifiers + /// case-insensitively, and the body parser preserves source casing while + /// DESCRIBE returns stored casing. + fn find(&self, name: &str) -> Option<&SchemaColumn> { + self.columns + .iter() + .find(|c| c.name.eq_ignore_ascii_case(name)) + } +} + +fn is_reserved(name: &str) -> bool { + RESERVED_COLUMNS + .iter() + .any(|r| r.eq_ignore_ascii_case(name)) +} + +/// Strip the `{partition}` token a declared URI may carry (`// on +/// ducklake://lake/t/{partition}` or pasted refs) so lookups hit the captured +/// path. Body-inferred paths never carry it, but annotation refs can. +pub fn normalize_asset_path(path: &str) -> String { + path.replace(&format!("/{}", PARTITION_TOKEN), "") + .replace(PARTITION_TOKEN, "") + .trim_end_matches('/') + .to_string() +} + +fn column_list(schema: &CapturedSchema) -> String { + schema + .columns + .iter() + .map(|c| c.name.as_str()) + .collect::>() + .join(", ") +} + +/// Pure comparison: consumer refs vs captured schemas. `schemas` is keyed by +/// normalized ducklake path (the `_current` → base-table fallback is resolved +/// by the wrapper before this runs); `ignored` holds paths whose producer +/// declared `on_schema_change=ignore`. +pub fn diff_contract( + assets: &[AssetWithAltAccessType], + column_lineage: &[ColumnLineage], + data_tests: &[DataTest], + materialize: Option<&MaterializeSpec>, + schemas: &HashMap, + ignored: &HashSet, +) -> Vec { + let mut warnings: Vec = vec![]; + + // W1 — body-read/written columns missing from the captured schema. Assets + // whose column set the parser could not derive (`columns: None`, e.g. + // wildcard SELECT or non-SQL access) are skipped fail-safe; a literal "*" + // key is skipped defensively for the same reason. + for a in assets { + if a.kind != AssetKind::Ducklake { + continue; + } + let Some(columns) = a.columns.as_ref() else { + continue; + }; + let path = normalize_asset_path(&a.path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + for col in columns.keys() { + if col == "*" || is_reserved(col) { + continue; + } + if schema.find(col).is_none() { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingColumn, + asset_path: path.clone(), + column: Some(col.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "column `{col}` of ducklake://{path} is not in its captured schema \ + (v{}, columns: {})", + schema.version, + column_list(schema) + ), + }); + } + } + } + + // W2 — `// column` lineage source refs. Only annotation-declared lineage + // reaches this fn (AST-inferred lineage is redundant with W1 and can + // mis-attribute aliases). + for cl in column_lineage { + for input in &cl.inputs { + if input.from_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let path = normalize_asset_path(&input.from_path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + if is_reserved(&input.from_column) { + continue; + } + if schema.find(&input.from_column).is_none() { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingLineageSource, + asset_path: path.clone(), + column: Some(input.from_column.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// column {}` reads `{}` from ducklake://{path}, which is not in \ + its captured schema (v{})", + cl.column, input.from_column, schema.version + ), + }); + } + } + } + + // W3 — relationships data-test refs: the referenced column must exist; + // when both sides have captured types, flag a difference. Types come from + // DuckDB DESCRIBE on both sides so verbatim spellings are comparable; the + // runtime probe's IN-subquery still coerces, so a difference is "differs", + // never "will fail". + let own_schema = materialize + .filter(|m| m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake) + .and_then(|m| schemas.get(&normalize_asset_path(&m.target_path))); + for dt in data_tests { + let DataTest::Relationships { column, to_kind, to_path, to_column } = dt else { + continue; + }; + if *to_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let path = normalize_asset_path(to_path); + let Some(schema) = schemas.get(&path) else { + continue; + }; + match schema.find(to_column) { + None => { + warnings.push(ContractWarning { + kind: ContractWarningKind::MissingRelationshipColumn, + asset_path: path.clone(), + column: Some(to_column.clone()), + expected_type: None, + found_type: None, + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// data_test relationships {column}` references \ + ducklake://{path}.{to_column}, which is not in its captured schema \ + (v{})", + schema.version + ), + }); + } + Some(ref_col) => { + if let Some(own_col) = own_schema.and_then(|s| s.find(column)) { + if !own_col.data_type.eq_ignore_ascii_case(&ref_col.data_type) { + warnings.push(ContractWarning { + kind: ContractWarningKind::RelationshipTypeMismatch, + asset_path: path.clone(), + column: Some(to_column.clone()), + expected_type: Some(own_col.data_type.clone()), + found_type: Some(ref_col.data_type.clone()), + schema_version: Some(schema.version), + captured_at: Some(schema.captured_at), + message: format!( + "`// data_test relationships {column}` joins `{}` ({}) to \ + ducklake://{path}.{to_column} ({}) — captured types differ", + column, own_col.data_type, ref_col.data_type + ), + }); + } + } + } + } + } + + // W4 — producer opted the asset out (`on_schema_change=ignore`): drop its + // warnings, leaving one informational entry per suppressed asset so the + // response still records that a mismatch exists but was muted upstream. + if !ignored.is_empty() { + let mut suppressed_assets: Vec = vec![]; + warnings.retain(|w| { + if ignored.contains(&w.asset_path) { + if !suppressed_assets.contains(&w.asset_path) { + suppressed_assets.push(w.asset_path.clone()); + } + false + } else { + true + } + }); + for path in suppressed_assets { + warnings.push(ContractWarning { + kind: ContractWarningKind::Suppressed, + asset_path: path.clone(), + column: None, + expected_type: None, + found_type: None, + schema_version: None, + captured_at: None, + message: format!( + "schema mismatches on ducklake://{path} suppressed by its producer's \ + `on_schema_change=ignore`" + ), + }); + } + } + + warnings +} + +/// Load captured schemas + producer modes and run the contract check for one +/// consumer script's parsed refs. +/// +/// Runs on the caller's RLS-scoped transaction (`user_db`), consistent with +/// the `listAssetSchemas` read path: a producer script the caller cannot read +/// simply stays unresolved and keeps the default `warn` behavior. Draft-only +/// producers have no `asset` write edges yet and likewise default to `warn`. +pub async fn check_schema_contracts( + tx: &mut Transaction<'_, Postgres>, + workspace_id: &str, + assets: &[AssetWithAltAccessType], + column_lineage: &[ColumnLineage], + data_tests: &[DataTest], + materialize: Option<&MaterializeSpec>, +) -> Result> { + // Referenced ducklake paths (normalized) across every ref family the diff + // inspects — plus the consumer's own materialize target (for W3 types). + let mut paths: HashSet = HashSet::new(); + for a in assets { + if a.kind == AssetKind::Ducklake && a.columns.is_some() { + paths.insert(normalize_asset_path(&a.path)); + } + } + for cl in column_lineage { + for input in &cl.inputs { + if input.from_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(&input.from_path)); + } + } + } + for dt in data_tests { + if let DataTest::Relationships { to_kind, to_path, .. } = dt { + if *to_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(to_path)); + } + } + } + if let Some(m) = materialize { + if m.target_kind == windmill_parser::asset_parser::AssetKind::Ducklake { + paths.insert(normalize_asset_path(&m.target_path)); + } + } + if paths.is_empty() { + return Ok(vec![]); + } + + // A managed scd2 producer (re)creates a `_current` view with the base + // table's columns; only the base table's schema is captured. Include the + // base path in the lookup so `_current` readers can fall back to it (the + // fallback itself is gated on the producer's spec below). + let mut lookup_paths: HashSet = paths.clone(); + for p in &paths { + if let Some(base) = p.strip_suffix("_current") { + if !base.is_empty() { + lookup_paths.insert(base.to_string()); + } + } + } + let lookup_vec: Vec = lookup_paths.into_iter().collect(); + + let schema_rows = sqlx::query!( + r#"SELECT DISTINCT ON (asset_path) + asset_path, version, columns AS "columns: Json>", captured_at + FROM materialized_asset_schema + WHERE workspace_id = $1 AND asset_kind = 'ducklake' AND asset_path = ANY($2) + ORDER BY asset_path, version DESC"#, + workspace_id, + &lookup_vec, + ) + .fetch_all(&mut **tx) + .await?; + let mut schemas: HashMap = schema_rows + .into_iter() + .map(|r| { + ( + r.asset_path, + CapturedSchema { + columns: r.columns.0, + version: r.version, + captured_at: r.captured_at, + }, + ) + }) + .collect(); + + // Producer resolution: write edges on the referenced assets → latest + // non-archived producer content → parsed `// materialize` spec. Drives + // both the `on_schema_change=ignore` suppression and the `_current` + // fallback. Flow writers are excluded — they cannot carry the annotation. + let paths_vec: Vec = paths.iter().cloned().collect(); + let producer_edges = sqlx::query!( + r#"SELECT DISTINCT path AS "asset_path!", usage_path AS "producer_path!" + FROM asset + WHERE workspace_id = $1 AND kind = 'ducklake' AND path = ANY($2) + AND usage_kind = 'script' AND usage_access_type IN ('w', 'rw')"#, + workspace_id, + &paths_vec, + ) + .fetch_all(&mut **tx) + .await?; + + let mut ignored: HashSet = HashSet::new(); + if !producer_edges.is_empty() { + let producer_paths: Vec = producer_edges + .iter() + .map(|e| e.producer_path.clone()) + .collect::>() + .into_iter() + .collect(); + // Same latest-content pattern as the asset-graph endpoint; NOT + // `get_latest_script_hash`, whose `lock IS NOT NULL` filter transiently + // excludes a just-deployed producer pending its dependency job. + let producer_rows = sqlx::query!( + r#"SELECT DISTINCT ON (path) path, content + FROM script + WHERE workspace_id = $1 AND path = ANY($2) + AND archived = false AND deleted = false + ORDER BY path, created_at DESC"#, + workspace_id, + &producer_paths, + ) + .fetch_all(&mut **tx) + .await?; + let producer_specs: HashMap> = producer_rows + .into_iter() + .map(|r| { + ( + r.path, + windmill_parser::asset_parser::parse_pipeline_annotations(&r.content) + .materialize, + ) + }) + .collect(); + + for edge in &producer_edges { + let Some(Some(spec)) = producer_specs.get(&edge.producer_path) else { + continue; + }; + if spec.target_kind != windmill_parser::asset_parser::AssetKind::Ducklake { + continue; + } + let target = normalize_asset_path(&spec.target_path); + // `on_schema_change=ignore` — any producer declaring it wins. + if spec.on_schema_change == OnSchemaChange::Ignore + && (target == edge.asset_path + || (spec.scd2 && format!("{target}_current") == edge.asset_path)) + { + ignored.insert(edge.asset_path.clone()); + } + // `_current` fallback: a managed scd2 producer's view has exactly + // the base table's columns. + if spec.scd2 + && !spec.manual + && format!("{target}_current") == edge.asset_path + && !schemas.contains_key(&edge.asset_path) + { + if let Some(base) = schemas.get(&target).cloned() { + schemas.insert(edge.asset_path.clone(), base); + } + } + } + } + + Ok(diff_contract( + assets, + column_lineage, + data_tests, + materialize, + &schemas, + &ignored, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeMap; + use windmill_parser::asset_parser::{parse_pipeline_annotations, ColumnRef}; + use windmill_types::assets::AssetUsageAccessType; + + fn schema(cols: &[(&str, &str)]) -> CapturedSchema { + CapturedSchema { + columns: cols + .iter() + .map(|(n, t)| SchemaColumn { name: n.to_string(), data_type: t.to_string() }) + .collect(), + version: 2, + captured_at: DateTime::::MIN_UTC, + } + } + + fn read_asset(path: &str, cols: &[&str]) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind: AssetKind::Ducklake, + access_type: Some(AssetUsageAccessType::R), + alt_access_type: None, + columns: Some( + cols.iter() + .map(|c| (c.to_string(), AssetUsageAccessType::R)) + .collect::>(), + ), + } + } + + #[test] + fn missing_read_column_warns_case_insensitively() { + let schemas = HashMap::from([( + "lake/orders".to_string(), + schema(&[("Order_ID", "BIGINT"), ("amount_usd", "DOUBLE")]), + )]); + let assets = vec![read_asset("lake/orders", &["order_id", "amount"])]; + let w = diff_contract(&assets, &[], &[], None, &schemas, &HashSet::new()); + // order_id matches case-insensitively; amount is gone + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::MissingColumn); + assert_eq!(w[0].column.as_deref(), Some("amount")); + assert_eq!(w[0].schema_version, Some(2)); + } + + #[test] + fn unknown_columns_wildcard_and_reserved_are_skipped() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + // columns: None (wildcard SELECT) — skipped entirely + let mut a = read_asset("lake/orders", &[]); + a.columns = None; + assert!(diff_contract(&[a], &[], &[], None, &schemas, &HashSet::new()).is_empty()); + // literal "*" and the reserved partition column are skipped + let a = read_asset("lake/orders", &["*", "_wm_partition", "id"]); + assert!(diff_contract(&[a], &[], &[], None, &schemas, &HashSet::new()).is_empty()); + } + + #[test] + fn asset_without_captured_schema_is_silent() { + let assets = vec![read_asset("lake/unknown", &["whatever"])]; + assert!( + diff_contract(&assets, &[], &[], None, &HashMap::new(), &HashSet::new()).is_empty() + ); + } + + #[test] + fn partition_token_is_normalized() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let assets = vec![read_asset("lake/orders/{partition}", &["gone"])]; + let w = diff_contract(&assets, &[], &[], None, &schemas, &HashSet::new()); + assert_eq!(w.len(), 1); + assert_eq!(w[0].asset_path, "lake/orders"); + } + + #[test] + fn lineage_ref_missing_column_warns() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let lineage = vec![ColumnLineage { + column: "total".to_string(), + inputs: vec![ColumnRef { + from_kind: windmill_parser::asset_parser::AssetKind::Ducklake, + from_path: "lake/orders".to_string(), + from_column: "amount".to_string(), + }], + }]; + let w = diff_contract(&[], &lineage, &[], None, &schemas, &HashSet::new()); + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::MissingLineageSource); + } + + #[test] + fn relationships_missing_and_type_mismatch() { + let schemas = HashMap::from([ + ("lake/customers".to_string(), schema(&[("id", "VARCHAR")])), + ( + "lake/orders".to_string(), + schema(&[("customer_id", "BIGINT")]), + ), + ]); + let ann = parse_pipeline_annotations( + "// materialize ducklake://lake/orders\n\ + // data_test relationships customer_id -> ducklake://lake/customers.id\n\ + // data_test relationships customer_id -> ducklake://lake/customers.uuid\n\ + SELECT 1;", + ); + let w = diff_contract( + &[], + &[], + &ann.data_tests, + ann.materialize.as_ref(), + &schemas, + &HashSet::new(), + ); + assert_eq!(w.len(), 2); + assert!(w + .iter() + .any(|w| w.kind == ContractWarningKind::RelationshipTypeMismatch + && w.expected_type.as_deref() == Some("BIGINT") + && w.found_type.as_deref() == Some("VARCHAR"))); + assert!(w + .iter() + .any(|w| w.kind == ContractWarningKind::MissingRelationshipColumn + && w.column.as_deref() == Some("uuid"))); + } + + #[test] + fn ignored_asset_suppresses_to_single_note() { + let schemas = HashMap::from([("lake/orders".to_string(), schema(&[("id", "BIGINT")]))]); + let assets = vec![read_asset("lake/orders", &["a", "b"])]; + let ignored = HashSet::from(["lake/orders".to_string()]); + let w = diff_contract(&assets, &[], &[], None, &schemas, &ignored); + assert_eq!(w.len(), 1); + assert_eq!(w[0].kind, ContractWarningKind::Suppressed); + // and nothing at all when there was nothing to suppress + let assets = vec![read_asset("lake/orders", &["id"])]; + assert!(diff_contract(&assets, &[], &[], None, &schemas, &ignored).is_empty()); + } +} diff --git a/cli/src/commands/pipeline/localGraph.ts b/cli/src/commands/pipeline/localGraph.ts index 9d40a0e4b7..98413876a5 100644 --- a/cli/src/commands/pipeline/localGraph.ts +++ b/cli/src/commands/pipeline/localGraph.ts @@ -44,9 +44,14 @@ export type GraphRunnable = { column_lineage?: unknown[]; // `// materialize ` target + strategy — the script's declared output, // so the UI anchors column lineage / the materialize badge to it (the producer - // write-edge is emitted separately). + // write-edge is emitted separately). `scd2` also identifies the producer of a + // `_current` view for the editor's schema-contract fallback. materialize_target?: { kind: string; path: string }; - materialize_strategy?: "replace" | "append" | "merge"; + materialize_strategy?: "replace" | "append" | "merge" | "scd2"; + // `on_schema_change=ignore` — producer's opt-out from downstream + // schema-contract warnings; only present when set (default `warn` is absent), + // mirroring the deployed graph node. + materialize_on_schema_change?: string; }; export type GraphEdge = { runnable_kind: string; @@ -110,6 +115,9 @@ type ParseAssetsRaw = { manual?: boolean; append?: boolean; unique_key?: string; + scd2?: boolean; + // "ignore" when set; the default `warn` is skipped in serialization. + on_schema_change?: string; }; // `// tag ` — the worker tag the deployed pipeline routes to. tag?: string; @@ -428,16 +436,18 @@ export async function buildLocalPipelineGraph(args: { // deployed pipeline would (both `pipeline run --local` and `/pipeline_dev`). pipelineScripts.push(out.tag ? { ...s, tag: out.tag } : s); const mat = out.materialize; - // Managed-materialize write strategy, derived like the deployed graph - // (append → append; key=→ merge; else replace). Manual mode has no - // managed strategy. + // Managed-materialize write strategy, derived like the deployed graph — + // precedence mirrors the runtime: scd2 (`history`) > append > merge + // (key=) > replace. Manual mode has no managed strategy. const materialize_strategy = mat && !mat.manual - ? mat.append - ? "append" - : mat.unique_key - ? "merge" - : "replace" + ? mat.scd2 + ? "scd2" + : mat.append + ? "append" + : mat.unique_key + ? "merge" + : "replace" : undefined; runnables.push({ path: s.path, @@ -453,6 +463,9 @@ export async function buildLocalPipelineGraph(args: { : {}), ...(mat ? { materialize_target: { kind: mat.target_kind, path: mat.target_path } } : {}), ...(materialize_strategy ? { materialize_strategy } : {}), + ...(mat?.on_schema_change === "ignore" + ? { materialize_on_schema_change: "ignore" } + : {}), }); for (const a of out.assets ?? []) { diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 4481d7555b..83c25f3eec 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -49,6 +49,8 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). // materialize ducklake://analytics/orders_daily append → managed, append // materialize ducklake://analytics/dim_customer key=id history → managed, SCD type 2 history // materialize manual ducklake://analytics/orders_daily → track-only escape hatch +// materialize ducklake://analytics/raw_events on_schema_change=ignore +// → managed, downstream contract warnings muted ``` - **managed (default)** — the script is *setup + one trailing `SELECT`*; Windmill @@ -366,6 +368,35 @@ load-bearing. - **Managed only.** `// materialize manual` + `// data_test` is rejected with a clear error (we can't know the manual script's target alias / partition col). +## Schema contracts (save-time, gap #2b) + +The captured schema (#2a) is read back as a *contract*: at save/deploy time, +every consumer's asset references — body-read/written columns, `// column` +lineage sources, `// data_test relationships` refs — are diffed against the +latest `materialized_asset_schema` version of each referenced ducklake asset, +and mismatches surface as **warnings** (deploy never blocks; dbt's +`on_schema_change=fail` is deliberately absent in v1). The diff lives in +`windmill_common::schema_contracts` (endpoint: +`POST /w/{ws}/scripts/check_schema_contracts`, called by the UI right after a +save) and is mirrored 1:1 by the editor (`schemaContracts.ts`): live Monaco +warning squiggles from the WASM buffer parse, plus column-name completion for +annotation refs fed by the same captured schemas. + +Comparison rules worth knowing: column names are case-insensitive (DuckDB +unquoted-identifier semantics); `_wm_partition` is whitelisted (it's excluded +from capture); `{partition}` tokens are stripped before lookup; a +`_current` reference falls back to the scd2 base table's capture; a +relationships join across two captured assets also flags a captured-type +*difference* (the runtime probe coerces, so it's "differs", not "will fail"). +An asset with no capture (never materialized, `manual` mode, any +`datatable://`) produces no warnings. + +The producer opts a deliberately unstable schema out with +`// materialize … on_schema_change=ignore` — consumers then get a single +informational "suppressed" note instead of per-column warnings. On `manual` +materialize the option is inert (nothing is captured) and deploy logs a +warning saying so. + ## Scoping decision: DuckLake vs DataTable **Make DuckLake the materialization/versioning substrate; keep DataTable as the diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index c073c88b8c..4ad8e65fb9 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -51,7 +51,7 @@ Asset-centric, polyglot, annotation-driven, event-aware: | Column lineage | No | **Shipped** (`// column`); docs site still TODO | | Snapshots / SCD2 | Yes (`key=… history`) | Managed strategy | | Selective execution grammar | No | UI/CLI surface | -| Schema contracts | No, but design metadata model | TODO with design work | +| Schema contracts | No, but design metadata model | **Shipped** (capture #2a + save-time check #2b) | | Packages / community / macros | No | **Shipped** (`// macros` workspace macro libraries) | | Semantic layer / metrics | No | Large additive scope | @@ -140,21 +140,43 @@ Today: `requestRunCascadeSignal` in the canvas, `// tag` annotation parsed. Graph + tags + last-run state has all the inputs. UI/CLI surface, not abstraction work. -### 6. Schema contracts +### 6. Schema contracts — **shipped** dbt: `contract: enforced` + `columns: [{name, data_type}]`. Compile-time check that model output matches the declaration. -Today: `// on datatable://users/active` is a string. Rename a column -upstream → downstream breaks at runtime, silently. +**Shipped**, capture-then-validate (no declaration to maintain): the schema a +managed `// materialize` run captures post-DESCRIBE into the versioned +`materialized_asset_schema` sidecar (#2a) *is* the contract, and every +consumer save/deploy is validated against the latest capture. Three surfaces, +same diff (`windmill_common::schema_contracts`, mirrored 1:1 in +`schemaContracts.ts`): -This is the item where the current asset abstraction is thinnest. -To do contracts well: capture output schemas after a run (substrate-specific -DESCRIBE), persist them as asset metadata, validate consumer references at -save time. The asset-as-typed-node model accommodates it — but **where** -schemas live (asset row, sidecar?), **when** they're captured (post-run? -edit-time?), and **how** versioning works are non-trivial design choices. -Worth doing intentionally now while the asset surface is still young. +- **Save-time (authoritative)**: `POST /w/{ws}/scripts/check_schema_contracts` + runs on the deployed content right after a save and returns **warnings — + never errors**. A deliberate upstream reshape must not fail every consumer + save; that's why dbt's `on_schema_change=fail` is deliberately not offered + in v1 (a CI/CLI gate can layer it on later without touching the grammar). +- **Editor flycheck**: the WASM parse that already runs on the open buffer + feeds the same diff, surfacing mismatches as live Monaco warning squiggles + anchored to the offending read / `// column` / `// data_test` line. +- **Autocompletion**: annotation refs (`// column out <- + ducklake://lake/orders.`, `// data_test relationships … ->`) complete + column names from the captured schema — the broken ref never gets typed. + +What's checked (ducklake-only — the only substrate with capture in v1; +datatable refs have no captured schema and stay silent): body-read/written +columns missing from the capture, `// column` lineage sources, and +`// data_test relationships` refs — including a captured-type *difference* on +the join columns when both sides are captured (phrased "differs", since the +runtime probe's IN-subquery still coerces). Column names compare +case-insensitively (DuckDB unquoted-identifier semantics); the managed +`_wm_partition` column is whitelisted; a `_current` scd2 view falls back +to its base table's capture (identical columns by construction). + +The producer-side escape hatch is `// materialize … on_schema_change=ignore`: +it declares the schema deliberately unstable and collapses downstream +warnings for that asset into a single informational note. Default is `warn`. ### 7. Packages / community / macros — **shipped** (macro libraries) diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index ae413d2484..78a75734c9 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -91,6 +91,11 @@ listWorkspaceMacrosCached, macroDefinitionSql } from '$lib/components/assets/workspaceMacros' + import { + fetchLatestSchema, + normalizeAssetPath, + type ContractMarker + } from '$lib/components/assets/AssetGraph/schemaContracts' import * as htmllang from '$lib/svelteMonarch' import { conf, language } from '$lib/vueMonarch' @@ -162,6 +167,11 @@ preparedAssetsSqlQueries?: InferAssetsSqlQueryDetails[] | undefined // To execute preview scripts with the right worker group customTag?: string + // Live schema-contract diagnostics (pipelines gap #2b): owner-scoped + // warning markers computed by the caller (ScriptEditor's contract + // mirror) from the buffer's asset refs vs captured producer schemas. + // Warning severity only — contracts never block; empty clears. + schemaContractMarkers?: ContractMarker[] } let { @@ -194,7 +204,8 @@ enablePreprocessorSnippet = false, rawAppRunnableKey = undefined, preparedAssetsSqlQueries, - customTag + customTag, + schemaContractMarkers = [] }: Props = $props() $effect.pre(() => { @@ -676,6 +687,53 @@ }) } + let schemaContractCompletor: IDisposable | undefined = undefined + + // Column-name completion for pipeline annotation refs (`// column out <- + // ducklake://lake/orders.|`, `// data_test relationships col -> + // ducklake://lake/customers.|`): suggests the referenced asset's *captured* + // columns (with types) so a broken ref never gets typed — the prevention + // side of the schema-contract check. Only fires on annotation comment lines + // with a ducklake URI right before the cursor's `.`; schemas come from the + // short-TTL contract cache, so per-keystroke cost is a map lookup. + function addSchemaContractCompletions() { + schemaContractCompletor?.dispose() + schemaContractCompletor = languages.registerCompletionItemProvider(lang, { + triggerCharacters: ['.'], + provideCompletionItems: async function (model, position) { + // Read the store per request, not at registration — the provider + // outlives a workspace switch. + const workspace = $workspaceStore + if (!workspace) return { suggestions: [] } + const before = model.getLineContent(position.lineNumber).slice(0, position.column - 1) + if (!/^\s*(\/\/|--|#)\s*(column|data_test|on|materialize)\b/.test(before)) { + return { suggestions: [] } + } + const uri = before.match(/ducklake:\/\/([\w/.{}-]+?)\.$/) + if (!uri) return { suggestions: [] } + const schema = await fetchLatestSchema(workspace, normalizeAssetPath(uri[1])) + if (!schema) return { suggestions: [] } + const word = model.getWordUntilPosition(position) + const range = { + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn + } + return { + suggestions: schema.columns.map((c) => ({ + label: c.name, + kind: languages.CompletionItemKind.Field, + detail: `${c.type} · captured schema v${schema.version}`, + insertText: c.name, + range, + sortText: 'a' + c.name + })) + } + } + }) + } + let sqlSchemaCompletor: IDisposable | undefined = undefined async function updateSchema(newSchemaRes: string | undefined) { @@ -1885,6 +1943,7 @@ sqlTypeCompletor && sqlTypeCompletor.dispose() resultCollectionCompletor && resultCollectionCompletor.dispose() workspaceMacroCompletor && workspaceMacroCompletor.dispose() + schemaContractCompletor && schemaContractCompletor.dispose() preprocessorCompletor && preprocessorCompletor.dispose() timeoutModel && clearTimeout(timeoutModel) changeChainStart = undefined @@ -1968,6 +2027,36 @@ : workspaceMacroCompletor?.dispose() }) + // Pipeline annotation grammar is language-agnostic (`//` / `--` / `#` + // comment headers), so contract-ref completions register for every script + // language that can be a pipeline member. The provider line-gates itself, + // so it is inert outside annotation lines. + $effect(() => { + initialized && ['duckdb', 'python3', 'bun', 'deno', 'nativets'].includes(scriptLang ?? '') + ? untrack(() => addSchemaContractCompletions()) + : schemaContractCompletor?.dispose() + }) + + // Schema-contract markers arrive as a prop because the mirror can finish + // computing before Monaco initializes on mount — reacting to `initialized` + // re-applies the pending set once the model exists. The ever-set latch + // keeps unrelated editors from calling setModelMarkers with [] forever. + let contractMarkersEverSet = false + $effect(() => { + const ms = schemaContractMarkers + if (!initialized || (ms.length === 0 && !contractMarkersEverSet)) return + contractMarkersEverSet = true + untrack(() => { + const model = editor?.getModel() + if (!model) return + meditor.setModelMarkers( + model, + 'schema-contracts', + ms.map((m) => ({ ...m, severity: MarkerSeverity.Warning })) + ) + }) + }) + $effect(() => { initialized && canHavePreprocessor(lang) && enablePreprocessorSnippet ? untrack(() => addPreprocessorCompletions(lang)) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index fbfa4f1c1b..7d639b2c75 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -41,6 +41,7 @@ } from '$lib/utils' import Path from './Path.svelte' import { invalidateWorkspacePaths } from './PathNameAutocomplete.svelte' + import { notifyContractWarnings } from './assets/AssetGraph/schemaContracts' import ScriptEditor from './ScriptEditor.svelte' import { Alert, Button, Drawer, SecondsInput, Tab, TabContent, Tabs } from './common' import LanguageIcon from './common/languageIcons/LanguageIcon.svelte' @@ -654,6 +655,11 @@ // cache so it shows up immediately instead of after the 60s TTL. invalidateWorkspacePaths($workspaceStore!) + // Authoritative save-time schema-contract check (pipelines gap #2b): + // warn-only, post-commit so a self-produced target resolves to the + // content just deployed. Fire-and-forget — must never gate the deploy. + notifyContractWarnings($workspaceStore!, script.language, script.content) + if (!initialPath) { await CaptureService.moveCapturesAndConfigs({ workspace: $workspaceStore!, diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index f1944c73cc..50b5c8da95 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -105,6 +105,11 @@ import { canHavePreprocessor } from '$lib/script_helpers' import { assetEq, type AssetWithAltAccessType } from './assets/lib' import type { ColumnLineage } from './assets/AssetGraph/parsePipelineAnnotations' + import { + computeContractMarkers, + type ContractMarker, + type SchemaContractGraphContext + } from './assets/AssetGraph/schemaContracts' import { editor as meditor } from 'monaco-editor' import type { ReviewChangesOpts } from './copilot/chat/monaco-adapter' import GitRepoViewer from './GitRepoViewer.svelte' @@ -202,6 +207,11 @@ // regular /scripts/edit route keeps its current open-by-default UX; // the session preview opts in to save vertical real estate. initialTestPanelCollapsed?: boolean + // Producer-side facts for the live schema-contract diagnostics + // (`on_schema_change=ignore` suppression + scd2 `_current` fallback), + // built by the pipeline page from the resolved graph. Absent outside the + // pipeline editor — the check still runs, just without suppression. + schemaContractContext?: SchemaContractGraphContext } let { @@ -242,7 +252,8 @@ previewLayout = 'right', onTestStateChange, onTestJob, - initialTestPanelCollapsed = false + initialTestPanelCollapsed = false, + schemaContractContext = undefined }: Props = $props() $effect(() => { @@ -614,6 +625,35 @@ } ) + // Live schema-contract diagnostics (pipelines gap #2b): diff the buffer's + // asset refs against the captured producer schemas and surface mismatches + // as Monaco warning squiggles — the as-you-type mirror of the authoritative + // save-time check. The result is a prop on Editor (not an imperative call) + // because this can resolve before Monaco initializes on mount. Sequenced so + // a slow schema fetch can't overwrite the markers of a newer keystroke. + let contractMarkers: ContractMarker[] = $state([]) + let contractCheckSeq = 0 + watch([() => inferAssetsRes.current, () => schemaContractContext], () => { + const res = inferAssetsRes.current + const workspace = $workspaceStore + const seq = ++contractCheckSeq + if (!workspace || !res || res.status === 'error') { + contractMarkers = [] + return + } + const bufferCode = code + computeContractMarkers( + workspace, + bufferCode, + (res.assets ?? []) as AssetWithAltAccessType[], + schemaContractContext + ) + .then((markers) => { + if (seq === contractCheckSeq) contractMarkers = markers + }) + .catch((e) => console.error('schema-contract diagnostics failed', e)) + }) + watch([() => code, () => lang], () => { if (lang !== 'ansible') return inferAnsibleExecutionMode(code).then((v) => { @@ -2602,6 +2642,7 @@ bind:code={editorCode} bind:websocketAlive bind:this={editor} + schemaContractMarkers={contractMarkers} {yContent} awareness={wsProvider?.awareness} on:change={(e) => { diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index 1abe21f76d..c9b921be2e 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -1,5 +1,5 @@ + +{#if !provider || !model} +
Select a model to configure reasoning effort.
+{:else if !capability.supported} +
The selected model does not support reasoning effort.
+{:else} + + {#snippet buttonReplacement()} + + {/snippet} + +{/if} diff --git a/frontend/src/lib/components/aiProviderStorage.ts b/frontend/src/lib/components/aiProviderStorage.ts index ce6af230bb..7f6d705d3a 100644 --- a/frontend/src/lib/components/aiProviderStorage.ts +++ b/frontend/src/lib/components/aiProviderStorage.ts @@ -50,6 +50,7 @@ export function isSameAsStoredConfig(config: ProviderConfig | undefined): boolea storedConfig !== undefined && storedConfig.kind === config?.kind && storedConfig.resource === config?.resource && - storedConfig.model === config?.model + storedConfig.model === config?.model && + storedConfig.reasoning_effort === config?.reasoning_effort ) } diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 4776e64c79..519c1a4021 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -319,7 +319,7 @@ export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' * model that reasons *by default* — omitting the field would silently keep * the default-on behavior. Undefined means omission is the correct off. */ -function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { +export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { switch (reasoningProviderFamily(provider, model)) { case 'googleai': // Gemini 2.5/3 think by default (dynamic budget / level). The backend diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index ab81f1ea14..b0508b2719 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -475,6 +475,9 @@ components: model: type: string description: Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro') + reasoning_effort: + type: string + description: Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default. required: - kind - resource @@ -1028,7 +1031,7 @@ components: - $ref: '#/components/schemas/InputTransform' description: | Boolean. If true, stream the AI response incrementally. - Streaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result + Streaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result memory: $ref: '#/components/schemas/MemoryTransform' output_schema: diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 7f70280e9f..70bec07fe6 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -295,4 +295,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 9578f1b62d..434a52da63 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -2664,7 +2664,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 26f06fb667..a2a5d294e7 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -381,4 +381,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file From dce247c6d2678a2c95bd728027e17ae3965638e2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 15:19:49 +0000 Subject: [PATCH 04/34] feat(pipeline): write-audit-publish for materialization data tests (#9911) * feat(pipeline): write-audit-publish for materialization data tests (EE) Co-Authored-By: Claude Fable 5 * docs: EE worktree E0583 troubleshooting + duckdb feature check row Co-Authored-By: Claude Fable 5 * docs: clarify EE symlink example (absolute target, EE repo layout) Co-Authored-By: Claude Fable 5 * fix(pipeline): move bootstrap DDL inside guarded WAP transaction Co-Authored-By: Claude Fable 5 * refactor(pipeline): move WAP guard SQL builder into EE, OSS keeps placement only Co-Authored-By: Claude Fable 5 * chore: bump ee-repo-ref to EE branch rebased on EE main Co-Authored-By: Claude Fable 5 * style: reword test comment as current invariant per AGENTS.md Co-Authored-By: Claude Fable 5 * chore: bump ee-repo-ref (EE module doc update) Co-Authored-By: Claude Fable 5 * refactor(pipeline): OSS emits typed materialize plan, EE owns WAP transform Co-Authored-By: Claude Fable 5 * test: make rewrite assertion build-aware; refresh oss module doc Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c This commit updates the EE repository reference after PR #644 was merged in windmill-ee-private. Previous ee-repo-ref: 63cabae75329429f647e01083936d70f8197dc9e New ee-repo-ref: 7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../windmill-parser/src/sql_materialize.rs | 213 +++++++++++++++--- .../src/pipeline_advanced_oss.rs | 27 ++- .../windmill-worker/src/duckdb_executor.rs | 43 +++- docs/ducklake-materialization.md | 20 +- docs/enterprise.md | 16 ++ docs/pipelines-vs-dbt.md | 6 + docs/validation.md | 1 + 8 files changed, 289 insertions(+), 39 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0cd164d1ce..e31f18cdd0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -6f5fe0f7f56696fbef5a8349da38496c32e71666 +7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index e40cd0f057..d684cf431b 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -440,7 +440,6 @@ impl<'a> MaterializeCodegen<'a> { "CREATE TABLE IF NOT EXISTS {t} AS SELECT * FROM ({sel}) WHERE false;" )); } - out.push("BEGIN TRANSACTION;".to_string()); // The rows to write, with the partition column appended when partitioned. let source = if self.partitioned { @@ -578,7 +577,7 @@ impl<'a> MaterializeCodegen<'a> { ]; // Hard-delete-close (`deletes=close`): the keys that vanished from the // snapshot — present-and-current in the table, absent from the SELECT. - // Captured before the transaction (like `changed`) and disjoint from it (a + // Captured before the close (like `changed`) and disjoint from it (a // key is either in the snapshot or not), so the two closes never overlap. if close_deleted { out.push(format!( @@ -646,6 +645,44 @@ pub fn snapshot_capture_sql(alias: &str) -> String { /// from the target ducklake's config and passes it in as `target_attach`. pub const TARGET_ALIAS: &str = "_wm_target"; +/// Structural role of one statement in a [`MaterializePlan`]. The public build +/// executes the plan verbatim, so the kinds are pure metadata there; they exist +/// so a downstream assembler (`pipeline_advanced::finalize_materialize_query`) +/// can reason about the plan without parsing SQL. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MaterializeStmtKind { + /// Pre-write statement: user setup, the target ATTACH, referenced-asset + /// ATTACHes. + Setup, + /// Write work against the target: bootstrap DDL, SCD2 temp-table captures, + /// the mutation itself, the `_current` view. + Write, + /// The `BEGIN TRANSACTION;` marker emitted by the strategy codegen. + TxnBegin, + /// The `COMMIT;` marker emitted by the strategy codegen. + TxnCommit, + /// The trailing one-row summary read (asset / rows / snapshot_id / + /// data_tests breakdown). + Summary, +} + +/// One planned statement: its structural role and the SQL text. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializeStmt { + pub kind: MaterializeStmtKind, + pub sql: String, +} + +/// The full ordered materialization plan [`build_wrap_blocks`] produces: +/// statements in execution order plus the compiled data-test checks (also +/// embedded in the summary statement's breakdown). Assembled into the final +/// statement list by `pipeline_advanced::finalize_materialize_query`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializePlan { + pub stmts: Vec, + pub checks: Vec, +} + /// Assemble the full ordered statement list the DuckDB executor runs for a /// managed `// materialize` script. This is the single entry point the worker /// calls; it composes the already-tested pieces (classifier split → target @@ -659,6 +696,14 @@ pub const TARGET_ALIAS: &str = "_wm_target"; /// the full `/
` for the result summary. The trailing statement is /// a one-row summary read (asset / rows / snapshot_id) that is both the job's /// result (a useful preview) and what the worker records. +/// +/// Returns a [`MaterializePlan`] — the statements plus their structural role +/// and the compiled data-test checks — rather than raw SQL: the executor hands +/// the plan to `windmill_common::pipeline_advanced::finalize_materialize_query` +/// (which this crate cannot depend on), whose public-build implementation +/// assembles the statements verbatim. Everything this function produces runs +/// as-is on the public build; the plan's structure is metadata about it, not a +/// second mode. pub fn build_wrap_blocks( plan: &WrapPlan, target_attach: &str, @@ -669,17 +714,9 @@ pub fn build_wrap_blocks( partitioned: bool, strategy: MaterializeStrategy, tests: &[DataTestResolved], -) -> Result, String> { +) -> Result { let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); let scd2 = matches!(strategy, MaterializeStrategy::Scd2 { .. }); - let cg = MaterializeCodegen { - target_qualified: &target_qualified, - select_sql: &plan.output, - partition_col, - partition_value_sql, - partitioned, - strategy, - }; let ctx = DataTestCtx { target_qualified: &target_qualified, asset_path, @@ -689,27 +726,49 @@ pub fn build_wrap_blocks( scd2, }; let test_sql = build_data_test_checks(tests, &ctx)?; - let mut blocks: Vec = Vec::new(); + let cg = MaterializeCodegen { + target_qualified: &target_qualified, + select_sql: &plan.output, + partition_col, + partition_value_sql, + partitioned, + strategy, + }; + let mut stmts: Vec = Vec::new(); + let setup = |sql: String| MaterializeStmt { kind: MaterializeStmtKind::Setup, sql }; // Setup blocks come from the splitter with their `;` stripped — re-terminate // each so that when the executor re-joins and re-splits the assembled query, // adjacent statements (e.g. the user ATTACH and the synthetic target ATTACH) // don't merge into one malformed statement. - blocks.extend(plan.setup.iter().map(|s| terminate(s))); - blocks.push(target_attach.to_string()); + stmts.extend(plan.setup.iter().map(|s| setup(terminate(s)))); + stmts.push(setup(target_attach.to_string())); // Referenced-asset ATTACHes (relationships tests) — read-only, before the // write and the summary that probes them. - blocks.extend(test_sql.attaches); - blocks.extend(cg.statements()); + stmts.extend(test_sql.attaches.into_iter().map(setup)); + // Classify the codegen statements by matching the exact transaction-marker + // literals this module emits (`BEGIN TRANSACTION;` / `COMMIT;`); everything + // else the codegen produces is write work. + stmts.extend(cg.statements().into_iter().map(|sql| { + let kind = match sql.as_str() { + "BEGIN TRANSACTION;" => MaterializeStmtKind::TxnBegin, + "COMMIT;" => MaterializeStmtKind::TxnCommit, + _ => MaterializeStmtKind::Write, + }; + MaterializeStmt { kind, sql } + })); // The summary read carries the per-test breakdown (when any tests apply). - blocks.push(materialize_result_sql( - &target_qualified, - asset_path, - partition_col, - partition_value_sql, - partitioned, - &test_sql.checks, - )); - Ok(blocks) + stmts.push(MaterializeStmt { + kind: MaterializeStmtKind::Summary, + sql: materialize_result_sql( + &target_qualified, + asset_path, + partition_col, + partition_value_sql, + partitioned, + &test_sql.checks, + ), + }); + Ok(MaterializePlan { stmts, checks: test_sql.checks }) } /// The trailing one-row summary the materialize run returns: the asset it @@ -1454,7 +1513,7 @@ mod tests { #[test] fn build_wrap_blocks_orders_setup_attach_codegen_snapshot() { let plan = ok("ATTACH 'ducklake://main' AS dl;\n SELECT a FROM dl.orders WHERE d = '{p}'"); - let blocks = build_wrap_blocks( + let blocks: Vec = build_wrap_blocks( &plan, "ATTACH 'ducklake:postgres:…' AS _wm_target (DATA_PATH 's3://b/p');", "orders_daily", @@ -1465,7 +1524,11 @@ mod tests { MaterializeStrategy::Replace, &[], ) - .unwrap(); + .unwrap() + .stmts + .into_iter() + .map(|s| s.sql) + .collect(); // setup block first, then the target ATTACH, then codegen, then result. assert!(blocks[0].starts_with("ATTACH 'ducklake://main' AS dl")); // every setup block must be `;`-terminated so re-splitting can't merge it @@ -1488,6 +1551,102 @@ mod tests { assert!(last.contains("ducklake_snapshots('_wm_target')")); } + // -- materialize plan structure ------------------------------------------ + + fn plan_for(strategy: MaterializeStrategy, partitioned: bool) -> MaterializePlan { + let plan = ok("SELECT a, b FROM src"); + build_wrap_blocks( + &plan, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "'2026-06-19'", + partitioned, + strategy, + &[ + DataTestResolved::BuiltIn(DataTest::NotNull { column: "a".into() }), + DataTestResolved::BuiltIn(DataTest::Unique { column: "b".into() }), + ], + ) + .unwrap() + } + + fn kidx(plan: &MaterializePlan, pred: impl Fn(&MaterializeStmt) -> bool) -> usize { + plan.stmts + .iter() + .position(|s| pred(s)) + .expect("stmt present") + } + + #[test] + fn plan_tags_structure_and_carries_checks() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, true); + // leading statements are Setup, ending with the target ATTACH + assert!(plan.stmts[0].kind == Setup); + assert!(plan + .stmts + .iter() + .take_while(|s| s.kind == Setup) + .any(|s| s.sql.contains("_wm_target"))); + // txn markers are tagged, everything between them is Write + let begin = kidx(&plan, |s| s.kind == TxnBegin); + let commit = kidx(&plan, |s| s.kind == TxnCommit); + assert!(begin < commit); + assert!(plan.stmts[begin + 1..commit] + .iter() + .all(|s| s.kind == Write)); + // bootstrap DDL is Write work (it targets the table, not the session) + let bootstrap = kidx(&plan, |s| s.sql.starts_with("CREATE TABLE IF NOT EXISTS")); + assert_eq!(plan.stmts[bootstrap].kind, Write); + // summary is last and carries the breakdown; checks ride along + let last = plan.stmts.last().unwrap(); + assert_eq!(last.kind, Summary); + assert!(last.sql.contains("AS data_tests")); + assert_eq!(plan.checks.len(), 2); + assert!(plan.checks[0].name.contains("not_null(a)")); + } + + #[test] + fn plan_whole_table_replace_has_no_txn_markers() { + use MaterializeStmtKind::*; + let plan = plan_for(MaterializeStrategy::Replace, false); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnBegin)); + assert!(!plan.stmts.iter().any(|s| s.kind == TxnCommit)); + assert_eq!( + plan.stmts.iter().filter(|s| s.kind == Write).count(), + 1, + "single atomic CREATE OR REPLACE" + ); + } + + #[test] + fn plan_scd2_captures_are_write_kind() { + use MaterializeStmtKind::*; + let plan = plan_for( + MaterializeStrategy::Scd2 { key: "a".into(), track: vec![], close_deleted: false }, + false, + ); + let capture = kidx(&plan, |s| s.sql.contains("TEMP TABLE _wm_scd2_changed")); + assert_eq!(plan.stmts[capture].kind, Write); + // no test declared ⇒ empty checks + let plain = ok("SELECT a FROM src"); + let no_tests = build_wrap_blocks( + &plain, + "ATTACH 'ducklake:…' AS _wm_target;", + "orders", + "main/orders", + "_wm_partition", + "''", + false, + MaterializeStrategy::Append, + &[], + ) + .unwrap(); + assert!(no_tests.checks.is_empty()); + } + // -- data tests --------------------------------------------------------- fn ctx_partitioned() -> DataTestCtx<'static> { diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs index 15bb4b127a..3220b251a0 100644 --- a/backend/windmill-common/src/pipeline_advanced_oss.rs +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -1,12 +1,27 @@ -//! OSS fallback: pipeline partition backfills are an enterprise feature; -//! their implementations live in windmill-ee-private (see -//! `pipeline_advanced_ee`). In the public build the entry points report that -//! the enterprise edition is required. (Freshness lives elsewhere: the -//! fresh/stale badge is CE in the assets API, the active watchdog is -//! windmill-queue's `freshness_watchdog`.) +//! OSS fallback for enterprise pipeline features (implementations in +//! windmill-ee-private, see `pipeline_advanced_ee`): partition backfill +//! reports that the enterprise edition is required, and materialization-plan +//! assembly runs the plan verbatim — dbt-like commit-then-test instead of the +//! enterprise write-audit-publish. (Freshness lives elsewhere: the fresh/stale +//! badge is CE in the assets API, the active watchdog is windmill-queue's +//! `freshness_watchdog`.) use crate::error::Error; pub fn backfill_todo() -> Error { Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) } + +/// Assemble a materialization plan into the statement list the DuckDB executor +/// runs. The public build executes the plan verbatim — dbt-like +/// commit-then-test: a failing `// data_test` still fails the run and stops +/// the cascade, but the written slice stays live. The enterprise +/// implementation (`pipeline_advanced_ee`) instead restructures the plan into +/// write-audit-publish, where a failing test rolls the whole write back before +/// anything is published. +pub fn finalize_materialize_query( + plan: windmill_parser::sql_materialize::MaterializePlan, + _asset_path: &str, +) -> Vec { + plan.stmts.into_iter().map(|s| s.sql).collect() +} diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 09e3158200..a90c80acfc 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -242,7 +242,7 @@ fn build_materialized_query( } } - let blocks = build_wrap_blocks( + let mat_plan = build_wrap_blocks( &plan, &synthetic_attach, table, @@ -254,6 +254,11 @@ fn build_materialized_query( &resolved, ) .map_err(Error::ExecutionErr)?; + // Enterprise seam: assembles the plan into the final statement list — + // verbatim on the public build (commit-then-test), restructured into + // write-audit-publish (guarded transaction, rollback on violation) on EE. + let blocks = + windmill_common::pipeline_advanced::finalize_materialize_query(mat_plan, &m.target_path); Ok(Some((Some(blocks.join("\n")), meta))) } @@ -1240,6 +1245,11 @@ pub async fn do_duckdb( // committed (like dbt), so the slice is recorded `Failed` and the // cascade stops. The error lists *every* test so the user sees the // whole picture, not just the first failure. + // Under enterprise write-audit-publish an in-transaction guard + // (the enterprise `finalize_materialize_query` restructure) already aborted a failing run before + // COMMIT — that surfaces on the Err path above with the same + // breakdown in the error string, and nothing was published; this + // post-commit path then only ever sees passing counts. let tests = extract_data_tests(&result); // Captured output schema (gap #2a) — recorded only on the successful // path below, not on the failure paths (a failed run shouldn't @@ -2498,6 +2508,37 @@ mod tests { ); } + // The rewritten SQL is the plan assembled by + // `pipeline_advanced::finalize_materialize_query`, so what it contains is + // build-dependent: the public assembly runs the plan verbatim (tests only + // in the post-commit summary breakdown), the enterprise assembly adds the + // in-transaction write-audit-publish guard (whose shape/placement is + // tested next to its implementation in windmill-common's + // `pipeline_advanced_ee`). + #[test] + fn materialize_rewrite_carries_data_test_summary() { + let script = "-- materialize ducklake://main/orders\n\ + -- data_test not_null id\n\ + SELECT id FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!(rewritten.contains("AS data_tests")); + assert_eq!(meta.n_data_tests, 1); + #[cfg(not(feature = "private"))] + assert!( + !rewritten.contains("error("), + "public assembly is commit-then-test (no guard)" + ); + #[cfg(all(feature = "private", feature = "enterprise"))] + assert!( + rewritten.contains("error("), + "enterprise assembly places the WAP guard" + ); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 83c25f3eec..f0a8c38f45 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -354,10 +354,22 @@ load-bearing. repeats the natural key across closed versions, so an unscoped `unique()` would fail the run on the second change of any key. Custom tests see the raw history and scope themselves. -- **Commit-then-test.** Like dbt, the write commits before tests run; a failed - test fails the *run* (and records `Failed`, so downstream cascade stops) but - does not roll back the committed snapshot. Time-travel still lets you inspect - exactly what failed. +- **Commit-then-test (public) / write-audit-publish (enterprise).** In the + public build, like dbt, the write commits before tests run; a failed test + fails the *run* (and records `Failed`, so downstream cascade stops) but does + not roll back the committed snapshot — time-travel still lets you inspect + exactly what failed. Enterprise upgrades this to write-audit-publish: the + same checks also run *inside* the write transaction as a guard statement + that raises on any violation, aborting the run before `COMMIT` — a failing + slice is never published, readers keep the previous version, and no snapshot + is created (even a *first* run of a new asset rolls back to "no table"). + The whole mechanism lives in the enterprise repo: + `sql_materialize::build_wrap_blocks` produces a typed statement plan + (`MaterializePlan` — statements plus structural kinds and the compiled + checks), and `pipeline_advanced::finalize_materialize_query` assembles it — + verbatim on the public build, restructured into the guarded transaction on + EE. The public build thus carries only plan metadata, not the + write-audit-publish transform itself. - **Custom = DuckDB SQL, server worker.** The escape hatch fetches the deployed script's content (a single DuckDB `SELECT`/CTE returning the violating rows — it's embedded as a subquery, so a multi-statement body is rejected with a diff --git a/docs/enterprise.md b/docs/enterprise.md index 5ab869a651..229585ad25 100644 --- a/docs/enterprise.md +++ b/docs/enterprise.md @@ -47,3 +47,19 @@ cargo check --features enterprise,private # EE code that also requires license validation cargo check --features enterprise,private,license ``` + +## Troubleshooting + +**`E0583: file not found for module _ee` on `cargo check --features enterprise,private`**: +the EE worktree is behind the OSS code it must satisfy. Fast-forward it to EE +`origin/main` — EE worktrees are shallow clones, so run `git fetch --unshallow origin` +first or the merge fails with "refusing to merge unrelated histories". After the +fast-forward, any `*_ee.rs` file that is *new* since the worktree was created still +needs its OSS symlink made by hand (worktree setup only links files that existed then). +The EE repo has no `backend/` prefix — crates sit at its root. Use an absolute target +(a relative one would resolve against the link's directory, not your cwd): + +```bash +# from the windmill repo root +ln -s ~/windmill-ee-private//src/_ee.rs backend//src/_ee.rs +``` diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 4ad8e65fb9..126a4ac215 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -75,6 +75,12 @@ the annotation→verifier pattern that column-lineage will reuse. The keyword is `data_test`, not `test`, to stay clear of the unrelated `// test:` CI-test annotation. +Enterprise goes one step *beyond* dbt here: write-audit-publish. dbt commits +the model then tests it, so a failing test leaves the bad table live for BI +readers; the enterprise build runs the same probes inside the write +transaction and rolls back on violation, so a failing slice is never +published (see `ducklake-materialization.md` §"Scoping decisions"). + ### 2. Incremental materializations See [Incremental deep-dive](#incremental-deep-dive) below. diff --git a/docs/validation.md b/docs/validation.md index ba775d01cf..3bfd1325ce 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -10,6 +10,7 @@ After making changes, run the appropriate checks and fix all errors before consi | Enterprise code (`*_ee.rs`) | `cargo check --features enterprise,private` | Also do EE PR workflow (see `docs/enterprise.md`) | | Enterprise + license-gated code | `cargo check --features enterprise,private,license` | When the feature requires a valid license key | | Kafka trigger code | `cargo check --features kafka` | | +| DuckDB executor code | `cargo check -p windmill-worker --features duckdb` | `duckdb_executor.rs` (and its `#[cfg(test)]` tests) only compile with this flag — a plain check/test silently skips them | | Native trigger code | `cargo check --features native_trigger` | | | Parquet code | `cargo check --features parquet` | | | Multiple gated modules | `cargo check --features enterprise,parquet` | Combine only the flags you need | From 33521505dbc34f22b575d21fda1cc76d698a8840 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 4 Jul 2026 15:23:23 +0000 Subject: [PATCH 05/34] feat(ducklake): scheduled lake maintenance (expiry, compaction, orphan cleanup) (#9916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ducklake): scheduled lake maintenance (snapshot expiry, compaction, orphan cleanup) Co-Authored-By: Claude Fable 5 * fix(ducklake): review fixes — starts_with not LIKE, CE license-lapse escape Co-Authored-By: Claude Fable 5 * docs(ducklake): auth-contract docs + _unchecked rename per codex review Co-Authored-By: Claude Fable 5 * refactor(ducklake): move maintenance payload construction into EE module Co-Authored-By: Claude Fable 5 * fix(ducklake): fall through to script resolution for non-managed reserved-prefix schedules Co-Authored-By: Claude Fable 5 * docs(ducklake): document accepted pre-existing-schedule limitation on the reserved prefix Co-Authored-By: Claude Fable 5 * fix(ducklake): CE save-off clears the managed schedule row and queued occurrence Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 2fab310d4f50ed7c34857d69c9b854f4491bf217 This commit updates the EE repository reference after PR #645 was merged in windmill-ee-private. Previous ee-repo-ref: fff1fd830a36beba732486f05941ec243cf6b640 New ee-repo-ref: 2fab310d4f50ed7c34857d69c9b854f4491bf217 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] --- ...d76a4e5702c0324a27ef3a5740ef5d98d8946.json | 31 +++ ...838563f28e832e37e5c165329441d8471acdb.json | 16 ++ ...da5d913caa8d935b33ad8f21336dedefd1c23.json | 15 ++ ...efef25885a9bca64bb2ba4bb173bc19dcbdba.json | 23 +++ ...08c547742851975163550ec4f3823057e91fe.json | 20 ++ ...4e6219434de4d2f9ae5e484ed8ce5da426109.json | 16 ++ ...c8453d53c973dce61cc8ace7d278773c7506f.json | 31 +++ ...6afc85a8350706fe694935b25ef46a847409e.json | 22 +++ ...6e1a9307335b0e7a27613cd690e99812a99f3.json | 23 +++ ...548e2783c2de7e92ba0ff7e6f4637419fcf7d.json | 16 ++ ...a14576509147e51a86a95a2cc786bee3199d9.json | 16 ++ ...6f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json | 16 ++ backend/ee-repo-ref.txt | 2 +- backend/windmill-api-schedule/src/lib.rs | 39 +++- backend/windmill-api-workspaces/Cargo.toml | 6 +- .../windmill-api-workspaces/src/workspaces.rs | 59 ++++-- backend/windmill-api/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 21 ++ backend/windmill-api/src/workspaces_export.rs | 7 +- backend/windmill-common/src/workspaces.rs | 74 +++++++ .../src/ducklake_maintenance_oss.rs | 96 +++++++++ backend/windmill-queue/src/lib.rs | 7 + backend/windmill-queue/src/schedule.rs | 23 ++- backend/windmill-queue/tests/schedule_push.rs | 168 ++++++++++++++++ .../workspaceSettings/DucklakeSettings.svelte | 187 +++++++++++++++++- 25 files changed, 907 insertions(+), 29 deletions(-) create mode 100644 backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json create mode 100644 backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json create mode 100644 backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json create mode 100644 backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json create mode 100644 backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json create mode 100644 backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json create mode 100644 backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json create mode 100644 backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json create mode 100644 backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json create mode 100644 backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json create mode 100644 backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json create mode 100644 backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json create mode 100644 backend/windmill-queue/src/ducklake_maintenance_oss.rs diff --git a/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json new file mode 100644 index 0000000000..0dbcf04c6e --- /dev/null +++ b/backend/.sqlx/query-0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4)\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "0f141ca6a58901dee1ddcf25694d76a4e5702c0324a27ef3a5740ef5d98d8946" +} diff --git a/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json new file mode 100644 index 0000000000..c19df789d4 --- /dev/null +++ b/backend/.sqlx/query-142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND path NOT LIKE $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "142ae6939440654da21ebb06acf838563f28e832e37e5c165329441d8471acdb" +} diff --git a/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json new file mode 100644 index 0000000000..a1e3edbbe2 --- /dev/null +++ b/backend/.sqlx/query-28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "28ec31481bdaae8b0512d21d074da5d913caa8d935b33ad8f21336dedefd1c23" +} diff --git a/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json new file mode 100644 index 0000000000..dfbc2b75e6 --- /dev/null +++ b/backend/.sqlx/query-333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND path LIKE $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "333a703a53cc760842b047ad420efef25885a9bca64bb2ba4bb173bc19dcbdba" +} diff --git a/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json new file mode 100644 index 0000000000..a8facb9d1b --- /dev/null +++ b/backend/.sqlx/query-5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, schedule, timezone, edited_by, script_path,\n is_flow, enabled, email, permissioned_as, summary, tag, cron_version\n ) VALUES ($1, $2, $3, 'Etc/UTC', $4, $2, false, true, $5, $6, $7, 'duckdb', 'v2')\n ON CONFLICT (workspace_id, path) DO UPDATE SET\n schedule = EXCLUDED.schedule,\n timezone = EXCLUDED.timezone,\n edited_by = EXCLUDED.edited_by,\n edited_at = now(),\n script_path = EXCLUDED.script_path,\n is_flow = false,\n enabled = true,\n email = EXCLUDED.email,\n permissioned_as = EXCLUDED.permissioned_as,\n summary = EXCLUDED.summary,\n tag = EXCLUDED.tag,\n cron_version = EXCLUDED.cron_version,\n error = NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "5323858a017814206179d35fdb708c547742851975163550ec4f3823057e91fe" +} diff --git a/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json new file mode 100644 index 0000000000..8bce508e75 --- /dev/null +++ b/backend/.sqlx/query-53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "53a1ec83aedcec2cdf6495637464e6219434de4d2f9ae5e484ed8ce5da426109" +} diff --git a/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json new file mode 100644 index 0000000000..5741a1aa5b --- /dev/null +++ b/backend/.sqlx/query-8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schedule.path, t.jobs FROM schedule,\n LATERAL(SELECT ARRAY(\n SELECT json_build_object('id', id, 'success', status = 'success', 'duration_ms', duration_ms)\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE trigger_kind = 'schedule'\n AND trigger = schedule.path\n AND c.workspace_id = $1\n AND j.workspace_id = $1\n AND parent_job IS NULL AND runnable_path = schedule.script_path\n AND status <> 'skipped'\n ORDER BY completed_at DESC\n LIMIT 20\n ) AS jobs) t\n WHERE workspace_id = $1 AND schedule.path NOT LIKE $4\n ORDER BY edited_at DESC\n LIMIT $2 OFFSET $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "jobs", + "type_info": "JsonArray" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int8", + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "8756a8100d6fd6ecb64ef73d6f2c8453d53c973dce61cc8ace7d278773c7506f" +} diff --git a/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json new file mode 100644 index 0000000000..829083e0ec --- /dev/null +++ b/backend/.sqlx/query-8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT ws.ducklake->'ducklakes' AS ducklake_name\n FROM workspace_settings ws\n WHERE ws.workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8837a126698d924eb9014cfee0e6afc85a8350706fe694935b25ef46a847409e" +} diff --git a/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json new file mode 100644 index 0000000000..9fc8cf1336 --- /dev/null +++ b/backend/.sqlx/query-8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "8a1e2e59c05b0f67e32c517a0c46e1a9307335b0e7a27613cd690e99812a99f3" +} diff --git a/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json new file mode 100644 index 0000000000..e3e8e3f2ad --- /dev/null +++ b/backend/.sqlx/query-a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND path LIKE $2 AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a0faa5b042bac00bcd09b925bcf548e2783c2de7e92ba0ff7e6f4637419fcf7d" +} diff --git a/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json new file mode 100644 index 0000000000..21e0f4613b --- /dev/null +++ b/backend/.sqlx/query-a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM schedule WHERE workspace_id = $1 AND starts_with(path, $2) AND path != ALL($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a7f0ed0081e98a9861ea4b081b8a14576509147e51a86a95a2cc786bee3199d9" +} diff --git a/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json new file mode 100644 index 0000000000..d316c16fff --- /dev/null +++ b/backend/.sqlx/query-c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO schedule (\n workspace_id, path, edited_by, edited_at, schedule, enabled, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n )\n SELECT\n $1, path, edited_by, edited_at, schedule, FALSE, script_path,\n args, extra_perms, is_flow, email, error, timezone, on_failure,\n on_recovery, on_failure_times, on_failure_exact, on_failure_extra_args,\n on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry,\n summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args,\n cron_version, description, dynamic_skip, permissioned_as, labels\n FROM schedule WHERE workspace_id = $2 AND NOT starts_with(path, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c832ebc7ff0be446bf36628f6e06f59d0fbe8df62e0bd0be636d7cf70dcf95f0" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e31f18cdd0..2c16f06c50 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -7be0bad1a6d6b5c3a107c0a2cd4bf003c36ec34c +2fab310d4f50ed7c34857d69c9b854f4491bf217 diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 7cc0111c28..573c47e355 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -182,6 +182,19 @@ fn to_json_raw_opt( value.map(|v| sqlx::types::Json(to_raw_value(&v))) } +/// Managed ducklake-maintenance schedules live under a reserved path prefix; +/// their lifecycle is owned by the workspace ducklake settings, so the +/// schedule API refuses to create/edit/delete/toggle them. +fn reject_reserved_schedule_path(path: &str) -> Result<()> { + if path.starts_with(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) { + return Err(Error::BadRequest(format!( + "Schedules under {} are managed by the workspace ducklake settings", + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX + ))); + } + Ok(()) +} + /// Validate that a dynamic skip handler (script or flow) exists async fn validate_dynamic_skip<'c>( tx: &mut Transaction<'c, Postgres>, @@ -219,6 +232,7 @@ async fn create_schedule( Json(ns): Json, ) -> Result { check_scopes(&authed, || format!("schedules:write:{}", ns.path))?; + reject_reserved_schedule_path(&ns.path)?; let authed = maybe_refresh_folders(&ns.path, &w_id, authed, &db).await; @@ -468,6 +482,7 @@ async fn edit_schedule( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; let authed = maybe_refresh_folders(&path, &w_id, authed, &db).await; let mut tx = user_db.begin(&authed).await?; @@ -750,6 +765,15 @@ async fn list_schedule( ) .order_by("edited_at", true) .and_where("workspace_id = ?".bind(&w_id)) + // managed ducklake-maintenance schedules are edited from the + // workspace ducklake settings, not the schedules UI/CLI. + // starts_with, not LIKE: the prefix contains `_` which LIKE treats as + // a wildcard, and a user folder like `ducklake-maintenance` must not + // be swept up. + .and_where( + "NOT starts_with(path, ?)" + .bind(&windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX), + ) .offset(offset) .limit(per_page) .clone(); @@ -923,12 +947,13 @@ async fn list_schedule_with_jobs( ORDER BY completed_at DESC LIMIT 20 ) AS jobs) t - WHERE workspace_id = $1 + WHERE workspace_id = $1 AND NOT starts_with(schedule.path, $4) ORDER BY edited_at DESC LIMIT $2 OFFSET $3", w_id, per_page as i64, - offset as i64 + offset as i64, + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX ) .fetch_all(&mut *tx) .await?; @@ -1014,6 +1039,7 @@ pub async fn set_enabled( let mut tx = user_db.begin(&authed).await?; let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; // Block enabling a schedule in a fork when the parent has the same path // (regardless of parent's enabled flag), unless force=true. Two enabled @@ -1190,6 +1216,7 @@ async fn delete_schedule( ) -> Result { let path = path.to_path(); check_scopes(&authed, || format!("schedules:write:{}", path))?; + reject_reserved_schedule_path(path)?; let mut tx = user_db.begin(&authed).await?; clear_schedule(&mut tx, path, &w_id).await?; @@ -1399,6 +1426,14 @@ async fn set_default_error_handler( } } for updated_schedule_path in updated_schedules { + // managed ducklake-maintenance rows get the handler update (their + // failures should reach workspace handlers) but must not be + // pushed into git-sync as deployed schedules + if updated_schedule_path + .starts_with(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) + { + continue; + } handle_deployment_metadata( &authed.email, &authed.username, diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index 4a9e04c417..fc5a1fe874 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -11,7 +11,11 @@ path = "src/lib.rs" [features] default = [] enterprise = ["windmill-common/enterprise"] -private = ["windmill-common/private"] +private = [ + "windmill-common/private", + "windmill-queue/private", + "windmill-audit/private", +] cloud = ["windmill-common/cloud"] no_auth = ["windmill-api-auth/no_auth"] parquet = ["windmill-object-store/parquet"] diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 43d4ef241b..aaf04972e1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2295,6 +2295,17 @@ async fn edit_ducklake_config( require_admin(is_admin, &username)?; let is_superadmin = require_super_admin(&db, &email).await.is_ok(); + // Lake names end up interpolated in `ATTACH 'ducklake://'`, + // generated maintenance SQL and the reserved maintenance schedule path + // (CHECK-constrained to [\w-]+ segments). + for name in new_config.settings.ducklakes.keys() { + if !windmill_common::workspaces::is_valid_ducklake_name(name) { + return Err(Error::BadRequest(format!( + "Invalid ducklake name '{name}': only letters, digits, '_' and '-' are allowed" + ))); + } + } + let mut tx = db.begin().await?; let args_for_audit = format!("{:?}", new_config.settings); @@ -2309,21 +2320,22 @@ async fn edit_ducklake_config( ) .await?; + let old_ducklakes = sqlx::query_scalar!( + r#" + SELECT ws.ducklake->'ducklakes' AS ducklake_name + FROM workspace_settings ws + WHERE ws.workspace_id = $1 + "#, + &w_id + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(serde_json::Value::Null); + let old_ducklakes: HashMap = + serde_json::from_value(old_ducklakes).unwrap_or_default(); + // Check that non-superadmins are not abusing Instance databases if !is_superadmin { - let old_ducklakes = sqlx::query_scalar!( - r#" - SELECT ws.ducklake->'ducklakes' AS ducklake_name - FROM workspace_settings ws - WHERE ws.workspace_id = $1 - "#, - &w_id - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(serde_json::Value::Null); - let old_ducklakes: HashMap = - serde_json::from_value(old_ducklakes).unwrap_or_default(); for (name, dl) in new_config.settings.ducklakes.iter() { if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance { let old_dl = old_ducklakes.get(name); @@ -2341,7 +2353,7 @@ async fn edit_ducklake_config( } } - let config: serde_json::Value = serde_json::to_value(new_config.settings) + let config: serde_json::Value = serde_json::to_value(&new_config.settings) .map_err(|err| Error::internal_err(err.to_string()))?; sqlx::query!( @@ -2352,6 +2364,19 @@ async fn edit_ducklake_config( .execute(&mut *tx) .await?; + // Same tx as the settings update: a failed schedule sync/push must fail + // the whole save — nothing reconciles a half-applied state later. + let tx = windmill_queue::ducklake_maintenance::sync_ducklake_maintenance_schedules( + &db, + tx, + &w_id, + &new_config.settings.ducklakes, + &old_ducklakes, + &username, + &email, + ) + .await?; + tx.commit().await?; Ok(format!("Edit ducklake config for workspace {}", &w_id)) @@ -4079,6 +4104,9 @@ async fn clone_triggers_and_schedules( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { + // Managed ducklake-maintenance schedules are excluded: the fork starts + // with no ducklake config, so cloned rows could never resolve a lake and + // the schedule API refuses mutations under the reserved prefix. sqlx::query!( r#"INSERT INTO schedule ( workspace_id, path, edited_by, edited_at, schedule, enabled, script_path, @@ -4095,9 +4123,10 @@ async fn clone_triggers_and_schedules( on_recovery_times, on_recovery_extra_args, ws_error_handler_muted, retry, summary, no_flow_overlap, tag, paused_until, on_success, on_success_extra_args, cron_version, description, dynamic_skip, permissioned_as, labels - FROM schedule WHERE workspace_id = $2"#, + FROM schedule WHERE workspace_id = $2 AND NOT starts_with(path, $3)"#, target_workspace_id, source_workspace_id, + windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX, ) .execute(&mut **tx) .await?; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index ebbc98e83e..fa9ed3f2e5 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private", "windmill-object-store/private"] enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1f836cfb2b..fe7edb26b0 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -28917,6 +28917,27 @@ components: - path extra_args: type: string + maintenance: + type: object + description: + Scheduled maintenance (enterprise) - snapshot expiry, adjacent-file + compaction and orphaned-file cleanup, run as a managed per-lake schedule + required: [enabled] + properties: + enabled: + type: boolean + schedule: + type: string + description: cron cadence (v2, UTC); defaults to daily at 03h with a per-lake minute offset + retention_days: + type: integer + description: snapshot retention window in days (default 7); time-travel older than this stops working + compaction: + type: boolean + description: merge adjacent small parquet files (default true) + orphan_cleanup: + type: boolean + description: delete orphaned files older than max(retention, 1 day) (default true) DataTableSettings: type: object diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 66bda77bac..fa112ef4e1 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -919,11 +919,16 @@ pub(crate) async fn tarball_workspace( } if include_schedules.unwrap_or(false) { + // Managed ducklake-maintenance schedules are excluded: they are + // derived from the workspace ducklake settings (and admins bypass the + // RLS that hides them), so exporting them would drag unsyncable rows + // into git. let schedules = sqlx::query_as::<_, Schedule>( "SELECT workspace_id, path, edited_by, edited_at, schedule, timezone, enabled, script_path, is_flow, args, extra_perms, email, permissioned_as, error, on_failure, on_failure_times, on_failure_exact, on_failure_extra_args, on_recovery, on_recovery_times, on_recovery_extra_args, on_success, on_success_extra_args, ws_error_handler_muted, retry, no_flow_overlap, summary, description, tag, paused_until, cron_version, dynamic_skip, labels FROM schedule - WHERE workspace_id = $1", + WHERE workspace_id = $1 AND NOT starts_with(path, $2)", ) .bind(&w_id) + .bind(windmill_common::workspaces::DUCKLAKE_MAINTENANCE_PATH_PREFIX) .fetch_all(&mut *tx) .await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 5593a565ef..293490e434 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -842,6 +842,80 @@ pub struct Ducklake { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub maintenance: Option, +} + +/// Scheduled maintenance for a ducklake (enterprise): snapshot expiry, +/// adjacent-file compaction and orphaned-file cleanup, run as a managed +/// per-lake schedule. Not mirrored in `instance_config::Ducklake`: +/// instance-level lakes have no workspace to schedule into. +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct DucklakeMaintenance { + pub enabled: bool, + /// Cron (v2/croner, seconds optional). None → daily at 03:00 UTC with a + /// deterministic per-(workspace, lake) minute offset. + #[serde(skip_serializing_if = "Option::is_none")] + pub schedule: Option, + /// Snapshot retention window in days (default 7). Snapshots older than + /// this are expired: time-travel reads (`AT (VERSION => n)`) older than + /// the window stop working. 0 keeps only the current snapshot. + #[serde(skip_serializing_if = "Option::is_none")] + pub retention_days: Option, + /// Merge adjacent small parquet files (default true). + #[serde(skip_serializing_if = "Option::is_none")] + pub compaction: Option, + /// Delete orphaned files older than max(retention, 1 day) (default true). + #[serde(skip_serializing_if = "Option::is_none")] + pub orphan_cleanup: Option, +} + +impl DucklakeMaintenance { + pub const DEFAULT_RETENTION_DAYS: u32 = 7; + + pub fn retention_days(&self) -> u32 { + self.retention_days.unwrap_or(Self::DEFAULT_RETENTION_DAYS) + } + pub fn compaction(&self) -> bool { + self.compaction.unwrap_or(true) + } + pub fn orphan_cleanup(&self) -> bool { + self.orphan_cleanup.unwrap_or(true) + } +} + +/// Reserved schedule path namespace for managed ducklake maintenance +/// schedules. Must satisfy the `schedule.path` CHECK constraint +/// (`^[ufg](\/[\w-]+){2,}$`), hence the `f/` prefix; the folder itself never +/// exists. The schedule API rejects user mutations under this prefix and the +/// list/export endpoints filter it out — the lifecycle is owned by the +/// workspace ducklake settings. +/// +/// Accepted limitation: a schedule that pre-dated this namespace under a real +/// `ducklake_maintenance` folder keeps running (tick dispatch falls through +/// to its script when its path's lake has no enabled maintenance config, and +/// the settings sync only touches rows derived from config) but stays hidden +/// from list/export and immutable via the schedule API until renamed out of +/// the namespace. Judged unlikely enough to not warrant a discriminator +/// column or a rename migration. +pub const DUCKLAKE_MAINTENANCE_PATH_PREFIX: &str = "f/ducklake_maintenance/"; + +pub fn ducklake_maintenance_schedule_path(lake: &str) -> String { + format!("{DUCKLAKE_MAINTENANCE_PATH_PREFIX}{lake}") +} + +pub fn lake_from_ducklake_maintenance_path(path: &str) -> Option<&str> { + path.strip_prefix(DUCKLAKE_MAINTENANCE_PATH_PREFIX) +} + +/// Lake names are interpolated into `ATTACH 'ducklake://'`, generated +/// maintenance SQL and the reserved schedule path (CHECK-constrained to +/// `[\w-]+` segments), so they must stay to this charset. +pub fn is_valid_ducklake_name(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') } #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/windmill-queue/src/ducklake_maintenance_oss.rs b/backend/windmill-queue/src/ducklake_maintenance_oss.rs new file mode 100644 index 0000000000..ab6e035847 --- /dev/null +++ b/backend/windmill-queue/src/ducklake_maintenance_oss.rs @@ -0,0 +1,96 @@ +//! OSS fallback: scheduled ducklake maintenance (snapshot expiry, adjacent-file +//! compaction, orphaned-file cleanup via managed per-lake schedules) is an +//! enterprise feature; the implementation lives in windmill-ee-private +//! (see `ducklake_maintenance_ee`). In the public build the entry points report +//! that the enterprise edition is required. + +use std::collections::HashMap; + +use sqlx::{Postgres, Transaction}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobPayload, + schedule::Schedule, + workspaces::Ducklake, + DB, +}; + +/// Reconcile the managed `f/ducklake_maintenance/` schedule rows with +/// the ducklake settings, inside the caller's transaction. +/// +/// Not an authorization boundary: it mutates schedule rows for `w_id` on +/// behalf of `edited_by`/`email`, so the caller MUST already have enforced +/// workspace-admin on `w_id` and that the identity is the authenticated +/// caller's (as `edit_ducklake_config` does via `require_admin`). +// Only newly-enabled maintenance is rejected: a config that already had it +// enabled (e.g. an enterprise license lapsed) must not make every unrelated +// ducklake settings save fail, and the admin must be able to save it off. +pub async fn sync_ducklake_maintenance_schedules<'c>( + _db: &DB, + mut tx: Transaction<'c, Postgres>, + w_id: &str, + ducklakes: &HashMap, + previous: &HashMap, + _edited_by: &str, + _email: &str, +) -> Result> { + let enabled = |dl: &Ducklake| -> bool { dl.maintenance.as_ref().is_some_and(|m| m.enabled) }; + if ducklakes + .iter() + .any(|(name, dl)| enabled(dl) && !previous.get(name).is_some_and(|prev| enabled(prev))) + { + return Err(Error::BadRequest( + "Ducklake scheduled maintenance is only available in the enterprise edition" + .to_string(), + )); + } + + // Saving maintenance off (e.g. after an enterprise license lapsed) must + // remove the managed row AND its already-queued occurrence here too — + // otherwise a maintenance job pushed under the enterprise edition still + // runs once after the admin disabled it. Like the enterprise + // implementation, the removed set is derived from config, never from the + // path prefix. + let removed = previous + .iter() + .filter(|(name, dl)| enabled(dl) && !ducklakes.get(*name).is_some_and(|cur| enabled(cur))) + .map(|(name, _)| windmill_common::workspaces::ducklake_maintenance_schedule_path(name)) + .collect::>(); + for path in removed.iter() { + crate::schedule::clear_schedule(&mut tx, path, w_id).await?; + } + sqlx::query!( + "DELETE FROM schedule WHERE workspace_id = $1 AND path = ANY($2)", + w_id, + &removed + ) + .execute(&mut *tx) + .await?; + + Ok(tx) +} + +/// Build the job payload for one occurrence of a managed maintenance schedule +/// (`push_scheduled_job` calls this for reserved-prefix schedule paths). +/// Returns `(payload, tag, timeout, on_behalf_of_email, created_by)`. +/// +/// Always `Ok(None)` in the public build: the caller falls through to normal +/// script resolution, so a pre-existing user schedule under a real +/// `ducklake_maintenance` folder keeps running, while a managed row left over +/// from an enterprise period fails script resolution with NotFound and is +/// auto-disabled with `schedule.error` recorded by the post-completion +/// scheduler. +pub async fn build_maintenance_schedule_payload<'c>( + _tx: &mut Transaction<'c, Postgres>, + _schedule: &Schedule, +) -> Result< + Option<( + JobPayload, + Option, + Option, + Option, + String, + )>, +> { + Ok(None) +} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 59ac7f1fa8..8d8534c926 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -15,6 +15,13 @@ pub use cascade_ee as cascade; #[cfg(not(feature = "private"))] pub use cascade_oss as cascade; #[cfg(feature = "private")] +pub mod ducklake_maintenance_ee; +pub mod ducklake_maintenance_oss; +#[cfg(feature = "private")] +pub use ducklake_maintenance_ee as ducklake_maintenance; +#[cfg(not(feature = "private"))] +pub use ducklake_maintenance_oss as ducklake_maintenance; +#[cfg(feature = "private")] pub mod freshness_watchdog_ee; pub mod freshness_watchdog_oss; #[cfg(feature = "private")] diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 8816eeae25..2b5aef96a5 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -227,11 +227,30 @@ pub async fn push_scheduled_job<'c>( } } + // Managed ducklake maintenance schedule (enterprise): the runnable is a + // generated DuckDB script, not a deployed one — built in the EE module. + // None (CE build, or no enabled maintenance config for the path's lake) + // falls through to normal script resolution, so a user schedule that + // pre-dates the reserved prefix keeps running its script and a stale + // managed row fails resolution with NotFound (auto-disabling it with + // schedule.error recorded). + let maintenance_payload = + if windmill_common::workspaces::lake_from_ducklake_maintenance_path(&schedule.path) + .is_some() + { + crate::ducklake_maintenance::build_maintenance_schedule_payload(&mut tx, schedule) + .await? + } else { + None + }; + // If schedule handler is defined, wrap the scheduled job in a synthetic flow // with the handler as the first step (with stop_after_if to skip if handler returns false) - let (payload, tag, timeout, on_behalf_of_email, created_by) = if let Some(handler_path) = - &schedule.dynamic_skip + let (payload, tag, timeout, on_behalf_of_email, created_by) = if let Some(maintenance_payload) = + maintenance_payload { + maintenance_payload + } else if let Some(handler_path) = &schedule.dynamic_skip { // Build skip handler args let mut skip_handler_args = HashMap::>::new(); skip_handler_args.insert( diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index 70a53763e0..745b2685e9 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -1592,4 +1592,172 @@ mod schedule_push { Ok(()) } + + // ----------------------------------------------------------------------- + // push_scheduled_job: reserved ducklake-maintenance prefix + // ----------------------------------------------------------------------- + + // A schedule that pre-dates the reserved prefix (a user schedule under a + // real `ducklake_maintenance` folder) must fall through to normal script + // resolution when its path's lake has no enabled maintenance config — + // never be hijacked into the maintenance payload builder and auto-disabled. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_push_reserved_prefix_no_config_falls_through_to_script( + db: Pool, + ) -> anyhow::Result<()> { + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (ws, path, trigger, _) = get_queued_job(&db).await.unwrap(); + assert_eq!(ws, "test-workspace"); + assert_eq!( + path.as_deref(), + Some("f/system/test_script"), + "must resolve the schedule's script_path, not the maintenance builder" + ); + assert_eq!(trigger.as_deref(), Some("f/ducklake_maintenance/legacy")); + Ok(()) + } + + // With maintenance enabled for the path's lake, the occurrence is a + // raw-code duckdb job (kind preview, runnable_path = schedule path, + // duckdb tag pinned) — enterprise builds only. + #[cfg(feature = "private")] + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_push_reserved_prefix_with_config_builds_maintenance_job( + db: Pool, + ) -> anyhow::Result<()> { + sqlx::query( + r#"UPDATE workspace_settings SET ducklake = '{"ducklakes": {"legacy": { + "catalog": {"resource_type": "postgresql", "resource_path": "u/test/pg"}, + "storage": {"path": "legacy"}, + "maintenance": {"enabled": true, "retention_days": 3} + }}}'::jsonb WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + s.script_path = "f/ducklake_maintenance/legacy".to_string(); + s.tag = Some("duckdb".to_string()); + }); + let authed = make_authed(); + + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + + assert_eq!(count_queued_jobs(&db).await, 1); + let (kind, path, tag, raw_code) = sqlx::query_as::<_, (String, Option, String, Option)>( + "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code + FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", + ) + .fetch_one(&db) + .await?; + assert_eq!(kind, "preview"); + assert_eq!(path.as_deref(), Some("f/ducklake_maintenance/legacy")); + assert_eq!(tag, "duckdb"); + let raw_code = raw_code.expect("maintenance job must carry generated SQL"); + assert!(raw_code.contains("ducklake_expire_snapshots")); + assert!(raw_code.contains("INTERVAL '3 days'")); + Ok(()) + } + + // Saving maintenance off must remove the managed row AND its queued + // occurrence in BOTH builds: the enterprise sync reconciles, and the + // public stub must not leave a job pushed under the enterprise edition + // to run after the admin disabled maintenance (EE-to-CE downgrade). + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn test_sync_disable_clears_managed_row_and_queued_occurrence( + db: Pool, + ) -> anyhow::Result<()> { + use std::collections::HashMap; + use windmill_common::workspaces::{ + Ducklake, DucklakeCatalog, DucklakeCatalogResourceType, DucklakeMaintenance, + DucklakeStorage, + }; + use windmill_queue::ducklake_maintenance::sync_ducklake_maintenance_schedules; + + fn lake(maintenance_enabled: bool) -> Ducklake { + Ducklake { + catalog: DucklakeCatalog { + resource_type: DucklakeCatalogResourceType::Postgresql, + resource_path: "u/test/pg".to_string(), + }, + storage: DucklakeStorage { storage: None, path: "legacy".to_string() }, + extra_args: None, + maintenance: Some(DucklakeMaintenance { + enabled: maintenance_enabled, + schedule: None, + retention_days: None, + compaction: None, + orphan_cleanup: None, + }), + } + } + + // a managed row with a queued occurrence (queued via fall-through: no + // lake config exists yet, so the push resolves the script path) + let schedule = make_schedule(|s| { + s.path = "f/ducklake_maintenance/legacy".to_string(); + }); + sqlx::query( + "INSERT INTO schedule (workspace_id, path, schedule, timezone, edited_by, script_path, + is_flow, enabled, email, permissioned_as, cron_version) + VALUES ($1, $2, $3, 'UTC', $4, $5, false, true, $6, $7, 'v2')", + ) + .bind(&schedule.workspace_id) + .bind(&schedule.path) + .bind(&schedule.schedule) + .bind(&schedule.edited_by) + .bind(&schedule.script_path) + .bind(&schedule.email) + .bind(&schedule.permissioned_as) + .execute(&db) + .await?; + let authed = make_authed(); + let tx = db.begin().await?; + let tx = push_scheduled_job(&db, tx, &schedule, Some(&authed), None).await?; + tx.commit().await?; + assert_eq!(count_queued_jobs(&db).await, 1); + + // maintenance saved off + let previous = HashMap::from([("legacy".to_string(), lake(true))]); + let current = HashMap::from([("legacy".to_string(), lake(false))]); + let tx = db.begin().await?; + let tx = sync_ducklake_maintenance_schedules( + &db, + tx, + &schedule.workspace_id, + ¤t, + &previous, + "test-user", + "test@windmill.dev", + ) + .await?; + tx.commit().await?; + + assert_eq!( + count_queued_jobs(&db).await, + 0, + "queued occurrence must be cleared when maintenance is saved off" + ); + let row_exists: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM schedule WHERE workspace_id = $1 AND path = $2)", + ) + .bind(&schedule.workspace_id) + .bind(&schedule.path) + .fetch_one(&db) + .await?; + assert!(!row_exists, "managed schedule row must be deleted"); + Ok(()) + } } diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 4dd0ed0053..3ba4e9a213 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -1,6 +1,14 @@
    {#each tests as t (t.test)} -
  • - {#if t.violating > 0} - - failed: - {t.test} - — {t.violating} violating row{t.violating === 1 ? '' : 's'} - {:else} - - passed: - {t.test} +
  • +
    + {#if t.violating > 0} + + failed: + {t.test} + — {t.violating} violating row{t.violating === 1 ? '' : 's'} + {#if t.sample && t.sample.length > 0} + + {/if} + {:else} + + passed: + {t.test} + {/if} +
    + {#if t.violating > 0 && t.sample && t.sample.length > 0 && expanded.has(t.test)} +
    +
    + sample of the violating rows ({t.sample.length} of {t.violating}, unordered) +
    + +
    {/if}
    • {/each} diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index cbd81086a9..efb1dd2da1 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -574,24 +574,53 @@ // formats are produced by this repo's worker (see duckdb_executor.rs); the // derivation is inert (undefined) for every other DisplayResult use. let dataTests = $derived.by(() => { + // Both structured shapes carry `[{ test, violating, sample? }]`; the + // sample (bounded violating-row rows) may arrive as a JSON string (the + // worker keeps it string-typed through the summary row) and is optional + // by contract — anything malformed degrades to no sample, never to a + // dropped checklist. + const normalize = ( + dt: any + ): Array<{ test: string; violating: number; sample?: Record[] }> | undefined => { + if (typeof dt === 'string') { + try { + dt = JSON.parse(dt) + } catch { + return undefined + } + } + if ( + !Array.isArray(dt) || + dt.length === 0 || + !dt.every((x) => x && typeof x.test === 'string' && typeof x.violating === 'number') + ) { + return undefined + } + return dt.map((x) => { + let sample = x.sample + if (typeof sample === 'string') { + try { + sample = JSON.parse(sample) + } catch { + sample = undefined + } + } + if (!Array.isArray(sample) || !sample.every((r) => r && typeof r === 'object')) { + sample = undefined + } + return { test: x.test, violating: x.violating, sample } + }) + } // Success: structured column on the summary row. const row = Array.isArray(result) ? (result as any)?.[0] : (result as any) - let dt = row?.data_tests - if (typeof dt === 'string') { - try { - dt = JSON.parse(dt) - } catch { - dt = undefined - } - } - if ( - Array.isArray(dt) && - dt.length > 0 && - dt.every((x) => x && typeof x.test === 'string' && typeof x.violating === 'number') - ) { - return dt as Array<{ test: string; violating: number }> - } - // Failure: parse the worker's breakdown out of the error message. + const fromRow = normalize(row?.data_tests) + if (fromRow) return fromRow + // Failure: the worker attaches the same structured breakdown (plus + // per-failed-test samples) to the error payload. + const fromError = normalize((result as any)?.error?.data_tests) + if (fromError) return fromError + // Failure fallback for results predating the structured error payload: + // parse the worker's breakdown out of the error message. const msg = (result as any)?.error?.message if (typeof msg === 'string' && msg.includes('data tests failed on')) { const out: Array<{ test: string; violating: number }> = [] From 39eb9de1bce400109c130a081807e40e995ae068 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 09:58:06 +0200 Subject: [PATCH 08/34] feat(pipelines): fork data environments for ducklake materialization (dev data) (#9915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(pipelines): fork-scoped ducklake namespaces with read-defer to parent Co-Authored-By: Claude Fable 5 * feat(pipelines): fork graph indicator + fork ducklake namespace cleanup endpoint Co-Authored-By: Claude Fable 5 * feat(pipelines): fork_views-keyed view transition, fork lineage clone, design doc Co-Authored-By: Claude Fable 5 * fix(pipelines): review hardening - fork DATA_PATH last-wins, registry cache TTL, defer tests Co-Authored-By: Claude Fable 5 * feat(pipelines): per-lake isolated/shared choice at fork creation Co-Authored-By: Claude Fable 5 * fix(pipelines): chain-aware defer discovery + per-location fork namespace registry Co-Authored-By: Claude Fable 5 * fix(pipelines): lake-scoped fork schemas, catalog identity in registry, chain-aware graph chips Co-Authored-By: Claude Fable 5 * fix(pipelines): cleanup deletes fork data from the registered storage identity Co-Authored-By: Claude Fable 5 * fix(pipelines): collapse fork data-path segment to one component (slash-safe ids) Co-Authored-By: Claude Fable 5 * fix(pipelines): per-catalog ancestor checks, ancestor extra_args passthrough, test compile fix Co-Authored-By: Claude Fable 5 * fix(pipelines): invalidate fork ancestor-chain cache on lineage mutations Co-Authored-By: Claude Fable 5 * fix(pipelines): sweep descendant ancestor-chain caches on delete/reparent Co-Authored-By: Claude Fable 5 * fix(pipelines): run fork ducklake cleanup inline in delete_workspace Co-Authored-By: Claude Fable 5 * fix(pipelines): resolve fork cleanup credentials pre-commit, destroy post-commit Co-Authored-By: Claude Fable 5 * fix(pipelines): shared dev-workspace authz gate for namespace drop, invalidatable registration cache, segment-boundary delete filter - extract require_prod_admin_for_dev_workspace, used by both delete_workspace and drop_forked_ducklake_namespaces so the gates cannot drift - key FORK_DUCKLAKE_REGISTERED per workspace and invalidate it in cleanup_fork_ducklake_namespaces so a same-id fork recreated within the TTL re-registers its namespaces - filter listed object locations to the segment boundary before deletion Co-Authored-By: Claude Fable 5 * fix(pipelines): keep orphaned wm-fork-* workspaces ducklake-isolated parent_workspace_id is ON DELETE SET NULL, so a fork can outlive its parent with an empty ancestor chain while its cloned config still points at the shared lake. Key the isolation gate on the wm-fork- prefix as well as the chain (mirroring workspace_is_fork): orphaned forks get the write redirect, registration and cleanup with zero ancestors (no defer), and keep their 'fork' graph chips. Co-Authored-By: Claude Fable 5 * fix(pipelines): attach orphaned wm-fork-* ancestors at their fork namespace Chain position alone classified the last ancestor as a root, but an orphaned wm-fork-* ancestor (its own parent deleted, SET NULL) ends the chain the same way while its data lives in its fork namespace — its descendants' defer views bound the dead root's lake instead. Key the root-vs-fork decision on the wm-fork- prefix too, matching the resolution gate. Co-Authored-By: Claude Fable 5 * fix(pipelines): never inherit shared lake opt-out; durable cleanup ledger for failed fork deletions - fork creation strips cloned fork_behavior stamps before applying the request's shared_ducklakes list: sharing is a per-creation choice, a fork of a shared fork defaults back to isolated - fork_ducklake_namespace loses its ON DELETE CASCADE FK: rows are the durable cleanup ledger and outlive the workspace when physical cleanup fails post-commit; fork creation retries leftover rows for the reused id and refuses to create while a metadata schema still cannot be dropped (data-file leftovers alone are inert once the schema is gone and are swept by the next successful same-prefix cleanup) Co-Authored-By: Claude Fable 5 * fix(pipelines): make orphaned-namespace cleanup retries independent of deleted fork resources - ledger rows gain a schema_dropped phase flag: set when the schema drop succeeded but data cleanup failed, so later retries skip the schema phase and need no catalog credentials at all; registration resets it on re-attach (ON CONFLICT DO UPDATE) since attaching recreates the schema - retry-path $res: resolution falls back to the workspace being forked (the deleted fork's resources were clones of a parent's); live paths (delete_workspace prepare, drop endpoint) pass no fallback Co-Authored-By: Claude Fable 5 * fix(pipelines): fork tables from failed-after-commit runs stay fork-owned in defer and graph A failed materialization must not disguise a physically existing fork table as deferred: CREATE VIEW IF NOT EXISTS silently yields to the table, so reads hit fork data while the graph claims parent defer. - record_mat upsert preserves the last committed snapshot_id on failure - defer discovery and graph chips treat fork rows with a committed snapshot as fork-owned even when status is failed - inspect_fork_catalog also lists live fork tables (same round trip) and the defer list is filtered against them — covers rows recorded before this fix and tables created by raw SQL - drop stale FK-cascade wording in the design doc and sidebar comment Co-Authored-By: Claude Fable 5 * feat(pipelines): fork-mode ducklake settings — per-lake isolated/shared chips + banner, fork_behavior round-trip The workspace-settings ducklake editor had no fork awareness: no reminder of each lake's isolated/shared choice and no warning about what edits mean in a fork. It also rebuilt each lake explicitly on save, silently dropping fork_behavior — any settings save in a shared fork flipped the lake back to isolated. - fork detection mirrors the backend gate (parent link or wm-fork- prefix) - info banner explaining isolated vs shared semantics in a fork - per-lake chip (emerald 'isolated' / amber 'shared with parent') with tooltips, matching the pipeline graph chip colors - fork_behavior added to DucklakeSettingsType and preserved through convertDucklakeSettingsToBackend Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- ...d15474599734bc46ede6930650e2cba5d8a46.json | 15 + ...950a1563bd516e730aaa9d0f1501ea8cea7f9.json | 58 ++ ...6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json | 36 + ...4a6ae5091c37ec901975c3d564fd817a405b7.json | 23 + ...d4e427342fb78c66980984aefa7b8c7592ecb.json | 22 + ...c20a9e310092ae787efe4b0657399c799424.json} | 4 +- ...01fcab837629fd3c1a09cf82b21641a890444.json | 29 + ...650fb861aa19a009d6dfe9937304057f2b91f.json | 22 + ...a48ebfed85ad8800e293945eadc874d7e42f3.json | 19 + ...88958b335f6a24d1e93641eab6aaac86301f7.json | 14 + ...cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json | 20 + ...92da8ee231c772f88dd9145f01d7cf3b8f8c1.json | 15 + ...b134780e55a521a5cee8eb657b67bbd4b2588.json | 19 + backend/Cargo.lock | 1 + ...703170745_fork_ducklake_namespace.down.sql | 1 + ...60703170745_fork_ducklake_namespace.up.sql | 51 ++ backend/windmill-api-assets/src/lib.rs | 89 +- backend/windmill-api-workspaces/Cargo.toml | 3 +- .../windmill-api-workspaces/src/workspaces.rs | 126 +++ .../src/workspaces_extra.rs | 602 ++++++++++++- backend/windmill-api/openapi.yaml | 33 + .../windmill-common/src/materialization.rs | 98 +- backend/windmill-common/src/workspaces.rs | 841 +++++++++++++++++- backend/windmill-queue/tests/schedule_push.rs | 12 +- .../windmill-worker/src/duckdb_executor.rs | 372 +++++++- docs/pipeline-env-isolation.md | 219 +++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 1 + .../AssetGraph/AssetGraphDetailsPane.svelte | 33 + .../assets/AssetGraph/AssetNode.svelte | 24 +- .../AssetGraph/PipelineGraphEditor.svelte | 4 + .../lib/components/assets/AssetGraph/types.ts | 5 + .../components/sidebar/SidebarContent.svelte | 19 + .../CreateWorkspaceInner.svelte | 4 + .../workspaceSettings/DucklakeSettings.svelte | 66 +- .../ForkDucklakeSection.svelte | 61 ++ .../(logged)/pipeline/[folder]/+page.svelte | 11 + 36 files changed, 2930 insertions(+), 42 deletions(-) create mode 100644 backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json create mode 100644 backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json create mode 100644 backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json create mode 100644 backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json create mode 100644 backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json rename backend/.sqlx/{query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json => query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json} (63%) create mode 100644 backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json create mode 100644 backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json create mode 100644 backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json create mode 100644 backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json create mode 100644 backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json create mode 100644 backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json create mode 100644 backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json create mode 100644 backend/migrations/20260703170745_fork_ducklake_namespace.down.sql create mode 100644 backend/migrations/20260703170745_fork_ducklake_namespace.up.sql create mode 100644 docs/pipeline-env-isolation.md create mode 100644 frontend/src/lib/components/workspaceSettings/ForkDucklakeSection.svelte diff --git a/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json new file mode 100644 index 0000000000..5f74d57491 --- /dev/null +++ b/backend/.sqlx/query-256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s)\n SELECT $2, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s\n FROM script_trigger WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "256a302afef987857016f2fa636d15474599734bc46ede6930650e2cba5d8a46" +} diff --git a/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json new file mode 100644 index 0000000000..4599073796 --- /dev/null +++ b/backend/.sqlx/query-3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9.json @@ -0,0 +1,58 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake_name AS \"ducklake_name!\", metadata_schema AS \"metadata_schema!\",\n catalog AS \"catalog!\", storage AS \"storage!\",\n storage_ref AS \"storage_ref!\", data_path AS \"data_path!\",\n schema_dropped AS \"schema_dropped!\"\n FROM fork_ducklake_namespace WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ducklake_name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "metadata_schema!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "catalog!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "storage!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "storage_ref!", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "data_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "schema_dropped!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "3eeecc4bb26163cf6c2c19bc7c2950a1563bd516e730aaa9d0f1501ea8cea7f9" +} diff --git a/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json new file mode 100644 index 0000000000..1bcb7e6e53 --- /dev/null +++ b/backend/.sqlx/query-63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH ancestor AS (\n SELECT wid, ord FROM unnest($1::text[]) WITH ORDINALITY AS a(wid, ord)\n ), anc_mat AS (\n -- Per table, the nearest ancestor (lowest ord) that materialized it.\n SELECT DISTINCT ON (mp.asset_path) mp.asset_path, a.wid, a.ord\n FROM materialized_partition mp\n JOIN ancestor a ON a.wid = mp.workspace_id\n WHERE mp.asset_kind = 'ducklake' AND mp.status = 'materialized'\n AND split_part(mp.asset_path, '/', 1) = $3 AND mp.asset_path LIKE '%/%'\n ORDER BY mp.asset_path, a.ord\n ), fork_mat AS (\n -- Fork-OWNED assets: anything whose physical table exists in the fork\n -- namespace, not just clean materializations. A committed write whose data\n -- tests failed afterwards records status='failed' WITH a snapshot — its table\n -- is real, and a defer view emitted over it would silently yield to it\n -- (CREATE VIEW IF NOT EXISTS) while claiming the read defers to the parent.\n SELECT DISTINCT asset_path FROM materialized_partition\n WHERE workspace_id = $2 AND asset_kind = 'ducklake'\n AND (status = 'materialized' OR snapshot_id IS NOT NULL)\n ), latest_schema AS (\n SELECT DISTINCT ON (workspace_id, asset_path) workspace_id, asset_path, columns\n FROM materialized_asset_schema\n WHERE workspace_id = ANY($1) AND asset_kind = 'ducklake'\n ORDER BY workspace_id, asset_path, version DESC\n )\n SELECT am.asset_path AS \"asset_path!\",\n am.ord AS \"ord!\",\n COALESCE(EXISTS (\n SELECT 1 FROM jsonb_array_elements(ls.columns) e\n WHERE e->>'name' = 'is_current'\n ), false) AS \"has_current!\"\n FROM anc_mat am\n LEFT JOIN latest_schema ls\n ON ls.asset_path = am.asset_path AND ls.workspace_id = am.wid\n WHERE am.asset_path NOT IN (SELECT asset_path FROM fork_mat)\n ORDER BY am.asset_path\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "ord!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "has_current!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "TextArray", + "Text", + "Text" + ] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "63b0f39e3ea2b8d01d4c055258b6fc3dc4e9e1f237ba7c86cbdd70140bc55541" +} diff --git a/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json new file mode 100644 index 0000000000..9a0bbf46ef --- /dev/null +++ b/backend/.sqlx/query-6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, ARRAY['ducklakes', $2, 'fork_behavior'], '\"shared\"')\n WHERE workspace_id = $1 AND ducklake->'ducklakes' ? $2\n RETURNING 1 AS \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6ca5cbe50df89e6eead890758e34a6ae5091c37ec901975c3d564fd817a405b7" +} diff --git a/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json new file mode 100644 index 0000000000..b6b0f06be5 --- /dev/null +++ b/backend/.sqlx/query-76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ducklake->'ducklakes' FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "76af9b978fccef61f50a5eab335d4e427342fb78c66980984aefa7b8c7592ecb" +} diff --git a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json similarity index 63% rename from backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json rename to backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json index 17d372249b..5aacf1a295 100644 --- a/backend/.sqlx/query-5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19.json +++ b/backend/.sqlx/query-853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n snapshot_id = EXCLUDED.snapshot_id,\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", + "query": "INSERT INTO materialized_partition\n (workspace_id, asset_kind, asset_path, partition, status,\n snapshot_id, row_count, job_id, materialized_at, error)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9)\n ON CONFLICT (workspace_id, asset_kind, asset_path, partition)\n DO UPDATE SET status = EXCLUDED.status,\n -- A failed run records no snapshot, but must not erase the last\n -- committed one: a physical table from an earlier commit (or from a\n -- committed write whose data tests then failed) still exists, and\n -- fork defer/graph state keys on that evidence.\n snapshot_id = COALESCE(EXCLUDED.snapshot_id, materialized_partition.snapshot_id),\n row_count = EXCLUDED.row_count,\n job_id = EXCLUDED.job_id,\n materialized_at = now(),\n error = EXCLUDED.error", "describe": { "columns": [], "parameters": { @@ -43,5 +43,5 @@ }, "nullable": [] }, - "hash": "5e50ba0ae27b09a3ea1530c223e5039aa631bb5b0993ad09bc9a1381f6715f19" + "hash": "853230da671371fd8bbeeb4c87f8c20a9e310092ae787efe4b0657399c799424" } diff --git a/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json new file mode 100644 index 0000000000..efd12114f3 --- /dev/null +++ b/backend/.sqlx/query-8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n -- Fork rows also count when a snapshot ever committed (a failed run\n -- preserves it): the physical table exists, so reads hit the FORK's\n -- data — showing 'deferred' would misstate what a query returns.\n -- Ancestor rows still require a clean materialization.\n SELECT DISTINCT asset_path AS \"asset_path!\", workspace_id AS \"workspace_id!\"\n FROM materialized_partition\n WHERE (workspace_id = $1 OR workspace_id = ANY($2))\n AND asset_kind = 'ducklake'\n AND (status = 'materialized'\n OR (workspace_id = $1 AND snapshot_id IS NOT NULL))\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8dec884a4c3b2d105afe22d1d7a01fcab837629fd3c1a09cf82b21641a890444" +} diff --git a/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json new file mode 100644 index 0000000000..0fac858572 --- /dev/null +++ b/backend/.sqlx/query-cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE depth > 0 ORDER BY depth\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "cbbc6a894b6421d2cd49e59b7ed650fb861aa19a009d6dfe9937304057f2b91f" +} diff --git a/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json new file mode 100644 index 0000000000..83f7394250 --- /dev/null +++ b/backend/.sqlx/query-cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE fork_ducklake_namespace SET schema_dropped = true\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cd492743a27a158c9f5ff1fdb50a48ebfed85ad8800e293945eadc874d7e42f3" +} diff --git a/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json new file mode 100644 index 0000000000..f705dbc337 --- /dev/null +++ b/backend/.sqlx/query-d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings\n SET ducklake = jsonb_set(ducklake, '{ducklakes}', (\n SELECT COALESCE(jsonb_object_agg(key, value - 'fork_behavior'), '{}'::jsonb)\n FROM jsonb_each(ducklake->'ducklakes')\n ))\n WHERE workspace_id = $1 AND jsonb_typeof(ducklake->'ducklakes') = 'object'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d1142126245cdaf5d01dc67ab2488958b335f6a24d1e93641eab6aaac86301f7" +} diff --git a/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json new file mode 100644 index 0000000000..37bd25eb98 --- /dev/null +++ b/backend/.sqlx/query-e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO fork_ducklake_namespace\n (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ON CONFLICT (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path)\n DO UPDATE SET schema_dropped = false", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e27cc16414c181a055bf0666eb0cb16a356cb6be5c48f95b5ba200e7e6ab0efd" +} diff --git a/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json new file mode 100644 index 0000000000..7049fafe37 --- /dev/null +++ b/backend/.sqlx/query-f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns)\n SELECT $2, path, kind, usage_access_type, usage_path, usage_kind, columns\n FROM asset WHERE workspace_id = $1 AND usage_kind IN ('script', 'flow')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f3fef6964a211872b01c984b19192da8ee231c772f88dd9145f01d7cf3b8f8c1" +} diff --git a/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json new file mode 100644 index 0000000000..ad17a60330 --- /dev/null +++ b/backend/.sqlx/query-fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM fork_ducklake_namespace\n WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3\n AND storage = $4 AND storage_ref = $5 AND data_path = $6", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fbd1b02b9bbf86e17d6ee0da3e9b134780e55a521a5cee8eb657b67bbd4b2588" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aa66b845b3..1153916767 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14449,6 +14449,7 @@ version = "1.747.0" dependencies = [ "axum 0.8.9", "chrono", + "futures", "hex", "http 1.4.2", "hyper 1.10.1", diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql new file mode 100644 index 0000000000..fd722bf414 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS fork_ducklake_namespace; diff --git a/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql new file mode 100644 index 0000000000..7762304199 --- /dev/null +++ b/backend/migrations/20260703170745_fork_ducklake_namespace.up.sql @@ -0,0 +1,51 @@ +-- Registry of ducklake namespaces provisioned for fork/dev workspaces. One row per +-- (fork workspace, lake name): records the exact catalog metadata schema and data +-- sub-path the fork's jobs attach to, so fork deletion can drop the pg schema and +-- delete the S3 prefix deterministically (the row is written on first resolution, +-- before any physical state exists). +-- +-- Deliberately NO foreign key to workspace(id): rows are the durable cleanup ledger and +-- must OUTLIVE the workspace row when physical cleanup fails after the delete commits +-- (unreachable catalog, storage outage) — a CASCADE would erase the only record of the +-- orphaned namespace, letting a recreated same-id fork silently reattach stale tables. +-- Rows are deleted explicitly after each successful cleanup; leftover rows for a reused +-- id are retried at fork creation, which refuses to proceed while they cannot be cleaned. +CREATE TABLE fork_ducklake_namespace ( + workspace_id VARCHAR(50) NOT NULL, + ducklake_name VARCHAR(255) NOT NULL, + metadata_schema VARCHAR(63) NOT NULL, + -- Canonical identity of the catalog database the metadata schema lives in + -- (`:`, e.g. `instance:wm_ducklake` or + -- `postgres:u/admin/pg`). Cleanup connects to THIS catalog, not whatever the fork's + -- settings point at by then — a drifted catalog resource must not make cleanup drop a + -- schema in the wrong database and orphan the real one. + catalog TEXT NOT NULL, + -- Named workspace storage holding the fork's data files; '' = the default storage + -- (part of the PK, which cannot hold NULL). + storage TEXT NOT NULL DEFAULT '', + -- The storage's RESOLVED identity at registration time (`:`, e.g. `s3:u/admin/minio` or `filesystem:/data/lfs`; '' = unknown). Cleanup + -- deletes the fork prefix from THIS storage, not whatever the logical name points at by + -- then — repointing a storage after attach must not orphan the original fork data (or + -- delete a colliding prefix from the new one). + storage_ref TEXT NOT NULL DEFAULT '', + -- The fork namespace's data path within that storage (a bucket-root + -- `__wm_forks//…` prefix). + data_path TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Cleanup phase state: true once the metadata schema has been dropped but data files (or + -- the row delete) still failed. Later retries then skip the schema phase entirely — they + -- need NO catalog credentials, which may be gone for good with the deleted fork's + -- resources. Reset to false whenever a live fork re-registers the row (re-attaching + -- recreates the schema). + schema_dropped BOOLEAN NOT NULL DEFAULT false, + -- One row per physical location EVER attached: if the fork's lake settings drift + -- (catalog/storage/path change), later attaches add rows rather than replace them, so + -- cleanup covers every location the fork wrote, not just the first. + PRIMARY KEY (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path) +); + +-- Resolution runs under user_db transactions (SET LOCAL ROLE) in API contexts, so the +-- windmill roles need explicit grants (default privileges don't apply to app-created tables). +GRANT ALL ON fork_ducklake_namespace TO windmill_user; +GRANT ALL ON fork_ducklake_namespace TO windmill_admin; diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 587307afbb..2cebf9ed13 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -614,6 +614,12 @@ struct GraphQuery { struct GraphAssetNode { kind: AssetKind, path: String, + // Fork workspaces only: 'fork' when the fork has materialized this ducklake asset itself, + // 'deferred' when reads fall back to the parent workspace's current table (defer view). + // Absent outside forks, for non-ducklake assets, and for assets never materialized + // anywhere. Lockstep with TS `AssetGraphAssetNode.fork_materialization`. + #[serde(skip_serializing_if = "Option::is_none")] + fork_materialization: Option, } #[derive(Serialize, Debug)] @@ -805,6 +811,7 @@ async fn asset_graph( authed: ApiAuthed, Path(w_id): Path, Extension(user_db): Extension, + Extension(db): Extension, Query(q): Query, ) -> JsonResult { let mut tx = user_db.begin(&authed).await?; @@ -1250,9 +1257,89 @@ async fn asset_graph( runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); } + // Fork data-environment state per ducklake asset. The parent's rows are read on the plain + // pool: fork membership does not imply parent membership, and defer already exposes the + // parent's data to fork jobs — surfacing its materialization status is strictly less. + let fork_materialization_by_path: std::collections::HashMap = { + // The WHOLE ancestor chain, matching defer discovery: a grandchild fork whose direct + // parent only defers a table still reads it (from the grandparent), so it must show + // as deferred, not unmarked. + let ancestors = windmill_common::workspaces::fork_ancestor_chain(&db, &w_id).await?; + match ancestors { + // An orphaned `wm-fork-*` (parent deleted, chain empty) is still isolated — its + // own materializations must keep their 'fork' chips (nothing can be 'deferred'). + a if a.is_empty() && !w_id.starts_with(windmill_common::workspaces::WM_FORK_PREFIX) => { + std::collections::HashMap::new() + } + ancestors => { + // Lakes the fork chose to SHARE at creation have no fork namespace — their + // assets live in the parent's tables for real, so no chip applies. + let shared_lakes: std::collections::HashSet = sqlx::query_scalar!( + "SELECT ducklake->'ducklakes' FROM workspace_settings WHERE workspace_id = $1", + &w_id, + ) + .fetch_optional(&db) + .await? + .flatten() + .and_then(|v| v.as_object().cloned()) + .map(|lakes| { + lakes + .into_iter() + .filter(|(_, cfg)| { + cfg.get("fork_behavior").and_then(|b| b.as_str()) == Some("shared") + }) + .map(|(name, _)| name) + .collect() + }) + .unwrap_or_default(); + sqlx::query!( + r#" + -- Fork rows also count when a snapshot ever committed (a failed run + -- preserves it): the physical table exists, so reads hit the FORK's + -- data — showing 'deferred' would misstate what a query returns. + -- Ancestor rows still require a clean materialization. + SELECT DISTINCT asset_path AS "asset_path!", workspace_id AS "workspace_id!" + FROM materialized_partition + WHERE (workspace_id = $1 OR workspace_id = ANY($2)) + AND asset_kind = 'ducklake' + AND (status = 'materialized' + OR (workspace_id = $1 AND snapshot_id IS NOT NULL)) + "#, + &w_id, + &ancestors, + ) + .fetch_all(&db) + .await? + .into_iter() + .filter(|r| { + !shared_lakes.contains(r.asset_path.split('/').next().unwrap_or_default()) + }) + .fold(std::collections::HashMap::new(), |mut m, r| { + // A fork row wins over an inherited 'deferred' from any ancestor's row. + if r.workspace_id == w_id { + m.insert(r.asset_path, "fork"); + } else { + m.entry(r.asset_path).or_insert("deferred"); + } + m + }) + } + } + }; + let mut assets: Vec = asset_set .into_iter() - .map(|(kind, path)| GraphAssetNode { kind, path }) + .map(|(kind, path)| GraphAssetNode { + fork_materialization: (kind == AssetKind::Ducklake) + .then(|| { + fork_materialization_by_path + .get(&path) + .map(|s| s.to_string()) + }) + .flatten(), + kind, + path, + }) .collect(); assets.sort_by(|a, b| a.path.cmp(&b.path)); diff --git a/backend/windmill-api-workspaces/Cargo.toml b/backend/windmill-api-workspaces/Cargo.toml index fc5a1fe874..c25bb8f42b 100644 --- a/backend/windmill-api-workspaces/Cargo.toml +++ b/backend/windmill-api-workspaces/Cargo.toml @@ -18,7 +18,7 @@ private = [ ] cloud = ["windmill-common/cloud"] no_auth = ["windmill-api-auth/no_auth"] -parquet = ["windmill-object-store/parquet"] +parquet = ["windmill-object-store/parquet", "dep:futures"] [dependencies] windmill-common = { workspace = true, default-features = false } @@ -35,6 +35,7 @@ windmill-store.workspace = true axum.workspace = true chrono.workspace = true +futures = { workspace = true, optional = true } hex.workspace = true magic-crypt.workspace = true http.workspace = true diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index aaf04972e1..228b8d1831 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -183,6 +183,10 @@ pub fn workspaced_service() -> Router { "/drop_forked_datatable_databases", post(crate::workspaces_extra::drop_forked_datatable_databases), ) + .route( + "/drop_forked_ducklake_namespaces", + post(crate::workspaces_extra::drop_forked_ducklake_namespaces), + ) .route( "/get_datatable_full_schema", post(get_datatable_full_schema), @@ -449,6 +453,11 @@ struct CreateWorkspaceFork { /// forked workspace's datatable config to point to the new database. #[serde(default)] forked_datatables: Vec, + /// Lakes the user explicitly chose to SHARE with the parent (the fork then reads and + /// writes the parent's lake directly). Every lake not listed gets the default isolated + /// fork namespace + read-defer. + #[serde(default)] + shared_ducklakes: Vec, /// Create the fork as a persistent dev workspace: the id is not required to carry the /// `wm-fork-` prefix, and at most one dev workspace may exist per parent. #[serde(default)] @@ -4063,6 +4072,7 @@ async fn clone_workspace_data( // Clone CI test references clone_ci_test_references(tx, source_workspace_id, target_workspace_id).await?; clone_macro_registry(tx, source_workspace_id, target_workspace_id).await?; + clone_asset_usages_and_triggers(tx, source_workspace_id, target_workspace_id).await?; // Clone flows with new versions clone_flows(tx, source_workspace_id, target_workspace_id).await?; @@ -4659,6 +4669,39 @@ async fn clone_macro_registry( Ok(()) } +// Asset usage rows and `// on` subscriber triggers are deploy-derived like +// ci_test_reference / the macro registry: without cloning them the fork's +// pipeline graph has no asset nodes or lineage edges, and — worse — the asset +// dispatch cascade never fires in the fork (it reads `script_trigger`), so +// materializing an upstream node can't trigger its consumers until every +// script is manually redeployed. `job`-kind usage rows (runtime-detected, +// ephemeral) are skipped like the graph does. +async fn clone_asset_usages_and_triggers( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind, columns) + SELECT $2, path, kind, usage_access_type, usage_path, usage_kind, columns + FROM asset WHERE workspace_id = $1 AND usage_kind IN ('script', 'flow')", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "INSERT INTO script_trigger (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s) + SELECT $2, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, debounce_s, retry_count, retry_delay_s + FROM script_trigger WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_flows( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -5380,6 +5423,42 @@ async fn create_workspace_fork( // conflict instead of a misleading "maximum number of workspaces" error. check_fork_w_id_conflict(&db, &nw.id).await?; purge_stale_fork_diff_state(&db, &nw.id).await?; + // A previously deleted fork with this id may have left ducklake namespaces behind if its + // physical cleanup failed after the delete committed (registry rows are the durable + // ledger — no FK, they outlive the workspace). Retry that cleanup now, and refuse to + // proceed while any metadata schema still can't be dropped: creating the fork anyway + // would silently reattach the deterministic namespace and its stale tables. Data-file + // leftovers alone don't block — once the schema is gone they are inert (a deleted fork's + // `$res:` storage resource is gone forever, so they may never resolve again), and the + // surviving registry row has the next successful same-prefix cleanup sweep them. + // `$res:` fallback = the workspace being forked: the deleted fork's resource clones came + // from a parent, so the new parent is the natural donor for the retry's credentials. + let leftover_issues = crate::workspaces_extra::drop_forked_ducklake_namespaces_impl( + &db, + &nw.id, + Some(&parent_workspace_id), + ) + .await?; + let blocking: Vec<&str> = leftover_issues + .iter() + .filter(|i| i.blocking) + .map(|i| i.msg.as_str()) + .collect(); + if !blocking.is_empty() { + return Err(Error::BadRequest(format!( + "a previously deleted workspace with id '{}' left ducklake namespaces that could \ + not be cleaned up: {}; retry once the catalog is reachable again", + nw.id, + blocking.join("; ") + ))); + } + for i in &leftover_issues { + tracing::warn!( + "creating fork {}: leftover ducklake cleanup: {}", + nw.id, + i.msg + ); + } #[cfg(not(feature = "enterprise"))] _check_nb_of_workspaces(&db).await?; @@ -5477,6 +5556,42 @@ async fn create_workspace_fork( apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; } + // The settings clone copies the source's ducklake config verbatim — including a parent + // fork's own `fork_behavior` stamps. Sharing is a per-fork-creation choice, never + // inherited: reset any cloned stamps first, then apply this fork's requested list. + sqlx::query!( + r#"UPDATE workspace_settings + SET ducklake = jsonb_set(ducklake, '{ducklakes}', ( + SELECT COALESCE(jsonb_object_agg(key, value - 'fork_behavior'), '{}'::jsonb) + FROM jsonb_each(ducklake->'ducklakes') + )) + WHERE workspace_id = $1 AND jsonb_typeof(ducklake->'ducklakes') = 'object'"#, + &forked_id, + ) + .execute(&mut *tx) + .await?; + + // Stamp the per-lake ducklake fork choice into the fork's own settings. Only the `shared` + // opt-out needs stamping — absent `fork_behavior` already means isolated (the default), + // so unlisted lakes and API callers that omit the field stay safe. + for lake in &nw.shared_ducklakes { + let stamped = sqlx::query_scalar!( + r#"UPDATE workspace_settings + SET ducklake = jsonb_set(ducklake, ARRAY['ducklakes', $2, 'fork_behavior'], '"shared"') + WHERE workspace_id = $1 AND ducklake->'ducklakes' ? $2 + RETURNING 1 AS "one!""#, + &forked_id, + lake, + ) + .fetch_optional(&mut *tx) + .await?; + if stamped.is_none() { + return Err(Error::BadRequest(format!( + "cannot mark ducklake `{lake}` as shared: no such lake in the workspace settings" + ))); + } + } + // Lock the parent ("prod") so edits are funneled through this dev workspace. let locked_prod = nw.is_dev_workspace && (nw.lock_prod_deploy || nw.lock_prod_forking); if locked_prod { @@ -5501,6 +5616,10 @@ async fn create_workspace_fork( .await?; tx.commit().await?; + // A pre-creation lookup could have cached an EMPTY ancestor chain for this id, which + // would bypass ducklake fork isolation for the TTL. + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&forked_id); + if locked_prod { windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id); } @@ -5674,6 +5793,13 @@ async fn attach_dev_workspace( // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping // so per-workspace job tags route to the prod family immediately rather than after the TTL. windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Drop the cached ancestor chains too — the workspace existed BEFORE the attach, so a + // cached empty chain reads as "not a fork" and its ducklake jobs would write the shared + // lake until the TTL. Descendants' chains also gained the new root. + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } // Same reparent invalidates the billing-workspace mapping so its usage meters to prod at once. The // candidate can bring its own fork subtree, whose descendants had resolved their (now-stale) root // to the candidate's old family; invalidate them too so they meter to prod without waiting out the diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index daf83171e4..9fbaad2bf3 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -749,6 +749,16 @@ pub(crate) async fn change_workspace_id( // (fork-of-fork) self-heal via the 60s billing-cache TTL. for child in &reparented_children { windmill_queue::tags::invalidate_fork_parent_cache(child); + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(child); + // Grandchildren's cached chains contain the old (renamed-away) ancestor id; a stale + // chain drops all defer ancestors in the ducklake resolver, so sweep the subtree + // rather than letting it wait out the TTL. + for id in windmill_common::workspaces::list_fork_descendants(&db, child) + .await + .unwrap_or_default() + { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } #[cfg(feature = "cloud")] windmill_common::workspaces::invalidate_billing_workspace_cache(child); } @@ -829,28 +839,19 @@ pub(crate) async fn delete_workspace( // Deleting an attached dev workspace removes the parent prod's dev_workspace_lock (below), so it // must be a prod-admin action, not just the dev's own owner (dev ownership can diverge from // prod's) — mirrors detach_dev_workspace, which is prod-admin gated. - if let Some(prod) = sqlx::query_scalar!( - "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", - &w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten() - { - let is_prod_admin = sqlx::query_scalar!( - "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", - &prod, - &authed.email - ) - .fetch_optional(&mut *tx) - .await? - .unwrap_or(false); - if !is_prod_admin && !is_super_admin_email(&db, &authed.email).await? { - return Err(Error::PermissionDenied(format!( - "Deleting dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" - ))); - } - } + require_prod_admin_for_dev_workspace(&db, &authed, &w_id).await?; + + // Snapshot the fork's ducklake namespaces + RESOLVE their connection material NOW — the + // registry rows and the fork's `$res:` resources both CASCADE with the workspace row — + // but the destructive cleanup itself runs only after the commit below: a delete that + // fails mid-way must never leave a live workspace with its fork data destroyed and no + // registry row to retry from. Read-only: nothing is dropped here. + let fork_ducklake_cleanups = prepare_fork_ducklake_cleanups(&db, &w_id, None) + .await + .unwrap_or_else(|e| { + tracing::warn!("deleting workspace {w_id}: preparing ducklake cleanup: {e:#}"); + vec![] + }); sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id) .execute(&mut *tx) @@ -1086,6 +1087,17 @@ pub(crate) async fn delete_workspace( .await?; tx.commit().await?; + // Physical ducklake-namespace cleanup, post-commit, from the pre-read snapshot: fork + // namespaces are deterministic from (id, lake), so an orphan would silently REATTACH to a + // recreated identical fork id. Runs inline so every delete path is covered (CLI, + // force-delete dialog, direct API — not just the sidebar flow, which still calls the + // endpoint first for per-lake error toasts; the rerun is an idempotent no-op). Best + // effort: failures are logged — the workspace row is already gone, and broken storage + // credentials must not have made it undeletable. + for e in cleanup_fork_ducklake_namespaces(&db, &w_id, fork_ducklake_cleanups).await { + tracing::warn!("deleted workspace {w_id}: ducklake namespace cleanup: {}", e.msg); + } + if let Some(parent) = dev_lock_parent { windmill_common::workspaces::invalidate_protection_rules_cache(&parent); } @@ -1099,12 +1111,26 @@ pub(crate) async fn delete_workspace( // billing TTL. for id in std::iter::once(&w_id).chain(orphaned_children.iter()) { windmill_queue::tags::invalidate_fork_parent_cache(id); + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(id); #[cfg(feature = "cloud")] { windmill_common::workspaces::invalidate_billing_workspace_cache(id); windmill_common::workspaces::invalidate_team_plan_cache(id); } } + // Deeper descendants' cached ancestor CHAINS still contain the deleted workspace; unlike + // the sibling caches (which self-heal harmlessly via TTL), a stale chain makes the + // ducklake resolver drop all defer ancestors (all-or-nothing on broken links) — a visible + // defer/chips outage for up to the TTL. Anchor at the orphaned children: the deleted row + // is gone, but their subtrees are intact. + for child in orphaned_children.iter() { + for id in windmill_common::workspaces::list_fork_descendants(&db, child) + .await + .unwrap_or_default() + { + windmill_common::workspaces::invalidate_fork_ancestor_chain_cache(&id); + } + } Ok(format!("Deleted workspace {}", &w_id)) } @@ -1266,6 +1292,538 @@ pub async fn drop_forked_datatable_databases( Ok(Json(errors)) } +/// Drop this fork workspace's ducklake namespaces: the `wm_fork_*` metadata schema in each +/// lake's catalog database, plus (best effort) the fork's `__wm_forks//…` data files in +/// the workspace storage. Driven by the `fork_ducklake_namespace` registry written at first +/// fork attach, so it works even after settings drift. Returns errors per lake that failed; +/// the registry row is only deleted once both cleanups succeeded, so a retry resumes. +/// Same permission as delete_workspace: fork owner or super admin. `delete_workspace` also +/// runs this inline (the UI calls this endpoint first for error visibility; the inline run +/// covers every other delete path — CLI, force-delete dialogs, direct API — whose row delete +/// would otherwise CASCADE the registry away while orphaning the physical namespaces, which a +/// recreated identical fork id would then silently reattach). +pub async fn drop_forked_ducklake_namespaces( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> Result>> { + let is_fork = workspace_is_fork(&db, &w_id).await?; + let mut tx = db.begin().await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Dropping forked ducklake namespaces requires being the fork's owner or a superadmin" + .to_string(), + )); + } + tx.commit().await?; + require_prod_admin_for_dev_workspace(&db, &authed, &w_id).await?; + Ok(Json( + drop_forked_ducklake_namespaces_impl(&db, &w_id, None) + .await? + .into_iter() + .map(|i| i.msg) + .collect(), + )) +} + +/// The cleanup itself, shared by the endpoint and the post-commit `delete_workspace` run. No +/// authorization of its own — callers gate. +pub(crate) async fn drop_forked_ducklake_namespaces_impl( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, +) -> Result> { + let prepared = prepare_fork_ducklake_cleanups(db, w_id, res_fallback_w_id).await?; + Ok(cleanup_fork_ducklake_namespaces(db, w_id, prepared).await) +} + +struct ForkDucklakeNamespaceRow { + ducklake_name: String, + metadata_schema: String, + catalog: String, + storage: String, + storage_ref: String, + schema_dropped: bool, + data_path: String, +} + +/// A namespace row plus its RESOLVED connection material. `delete_workspace` prepares these +/// BEFORE its transaction commits — the registry rows AND the fork's `$res:` resources both +/// disappear with the workspace row — and runs the destructive cleanup only AFTER the commit: +/// a delete that fails mid-way must never leave a live workspace with its fork data +/// destroyed, and a committed delete must still be able to reach catalogs/storages whose +/// credentials lived in the (now gone) fork resources. Per-row resolution failures are +/// carried as strings so they surface with the other cleanup errors. +pub(crate) struct PreparedForkDucklakeCleanup { + ns: ForkDucklakeNamespaceRow, + catalog_pg: std::result::Result, + store: std::result::Result<(windmill_types::s3::LargeFileStorage, serde_json::Value), String>, +} + +/// One row per (lake, catalog, storage, data path) ever attached — settings drift adds rows, +/// so every location the fork wrote gets cleaned, not just the first. Read-only: resolves +/// credentials but destroys nothing. +/// `res_fallback_w_id`: second workspace to resolve `$res:` catalog/storage paths against +/// when the row's own workspace no longer has them — the retry path runs AFTER the fork and +/// its resources were deleted. Fork resources are clones of a parent's, so the workspace +/// being forked again is the natural donor. None on live-workspace paths. +pub(crate) async fn prepare_fork_ducklake_cleanups( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, +) -> Result> { + let rows = sqlx::query_as!( + ForkDucklakeNamespaceRow, + r#"SELECT ducklake_name AS "ducklake_name!", metadata_schema AS "metadata_schema!", + catalog AS "catalog!", storage AS "storage!", + storage_ref AS "storage_ref!", data_path AS "data_path!", + schema_dropped AS "schema_dropped!" + FROM fork_ducklake_namespace WHERE workspace_id = $1"#, + w_id + ) + .fetch_all(db) + .await?; + let mut prepared = Vec::with_capacity(rows.len()); + for ns in rows { + let catalog_pg = if ns.schema_dropped { + // Schema phase already done — the retry needs no catalog connection at all + // (its credentials may be unresolvable for good with the fork's resources gone). + Err("unused: metadata schema already dropped".to_string()) + } else { + resolve_fork_catalog_pg(db, w_id, res_fallback_w_id, &ns.ducklake_name, &ns.catalog) + .await + .map_err(|e| e.to_string()) + }; + let storage = Some(ns.storage.as_str()).filter(|s| !s.is_empty()); + let store = resolve_fork_storage(db, w_id, res_fallback_w_id, storage, &ns.storage_ref) + .await + .map_err(|e| e.to_string()); + prepared.push(PreparedForkDucklakeCleanup { ns, catalog_pg, store }); + } + Ok(prepared) +} + +/// Drop the prepared namespaces' metadata schemas + data files, deleting each registry row +/// only after both succeed (a no-op when the row already cascaded away with the workspace). +/// Returns per-namespace error strings; never fails as a whole. +/// One failed step of a fork ducklake cleanup. `blocking` = the metadata schema (or its +/// guards) failed, so the namespace is still attachable and a same-id fork must NOT be +/// created. Non-blocking = the schema is gone and only data files (or the registry-row +/// delete) failed — inert storage leftovers, tracked by the surviving row and swept by the +/// next successful cleanup of the same prefix. +pub(crate) struct ForkDucklakeCleanupIssue { + pub(crate) blocking: bool, + pub(crate) msg: String, +} + +pub(crate) async fn cleanup_fork_ducklake_namespaces( + db: &DB, + w_id: &str, + prepared: Vec, +) -> Vec { + // The registration once-cache must not outlive the rows it mirrors: a same-id fork + // recreated within the TTL would otherwise skip re-registration, and ITS eventual + // deletion would find no rows — orphaning the deterministic namespace for the next + // same-id fork to silently reattach. + windmill_common::workspaces::invalidate_fork_ducklake_registration_cache(w_id); + let mut errors: Vec = Vec::new(); + for PreparedForkDucklakeCleanup { ns, catalog_pg, store } in prepared { + // Hard guards mirroring the forked-datatable drop: never touch a schema outside the + // fork prefix, never delete outside the fork data dir — even if a registry row was + // somehow tampered with. + if !ns + .metadata_schema + .starts_with(windmill_common::workspaces::FORK_DUCKLAKE_SCHEMA_PREFIX) + { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Refusing to drop schema '{}' for ducklake://{}: name does not start with '{}'", + ns.metadata_schema, + ns.ducklake_name, + windmill_common::workspaces::FORK_DUCKLAKE_SCHEMA_PREFIX + ), + }); + continue; + } + // The fork's directory segment, NOT the raw workspace id: ids are only + // git-branch-safe and may contain `/`, which raw would let one fork's prefix nest + // inside a sibling's (`wm-fork-a/b` under `wm-fork-a`) and be swept by its cleanup. + let expected_prefix = format!( + "{}/{}/", + windmill_common::workspaces::FORK_DUCKLAKE_DATA_DIR, + windmill_common::workspaces::fork_data_dir_segment(w_id) + ); + if !format!("{}/", ns.data_path.trim_end_matches('/')).starts_with(&expected_prefix) { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Refusing to delete data path '{}' for ducklake://{}: not under '{}'", + ns.data_path, ns.ducklake_name, expected_prefix + ), + }); + continue; + } + + let drop_res = if ns.schema_dropped { + // Recorded as already dropped by a prior partial cleanup; skipping means no + // catalog credentials are needed. Registration resets the flag when a live fork + // re-attaches (recreating the schema). + Ok(()) + } else { + match catalog_pg { + Ok(pg) => drop_fork_ducklake_metadata_schema(db, pg, &ns.metadata_schema).await, + Err(e) => Err(Error::internal_err(e)), + } + }; + if let Err(e) = drop_res { + errors.push(ForkDucklakeCleanupIssue { + blocking: true, + msg: format!( + "Could not drop metadata schema '{}' for ducklake://{}: {e}", + ns.metadata_schema, ns.ducklake_name + ), + }); + continue; + } + let delete_res = match store { + Ok((lfs, resource_value)) => { + delete_fork_ducklake_data(lfs, resource_value, &ns.data_path).await + } + Err(e) => Err(Error::internal_err(e)), + }; + if let Err(e) = delete_res { + // Record the completed schema phase so retries never need catalog credentials + // again (best effort — a failed update just means the next retry re-drops an + // absent schema, which requires the catalog to be reachable). + sqlx::query!( + "UPDATE fork_ducklake_namespace SET schema_dropped = true + WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3 + AND storage = $4 AND storage_ref = $5 AND data_path = $6", + w_id, + &ns.ducklake_name, + &ns.catalog, + &ns.storage, + &ns.storage_ref, + &ns.data_path, + ) + .execute(db) + .await + .ok(); + errors.push(ForkDucklakeCleanupIssue { + blocking: false, + msg: format!( + "Dropped metadata schema but could not delete data files under '{}' for ducklake://{}: {e}", + ns.data_path, ns.ducklake_name + ), + }); + continue; + } + sqlx::query!( + "DELETE FROM fork_ducklake_namespace + WHERE workspace_id = $1 AND ducklake_name = $2 AND catalog = $3 + AND storage = $4 AND storage_ref = $5 AND data_path = $6", + w_id, + &ns.ducklake_name, + &ns.catalog, + &ns.storage, + &ns.storage_ref, + &ns.data_path, + ) + .execute(db) + .await + .map_err(|e| { + errors.push(ForkDucklakeCleanupIssue { + blocking: false, + msg: format!( + "Cleaned ducklake://{} but could not delete its registry row: {e}", + ns.ducklake_name + ), + }) + }) + .ok(); + } + errors +} + +/// Drop the fork's metadata schema in the catalog database recorded by the registry row — +/// NOT whatever the fork's settings point at by now: a drifted catalog resource must not make +/// cleanup drop a schema in the wrong database while orphaning the real one. The pg schema +/// holds only DuckLake metadata tables (auto-created at first fork attach), so a plain +/// `DROP SCHEMA … CASCADE` on the catalog connection removes the whole fork namespace. +/// Resolve the catalog connection recorded in the registry row (read-only): instance +/// identities rebuild instance creds; resource identities resolve their `$res:` in the fork's +/// workspace — which is why this must run BEFORE the workspace (and its resources) are +/// deleted. Mysql never registers (rejected at resolution). +async fn resolve_fork_catalog_pg( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + ducklake_name: &str, + catalog: &str, +) -> Result { + let (resource_type, resource_path) = catalog.split_once(':').ok_or_else(|| { + Error::internal_err(format!( + "ducklake://{ducklake_name}: malformed registry catalog identity `{catalog}`" + )) + })?; + let catalog_resource = if resource_type == "instance" { + let mut pg_creds = windmill_common::PgDatabase::parse_uri( + &windmill_common::get_database_url().await?.as_str().await, + )?; + pg_creds.dbname = resource_path.to_string(); + pg_creds.user = Some("custom_instance_user".to_string()); + pg_creds.password = + Some(windmill_common::utils::get_custom_pg_instance_password(db).await?); + serde_json::to_value(&pg_creds) + .map_err(|e| Error::internal_err(format!("serializing pg creds: {e}")))? + } else { + resolve_res_with_fallback(db, w_id, res_fallback_w_id, resource_path).await? + }; + serde_json::from_value(catalog_resource).map_err(|e| { + Error::internal_err(format!( + "ducklake://{ducklake_name}: catalog resource is not a postgres database: {e}" + )) + }) +} + +async fn drop_fork_ducklake_metadata_schema( + db: &DB, + pg: windmill_common::PgDatabase, + metadata_schema: &str, +) -> Result<()> { + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let res = client + .execute( + &format!( + "DROP SCHEMA IF EXISTS \"{}\" CASCADE", + metadata_schema.replace('"', "\"\"") + ), + &[], + ) + .await; + drop(client); + let _ = join_handle.await; + res.map_err(|e| Error::internal_err(format!("{e:#}")))?; + Ok(()) +} + +/// Resolve the storage identified by the registry's `storage_ref` (read-only) — the storage +/// that was active when the fork's data was WRITTEN, not whatever the logical storage name +/// points at by deletion time (a repointed storage must not orphan the original fork data, +/// nor get a colliding prefix deleted). `storage_ref` = '' falls back to resolving the +/// logical name against current settings (registration couldn't identify the storage — best +/// effort). Resolves the `$res:` in the fork's workspace, which is why this must run BEFORE +/// the workspace is deleted. +async fn resolve_fork_storage( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + storage: Option<&str>, + storage_ref: &str, +) -> Result<(windmill_types::s3::LargeFileStorage, serde_json::Value)> { + use windmill_types::s3::{ + AzureBlobStorage, FilesystemStorage, GoogleCloudStorage, LargeFileStorage, S3Storage, + }; + + let lfs: LargeFileStorage = if let Some((typ, path)) = storage_ref.split_once(':') { + // Rebuild the LFS entry from the registered identity; only the variant (which + // resource parser applies) and the path matter to `lfs_to_object_store_resource`. + let s3 = |p: &str| S3Storage { + s3_resource_path: p.to_string(), + public_resource: None, + advanced_permissions: None, + }; + match typ { + "S3Storage" => LargeFileStorage::S3Storage(s3(path)), + "S3AwsOidc" => LargeFileStorage::S3AwsOidc(s3(path)), + "AzureBlobStorage" => LargeFileStorage::AzureBlobStorage(AzureBlobStorage { + azure_blob_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "AzureWorkloadIdentity" => LargeFileStorage::AzureWorkloadIdentity(AzureBlobStorage { + azure_blob_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "GoogleCloudStorage" => LargeFileStorage::GoogleCloudStorage(GoogleCloudStorage { + gcs_resource_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + "FilesystemStorage" => LargeFileStorage::FilesystemStorage(FilesystemStorage { + root_path: path.to_string(), + public_resource: None, + advanced_permissions: None, + }), + other => { + return Err(Error::internal_err(format!( + "unknown registered storage type `{other}`" + ))) + } + } + } else { + let lfs_json = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await? + .flatten() + .ok_or_else(|| Error::BadRequest("workspace has no storage configured".to_string()))?; + // Named storages live under `secondary_storage`; `None`/`_default_` is the primary. + match storage.filter(|s| *s != "_default_") { + None => serde_json::from_value(lfs_json.clone()) + .map_err(|e| Error::internal_err(format!("parsing large_file_storage: {e}")))?, + Some(name) => serde_json::from_value( + lfs_json + .get("secondary_storage") + .and_then(|s| s.get(name)) + .cloned() + .ok_or_else(|| { + Error::BadRequest(format!("workspace has no storage named {name}")) + })?, + ) + .map_err(|e| Error::internal_err(format!("parsing storage {name}: {e}")))?, + } + }; + // Filesystem storage stores a direct path (`lfs_to_object_store_resource` ignores the + // resource value); everything else references a resource whose stored path may or may not + // carry the `$res:` prefix — same normalization as `get_workspace_s3_resource_from_lfs`. + let resource_value = if matches!(lfs, LargeFileStorage::FilesystemStorage(_)) { + serde_json::Value::Null + } else { + let path = lfs.get_s3_resource_path(); + let path = path.strip_prefix("$res:").unwrap_or(path); + resolve_res_with_fallback(db, w_id, res_fallback_w_id, path).await? + }; + Ok((lfs, resource_value)) +} + +/// Resolve a `$res:` path in `w_id`, falling back to the same path in `res_fallback_w_id` +/// when the first lookup fails — retry-path cleanups run after the fork workspace (and its +/// cloned resource rows) were deleted, and the fork's resources were clones of a parent's. +async fn resolve_res_with_fallback( + db: &DB, + w_id: &str, + res_fallback_w_id: Option<&str>, + resource_path: &str, +) -> Result { + let res = windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{resource_path}")), + w_id, + db, + ) + .await; + match (res, res_fallback_w_id) { + (Ok(v), _) => Ok(v), + (Err(e), None) => Err(e), + (Err(_), Some(fb)) => { + windmill_common::workspaces::transform_json_value_unchecked( + &serde_json::Value::String(format!("$res:{resource_path}")), + fb, + db, + ) + .await + } + } +} + +/// Delete every object under the fork's data prefix in the pre-resolved storage. Requires the +/// `parquet` (object store) feature; without it the metadata schema is still dropped and the +/// unreachable data files are left for manual cleanup. +#[cfg(feature = "parquet")] +async fn delete_fork_ducklake_data( + lfs: windmill_types::s3::LargeFileStorage, + resource_value: serde_json::Value, + data_path: &str, +) -> Result<()> { + use futures::{StreamExt, TryStreamExt}; + + let store = windmill_object_store::build_object_store_client( + &windmill_object_store::lfs_to_object_store_resource(&lfs, resource_value)?, + ) + .await?; + + let prefix = windmill_object_store::object_store_reexports::Path::from( + data_path.trim_matches('/').to_string(), + ); + let locations: Vec<_> = store + .list(Some(&prefix)) + .map_ok(|m| m.location) + .try_collect() + .await + .map_err(windmill_object_store::object_store_error_to_error)?; + // The object_store crate evaluates list prefixes on a path-SEGMENT basis (`a/b` does not + // match `a/bc/…`), so sibling fork segments sharing a string prefix are already excluded. + // Filter anyway — deletion must not depend on a listing implementation detail. + let boundary = format!("{}/", prefix.as_ref()); + let locations: Vec<_> = locations + .into_iter() + .filter(|l| l.as_ref().starts_with(&boundary)) + .collect(); + // 1000-object chunks: S3 DeleteObjects caps a batch at 1000 keys. + for chunk in locations.chunks(1000) { + store + .delete_stream(futures::stream::iter(chunk.iter().cloned().map(Ok)).boxed()) + .try_collect::>() + .await + .map_err(windmill_object_store::object_store_error_to_error)?; + } + Ok(()) +} + +#[cfg(not(feature = "parquet"))] +async fn delete_fork_ducklake_data( + _lfs: windmill_types::s3::LargeFileStorage, + _resource_value: serde_json::Value, + _data_path: &str, +) -> Result<()> { + Err(Error::internal_err( + "object storage support (parquet feature) is not compiled in".to_string(), + )) +} + +/// Destroying an ATTACHED dev workspace (or its data environments) must be a prod-admin +/// action, not just the dev's own owner (dev ownership can diverge from prod's) — mirrors +/// detach_dev_workspace. Shared by `delete_workspace` and `drop_forked_ducklake_namespaces` +/// so the two gates cannot drift: the sidebar calls the drop endpoint BEFORE deleteWorkspace, +/// and a weaker gate on the drop would let a non-prod-admin dev owner destroy the live dev's +/// materializations and then have the deletion itself rejected. No-op for non-dev workspaces. +async fn require_prod_admin_for_dev_workspace( + db: &DB, + authed: &ApiAuthed, + w_id: &str, +) -> Result<()> { + if let Some(prod) = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + w_id + ) + .fetch_optional(db) + .await? + .flatten() + { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &prod, + &authed.email + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if !is_prod_admin && !is_super_admin_email(db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Destroying dev workspace '{w_id}' or its data requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } + } + Ok(()) +} + async fn is_workspace_owner( authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fe7edb26b0..0c080b0a2a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4666,6 +4666,24 @@ paths: items: type: string + /w/{workspace}/workspaces/drop_forked_ducklake_namespaces: + post: + summary: drop this fork workspace's ducklake namespaces (catalog metadata schemas + data files) + operationId: dropForkedDucklakeNamespaces + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: list of errors (empty if all succeeded) + content: + application/json: + schema: + type: array + items: + type: string + /w/{workspace}/workspaces/import_pg_database: post: summary: import a PostgreSQL database from source to target via pg_dump @@ -21464,6 +21482,10 @@ paths: $ref: "#/components/schemas/AssetKind" path: type: string + fork_materialization: + type: string + enum: [fork, deferred] + description: Fork workspaces only — 'fork' when this ducklake asset was materialized in the fork itself, 'deferred' when reads fall back to the parent workspace's current table via a defer view. Omitted otherwise. runnables: type: array items: @@ -28178,6 +28200,11 @@ components: new_dbname: type: string description: "New database name for the fork" + shared_ducklakes: + type: array + items: + type: string + description: "Lake names the fork SHARES with the parent (reads and writes the parent's lake directly). Every lake not listed gets the default isolated fork namespace with read-defer to the parent." is_dev_workspace: type: boolean description: "Create the fork as a persistent dev workspace (id not required to carry the wm-fork- prefix; at most one per parent)" @@ -28917,6 +28944,12 @@ components: - path extra_args: type: string + fork_behavior: + type: string + enum: [isolated, shared] + description: + Fork workspaces only - how this lake behaves in the fork, stamped at fork + creation. Absent = isolated (fork-scoped namespace + read-defer to parent). maintenance: type: object description: diff --git a/backend/windmill-common/src/materialization.rs b/backend/windmill-common/src/materialization.rs index 8d9e31899f..236a71c855 100644 --- a/backend/windmill-common/src/materialization.rs +++ b/backend/windmill-common/src/materialization.rs @@ -94,7 +94,11 @@ pub async fn record_materialization<'e>( VALUES ($1, $2, $3, $4, $5, $6, $7, $8, now(), $9) ON CONFLICT (workspace_id, asset_kind, asset_path, partition) DO UPDATE SET status = EXCLUDED.status, - snapshot_id = EXCLUDED.snapshot_id, + -- A failed run records no snapshot, but must not erase the last + -- committed one: a physical table from an earlier commit (or from a + -- committed write whose data tests then failed) still exists, and + -- fork defer/graph state keys on that evidence. + snapshot_id = COALESCE(EXCLUDED.snapshot_id, materialized_partition.snapshot_id), row_count = EXCLUDED.row_count, job_id = EXCLUDED.job_id, materialized_at = now(), @@ -255,6 +259,98 @@ pub async fn record_asset_schema( Ok(true) } +/// One table a fork workspace should read from an ancestor through a defer view. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ForkDeferTable { + /// Lake-internal table name: the `asset_path` minus its `/` prefix (may itself be + /// `schema.table`). + pub table: String, + /// The owning ancestor's latest captured schema carries the SCD2 marker column + /// (`is_current`), so that lake also holds a managed `
_current` companion view + /// that consumers read — defer it alongside the table. + #[serde(default)] + pub with_current_view: bool, + /// Index into `DucklakeForkDefer.ancestors` (nearest-first) of the NEAREST ancestor that + /// materialized this table — the defer view must target that ancestor's namespace. In a + /// `fork → parent → root` chain where only root materialized a table, the parent has no + /// physical copy (it defers too), so a view over the parent would not bind. Defaults to 0 + /// (the direct parent) for wire compatibility with agents that predate the field. + #[serde(default)] + pub ancestor_idx: u32, +} + +/// Tables of lake `lake_name` materialized somewhere in the fork's ancestor chain +/// (nearest-first) but not (yet) in the fork — the fork's read-defer set, each mapped to the +/// nearest ancestor that owns a physical copy. Only `Materialized` rows count on every side: a +/// deferred table must physically exist in the targeted ancestor (defer views bind at CREATE +/// and would otherwise fail the whole job), and any successful fork materialization makes the +/// fork's own table authoritative. +/// +/// **Authorization:** performs no access control; trusted server-side callers only. It reads +/// ancestor workspaces' rows on behalf of a fork — acceptable because defer itself exposes +/// the ancestors' table contents to fork members. +pub async fn list_fork_defer_tables<'e>( + executor: impl PgExecutor<'e>, + ancestor_workspace_ids: &[String], + fork_workspace_id: &str, + lake_name: &str, +) -> Result> { + let rows = sqlx::query!( + r#" + WITH ancestor AS ( + SELECT wid, ord FROM unnest($1::text[]) WITH ORDINALITY AS a(wid, ord) + ), anc_mat AS ( + -- Per table, the nearest ancestor (lowest ord) that materialized it. + SELECT DISTINCT ON (mp.asset_path) mp.asset_path, a.wid, a.ord + FROM materialized_partition mp + JOIN ancestor a ON a.wid = mp.workspace_id + WHERE mp.asset_kind = 'ducklake' AND mp.status = 'materialized' + AND split_part(mp.asset_path, '/', 1) = $3 AND mp.asset_path LIKE '%/%' + ORDER BY mp.asset_path, a.ord + ), fork_mat AS ( + -- Fork-OWNED assets: anything whose physical table exists in the fork + -- namespace, not just clean materializations. A committed write whose data + -- tests failed afterwards records status='failed' WITH a snapshot — its table + -- is real, and a defer view emitted over it would silently yield to it + -- (CREATE VIEW IF NOT EXISTS) while claiming the read defers to the parent. + SELECT DISTINCT asset_path FROM materialized_partition + WHERE workspace_id = $2 AND asset_kind = 'ducklake' + AND (status = 'materialized' OR snapshot_id IS NOT NULL) + ), latest_schema AS ( + SELECT DISTINCT ON (workspace_id, asset_path) workspace_id, asset_path, columns + FROM materialized_asset_schema + WHERE workspace_id = ANY($1) AND asset_kind = 'ducklake' + ORDER BY workspace_id, asset_path, version DESC + ) + SELECT am.asset_path AS "asset_path!", + am.ord AS "ord!", + COALESCE(EXISTS ( + SELECT 1 FROM jsonb_array_elements(ls.columns) e + WHERE e->>'name' = 'is_current' + ), false) AS "has_current!" + FROM anc_mat am + LEFT JOIN latest_schema ls + ON ls.asset_path = am.asset_path AND ls.workspace_id = am.wid + WHERE am.asset_path NOT IN (SELECT asset_path FROM fork_mat) + ORDER BY am.asset_path + "#, + ancestor_workspace_ids, + fork_workspace_id, + lake_name, + ) + .fetch_all(executor) + .await?; + Ok(rows + .into_iter() + .map(|r| ForkDeferTable { + table: r.asset_path[lake_name.len() + 1..].to_string(), + with_current_view: r.has_current, + // WITH ORDINALITY is 1-based; ancestors vec is 0-based. + ancestor_idx: (r.ord - 1).max(0) as u32, + }) + .collect()) +} + /// All captured schema versions for one asset, newest version first. /// /// **Authorization:** performs no access control (mirrors diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 293490e434..14bc5fc344 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -842,10 +842,27 @@ pub struct Ducklake { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + /// How this lake behaves when the workspace is a fork/dev workspace. Only meaningful in a + /// fork's own settings; stamped at fork creation from the user's per-lake choice. Absent = + /// `Isolated` — the safe default, so forks created before this field existed (and API + /// callers that omit it) never write the parent's lake. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fork_behavior: Option, #[serde(skip_serializing_if = "Option::is_none")] pub maintenance: Option, } +/// Per-lake fork data-environment choice, made at fork creation. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum DucklakeForkBehavior { + /// Fork-scoped namespace + read-defer to the parent (default). + Isolated, + /// The fork reads AND WRITES the parent's lake directly — explicit opt-out of isolation + /// (e.g. a fork meant to run prod-equivalent backfills). + Shared, +} + /// Scheduled maintenance for a ducklake (enterprise): snapshot expiry, /// adjacent-file compaction and orphaned-file cleanup, run as a managed /// per-lake schedule. Not mirrored in `instance_config::Ducklake`: @@ -949,6 +966,213 @@ pub struct DucklakeWithConnData { pub storage: DucklakeStorage, #[serde(skip_serializing_if = "Option::is_none")] pub extra_args: Option, + /// Present when the resolved workspace is a fork/dev workspace. Carries the read-defer + /// context (ancestor namespaces + tables to expose as views over the direct parent). The + /// fork *write* redirect is folded into `storage.path`/`extra_args` above, so an agent + /// worker that predates this field still writes the fork namespace (it only misses the + /// defer views). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fork_defer: Option, +} + +/// Read-defer context for a fork workspace's ducklake: which ancestor namespaces to attach +/// read-only, and which tables to expose as views over the direct parent because the fork has +/// not materialized them yet. `ancestors` is empty when defer is unavailable (an ancestor no +/// longer defines the lake) — the fork namespace still isolates writes in that case. +#[derive(Deserialize, Serialize)] +pub struct DucklakeForkDefer { + /// Nearest-first (direct parent … root), each resolved from that workspace's own settings. + pub ancestors: Vec, + pub defer_tables: Vec, + /// Views currently live in the fork namespace (read from the catalog's `ducklake_view` at + /// resolution time, lake-internal names). The worker's view→table transition (DROP VIEW + /// before a managed materialize) keys on THIS, not on recorded materialization status: + /// after a failed run the status can't distinguish a defer view from a real table, and + /// `DROP VIEW` against a table (or `CREATE TABLE` against a view) errors — either guess + /// would wedge the asset until manual repair. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub fork_views: Vec, +} + +/// Connection data for one ancestor namespace of a fork's ducklake. +#[derive(Deserialize, Serialize)] +pub struct DucklakeAncestorAttach { + pub workspace_id: String, + /// DuckDB catalog alias this namespace must be attached under. Persisted defer-view SQL + /// references it, so it is a pure function of (lake name, ancestor workspace id) and every + /// session reading those views attaches the ancestor under this exact alias. + pub alias: String, + pub catalog: DucklakeCatalog, + pub catalog_resource: serde_json::Value, + pub storage: DucklakeStorage, + /// None = the lake's default metadata schema (the ancestor is a root/non-fork workspace). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub metadata_schema: Option, + /// The ancestor config's own non-reserved ATTACH args (e.g. `ENCRYPTED true`), already + /// stripped of the fork-owned `METADATA_SCHEMA`/`DATA_PATH`/`OVERRIDE_DATA_PATH` — an + /// option-dependent lake would otherwise fail its read-only ancestor attach even though + /// the same lake attaches fine everywhere else. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub extra_args: Option, +} + +/// Prefix of fork-scoped ducklake metadata schemas. Cleanup refuses to drop any pg schema not +/// carrying it, mirroring the `wm_fork_` guard on forked datatable databases. +pub const FORK_DUCKLAKE_SCHEMA_PREFIX: &str = "wm_fork_"; + +/// Bucket-root directory holding all fork namespaces' data files: each fork writes under +/// `{FORK_DUCKLAKE_DATA_DIR}//` where the segment is +/// [`fork_data_dir_segment`] (see [`fork_data_path`] for why it wraps the lake's path instead +/// of nesting under it). +pub const FORK_DUCKLAKE_DATA_DIR: &str = "__wm_forks"; + +fn mangle_identifier(s: &str, max: usize) -> String { + s.chars() + .take(max) + .map(|c| { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphanumeric() { + c + } else { + '_' + } + }) + .collect() +} + +/// Deterministic, injective pg-schema name for a fork workspace's namespace of ONE lake: +/// `wm_fork___<8-hex sha256>`, ≤58 chars (pg limit 63). +/// Lake-scoped, not just workspace-scoped: two lakes of one workspace may share a catalog +/// database, and a per-workspace schema would merge their namespaces in the fork (tables and +/// snapshots colliding across `ducklake://a/…` and `ducklake://b/…`). The hash keeps distinct +/// (workspace, lake) pairs distinct after mangling. The registry row records the computed name +/// for cleanup, but ATTACH recomputes it — so this function must stay stable across releases +/// or existing forks would silently lose their namespace. +pub fn fork_ducklake_metadata_schema(w_id: &str, lake_name: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = hex::encode(&Sha256::digest(format!("{w_id}\0{lake_name}").as_bytes())[..4]); + format!( + "{FORK_DUCKLAKE_SCHEMA_PREFIX}{}_{}_{hash}", + mangle_identifier(w_id, 24), + mangle_identifier(lake_name, 16) + ) +} + +/// The `METADATA_SCHEMA ''` value carried in a lake config's `extra_args`, if any — the +/// schema the lake's OWN catalog namespace lives in (how one catalog database hosts several +/// lakes). Ancestor read-only attaches must preserve it or they'd bind the wrong namespace. +pub fn extract_metadata_schema_arg(extra_args: &str) -> Option { + lazy_static::lazy_static! { + static ref MS: regex::Regex = regex::Regex::new( + r"(?i)\bMETADATA_SCHEMA\s*(?:'([^']*)'|([A-Za-z0-9_]+))" + ) + .unwrap(); + } + // Last occurrence wins, matching DuckDB's duplicate-option semantics. + MS.captures_iter(extra_args) + .last() + .and_then(|c| c.get(1).or_else(|| c.get(2))) + .map(|m| m.as_str().to_string()) +} + +/// Deterministic DuckDB attach alias for an ancestor namespace of a fork's lake. Persisted +/// defer-view SQL references it (same stability requirement as +/// [`fork_ducklake_metadata_schema`]). +pub fn fork_ducklake_ancestor_alias(lake_name: &str, ancestor_w_id: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = + hex::encode(&Sha256::digest(format!("{lake_name}\0{ancestor_w_id}").as_bytes())[..4]); + format!( + "__wm_dl_{}_{}_{hash}", + mangle_identifier(lake_name, 20), + mangle_identifier(ancestor_w_id, 20) + ) +} + +/// Strip `METADATA_SCHEMA` / `DATA_PATH` / `OVERRIDE_DATA_PATH` tokens from ducklake ATTACH +/// extra args. In a fork these options are injected by the fork resolution; a user- or +/// settings-supplied duplicate silently wins (DuckDB keeps the last occurrence) and would +/// escape the fork namespace back to the parent's, so they are removed rather than overridden. +pub fn strip_fork_reserved_attach_args(extra_args: &str) -> String { + lazy_static::lazy_static! { + static ref RESERVED: regex::Regex = regex::Regex::new( + r"(?i)\b(METADATA_SCHEMA|DATA_PATH|OVERRIDE_DATA_PATH)\s*('[^']*'|[A-Za-z0-9_]+)" + ) + .unwrap(); + } + RESERVED + .replace_all(extra_args, "") + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect::>() + .join(", ") +} + +lazy_static::lazy_static! { + /// fork workspace id -> (ancestor chain nearest-first, expiry ts). Empty chain = not a fork. + /// `parent_workspace_id` only changes on dev-workspace attach/detach, so a short TTL is safe. + static ref FORK_ANCESTOR_CHAIN_CACHE: Cache, i64)> = Cache::new(5000); + /// fork workspace id -> (locations recently upserted into `fork_ducklake_namespace`, + /// expiry ts), so the registry write doesn't run on every job of a fork. Keyed by + /// workspace id so cleanup can drop a fork's whole entry: a same-id fork recreated within + /// the TTL must re-register, or its materializations would carry no registry row and leak + /// at ITS deletion. TTL'd as a second line of defense for cleanup paths that bypass + /// `cleanup_fork_ducklake_namespaces` (e.g. manual registry edits). + static ref FORK_DUCKLAKE_REGISTERED: Cache, i64)> = + Cache::new(5000); +} + +/// Drop the "already registered" once-cache for a workspace. MUST be called whenever +/// `fork_ducklake_namespace` rows for that workspace are deleted (namespace cleanup, workspace +/// deletion): a surviving entry would make a same-id fork recreated within the TTL skip +/// re-registration, orphaning its namespace at deletion time. +pub fn invalidate_fork_ducklake_registration_cache(w_id: &str) { + FORK_DUCKLAKE_REGISTERED.remove(w_id); +} + +/// Drop the cached ancestor chain for a workspace. MUST be called wherever +/// `parent_workspace_id` lineage changes (fork creation, dev-workspace attach, reparenting +/// rename, deletion): a cached EMPTY chain reads as "not a fork" and would bypass ducklake +/// fork isolation for the TTL — the first jobs after a dev-workspace attach would write the +/// shared lake. +pub fn invalidate_fork_ancestor_chain_cache(w_id: &str) { + FORK_ANCESTOR_CHAIN_CACHE.remove(w_id); +} + +/// Ancestors of `w_id`, nearest-first (direct parent … root). Empty for a non-fork workspace or +/// an unknown id. The depth bound is a cycle-safety backstop, same convention as +/// [`fork_chain_depth`]. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_ancestor_chain(db: &crate::DB, w_id: &str) -> Result> { + let now = chrono::Utc::now().timestamp(); + if let Some((chain, expiry)) = FORK_ANCESTOR_CHAIN_CACHE.get(w_id) { + if expiry > now { + return Ok(chain); + } + } + let chain = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT id AS "id!" FROM chain WHERE depth > 0 ORDER BY depth + "#, + w_id + ) + .fetch_all(db) + .await + .map_err(|e| Error::internal_err(format!("resolving fork ancestors of {w_id}: {e:#}")))?; + FORK_ANCESTOR_CHAIN_CACHE.insert(w_id.to_string(), (chain.clone(), now + 60)); + Ok(chain) } pub async fn get_ducklake_from_db_unchecked( @@ -956,6 +1180,29 @@ pub async fn get_ducklake_from_db_unchecked( w_id: &str, db: &DB, ) -> Result { + let (base, fork_behavior) = ducklake_conn_data(name, w_id, db).await?; + let chain = fork_ancestor_chain(db, w_id).await?; + // `Shared` is the explicit fork-creation opt-out of isolation: the fork reads and writes + // the parent's lake through its own (cloned) config, exactly like a non-fork workspace. + // An empty chain alone does NOT mean "not a fork": `parent_workspace_id` is ON DELETE SET + // NULL, so a `wm-fork-*` workspace can outlive its parent — its cloned config still points + // at the shared lake, so the prefix keeps it isolated (mirrors `workspace_is_fork`). No + // ancestors ⇒ no defer, write redirect only. Prefix-less dev workspaces can't be orphaned + // (deletion of their prod is blocked while attached). + if (chain.is_empty() && !w_id.starts_with(WM_FORK_PREFIX)) + || fork_behavior == Some(DucklakeForkBehavior::Shared) + { + return Ok(base); + } + fork_scoped_ducklake(name, w_id, base, chain, db).await +} + +/// Resolve one workspace's own config for lake `name` — no fork awareness. +async fn ducklake_conn_data( + name: &str, + w_id: &str, + db: &DB, +) -> Result<(DucklakeWithConnData, Option)> { let ducklake = sqlx::query_scalar!( r#" SELECT ws.ducklake->'ducklakes'->$2 AS config @@ -988,13 +1235,447 @@ pub async fn get_ducklake_from_db_unchecked( ) .await? }; + let fork_behavior = ducklake.fork_behavior; let ducklake = DucklakeWithConnData { catalog_resource, catalog: ducklake.catalog, storage: ducklake.storage, extra_args: ducklake.extra_args, + fork_defer: None, }; - Ok(ducklake) + Ok((ducklake, fork_behavior)) +} + +/// The fork's directory segment under `__wm_forks/`: mangled + hashed into ONE path component +/// (same recipe and stability requirement as [`fork_ducklake_metadata_schema`]). Fork/dev +/// workspace ids are only git-branch-safe and may contain `/` (e.g. `wm-fork-a/b`) — used +/// raw, such an id would nest inside the sibling `wm-fork-a`'s prefix and be swept by ITS +/// cleanup. The hash keeps distinct ids distinct after mangling. +pub fn fork_data_dir_segment(fork_w_id: &str) -> String { + use sha2::{Digest, Sha256}; + let hash = hex::encode(&Sha256::digest(fork_w_id.as_bytes())[..4]); + format!("{}_{hash}", mangle_identifier(fork_w_id, 40)) +} + +/// The fork namespace's data path within the same bucket: a bucket-root +/// `__wm_forks//` prefix wrapping the lake's own path, NOT a sub-path under it — +/// the parent lake's maintenance (snapshot expiry / `ducklake_delete_orphaned_files`) scans +/// everything under the parent's DATA_PATH and would treat live fork files nested there as +/// orphans and delete them. +pub fn fork_data_path(base_path: &str, fork_w_id: &str) -> String { + let segment = fork_data_dir_segment(fork_w_id); + let base = base_path.trim_matches('/'); + if base.is_empty() { + format!("{FORK_DUCKLAKE_DATA_DIR}/{segment}") + } else { + format!("{FORK_DUCKLAKE_DATA_DIR}/{segment}/{base}") + } +} + +/// Redirect a fork workspace's lake to its fork-scoped namespace (same catalog DB, fork +/// metadata schema + data sub-path) and assemble the read-defer context over its ancestors. +async fn fork_scoped_ducklake( + name: &str, + w_id: &str, + mut base: DucklakeWithConnData, + chain: Vec, + db: &DB, +) -> Result { + if base.catalog.resource_type == DucklakeCatalogResourceType::Mysql { + return Err(Error::BadRequest(format!( + "ducklake {name}: mysql-catalog lakes are not supported in fork workspaces — \ + running against the shared catalog would write the parent workspace's data" + ))); + } + let metadata_schema = fork_ducklake_metadata_schema(w_id, name); + let fork_path = fork_data_path(&base.storage.path, w_id); + let catalog_identity = ducklake_catalog_identity(&base.catalog); + register_fork_ducklake_namespace( + db, + w_id, + name, + &metadata_schema, + &catalog_identity, + base.storage.storage.as_deref(), + &fork_path, + ) + .await?; + + // Ancestor namespaces, nearest-first, each from its own settings so a fork-side settings + // edit can't silently repoint what "parent" means. All-or-nothing: a broken link anywhere + // in the chain disables defer entirely (a defer view over a missing ancestor attach fails + // at bind time and would kill unrelated jobs), but write isolation still applies. + let mut ancestors = Vec::with_capacity(chain.len()); + for (i, ancestor_id) in chain.iter().enumerate() { + match ducklake_conn_data(name, ancestor_id, db).await { + Ok((mut a, ancestor_behavior)) => { + // A fork ancestor lives in its own namespace UNLESS its lake is `Shared` — + // then it never redirected and its data sits at its config's default + // location, exactly like a root workspace. Chain position alone can't tell + // the two apart: an orphaned `wm-fork-*` ancestor (its own parent deleted, + // `parent_workspace_id` SET NULL) ends the chain like a root but its data + // lives in ITS fork namespace — key on the prefix too, as in resolution. + let is_isolated_fork = (i + 1 < chain.len() + || ancestor_id.starts_with(WM_FORK_PREFIX)) + && ancestor_behavior != Some(DucklakeForkBehavior::Shared); + let metadata_schema = if is_isolated_fork { + a.storage.path = fork_data_path(&a.storage.path, ancestor_id); + Some(fork_ducklake_metadata_schema(ancestor_id, name)) + } else { + // Root / shared ancestors live in their config's OWN catalog namespace — + // which may be a non-default schema when one catalog database hosts + // several lakes (`extra_args METADATA_SCHEMA '…'`). Preserve it, or the + // read-only attach would bind the wrong (or a nonexistent) lake. + a.extra_args + .as_deref() + .and_then(extract_metadata_schema_arg) + }; + let extra_args = a + .extra_args + .as_deref() + .map(strip_fork_reserved_attach_args) + .filter(|s| !s.is_empty()); + ancestors.push(DucklakeAncestorAttach { + workspace_id: ancestor_id.clone(), + alias: fork_ducklake_ancestor_alias(name, ancestor_id), + catalog: a.catalog, + catalog_resource: a.catalog_resource, + storage: a.storage, + metadata_schema, + extra_args, + }); + } + Err(e) => { + tracing::warn!( + "fork {w_id}: ducklake {name} not resolvable in ancestor {ancestor_id} \ + ({e:#}); read-defer disabled, fork namespace still isolated" + ); + ancestors.clear(); + break; + } + } + } + // Drop fork ancestors whose namespace was never bootstrapped (e.g. a fresh intermediate + // fork that never attached this lake): their READ_ONLY attach would fail the whole job + // ("DuckLake does not exist" + creation disabled), and a nonexistent namespace can't own + // tables or be referenced by any persisted view. Checked against the fork's catalog DB — + // must happen BEFORE defer discovery so `ancestor_idx` values index the filtered list. + let (existing_schemas, fork_views, fork_tables) = + inspect_fork_catalog(&base, &metadata_schema, &ancestors, db).await?; + ancestors.retain(|a| { + a.metadata_schema + .as_ref() + .map_or(true, |s| existing_schemas.contains(s)) + }); + let defer_tables = if ancestors.is_empty() { + vec![] + } else { + // Discovery walks the WHOLE chain: a table only a grandparent materialized has no + // physical copy in the direct parent (it defers too), so its view must target the + // nearest ancestor that owns one. + let ancestor_ids: Vec = ancestors.iter().map(|a| a.workspace_id.clone()).collect(); + let mut tables = + crate::materialization::list_fork_defer_tables(db, &ancestor_ids, w_id, name).await?; + // Catalog truth beats recorded status: any table live in the fork namespace is + // fork-owned, whatever its materialized_partition row says (failed-after-commit, + // raw SQL) — a defer view over it would silently yield to it. + tables.retain(|t| !fork_tables.contains(&t.table)); + tables + }; + + base.storage.path = fork_path; + base.extra_args = Some(match base.extra_args.take() { + Some(e) => { + let sanitized = strip_fork_reserved_attach_args(&e); + if sanitized.is_empty() { + format!("METADATA_SCHEMA '{metadata_schema}'") + } else { + format!("{sanitized}, METADATA_SCHEMA '{metadata_schema}'") + } + } + None => format!("METADATA_SCHEMA '{metadata_schema}'"), + }); + base.fork_defer = Some(DucklakeForkDefer { ancestors, defer_tables, fork_views }); + Ok(base) +} + +/// One round trip to the fork's catalog DB for both namespace introspections: which fork +/// ancestors' metadata schemas actually exist (a fresh intermediate fork may never have +/// bootstrapped its namespace), and the views currently live in THIS fork's namespace +/// (straight from DuckLake's `ducklake_view` metadata — missing metadata tables read as "no +/// views", correct since nothing can exist there). Each ancestor's schema is checked in the +/// ancestor's OWN catalog database — a fork whose catalog drifted away from an ancestor's +/// must not misread that ancestor's (existing, elsewhere) namespace as missing. Ancestors +/// whose catalog is unreachable read as missing — the safe direction: their defer is skipped +/// (loud absent-table reads) rather than emitting a READ_ONLY attach that would fail every +/// job in this fork. +async fn inspect_fork_catalog( + base: &DucklakeWithConnData, + metadata_schema: &str, + ancestors: &[DucklakeAncestorAttach], + db: &DB, +) -> Result<( + std::collections::HashSet, + Vec, + std::collections::HashSet, +)> { + // Ancestor fork-schemas grouped by which catalog database they live in. + let mut groups: std::collections::HashMap)> = + std::collections::HashMap::new(); + for a in ancestors { + if let Some(ms) = &a.metadata_schema { + groups + .entry(ducklake_catalog_identity(&a.catalog)) + .or_insert_with(|| (a.catalog_resource.clone(), vec![])) + .1 + .push(ms.clone()); + } + } + + async fn query_schemas( + client: &tokio_postgres::Client, + schemas: Vec, + ) -> std::result::Result, tokio_postgres::Error> { + client + .query( + "SELECT schema_name::text FROM information_schema.schemata + WHERE schema_name = ANY($1)", + &[&schemas], + ) + .await + .map(|rows| rows.iter().map(|r| r.get::<_, String>(0)).collect()) + } + + let mut existing_schemas: std::collections::HashSet = Default::default(); + + // The fork's own catalog: same-catalog ancestor schemas + this fork's live views. + let pg: crate::PgDatabase = + serde_json::from_value(base.catalog_resource.clone()).map_err(|e| { + Error::internal_err(format!("ducklake catalog resource is not postgres: {e}")) + })?; + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let same_catalog = groups.remove(&ducklake_catalog_identity(&base.catalog)); + let same_catalog_res = match same_catalog { + Some((_, schemas)) => query_schemas(&client, schemas).await.map(Some), + None => Ok(None), + }; + // Identifier-quoted schema: the name is server-derived (mangled + hashed) but quote anyway. + let q = format!( + r#"SELECT CASE WHEN s.schema_name = 'main' THEN v.view_name + ELSE s.schema_name || '.' || v.view_name END AS name + FROM "{ms}".ducklake_view v + JOIN "{ms}".ducklake_schema s ON s.schema_id = v.schema_id AND s.end_snapshot IS NULL + WHERE v.end_snapshot IS NULL"#, + ms = metadata_schema.replace('"', "\"\"") + ); + let view_rows = client.query(&q, &[]).await; + // Live TABLES in the fork's namespace, same name shape as the views — the defer list is + // filtered against them: `CREATE VIEW IF NOT EXISTS` silently yields to an existing + // table, so emitting a defer view over one would leave reads on the fork table while + // claiming they defer (recorded status can't tell — a committed write whose data tests + // failed, or a table left by raw SQL, has no `materialized` row). + let qt = format!( + r#"SELECT CASE WHEN s.schema_name = 'main' THEN t.table_name + ELSE s.schema_name || '.' || t.table_name END AS name + FROM "{ms}".ducklake_table t + JOIN "{ms}".ducklake_schema s ON s.schema_id = t.schema_id AND s.end_snapshot IS NULL + WHERE t.end_snapshot IS NULL"#, + ms = metadata_schema.replace('"', "\"\"") + ); + let table_rows = client.query(&qt, &[]).await; + drop(client); + let _ = join_handle.await; + + existing_schemas.extend( + same_catalog_res + .map_err(|e| { + Error::internal_err(format!("checking fork ducklake ancestor schemas: {e}")) + })? + .unwrap_or_default(), + ); + let fork_views = match view_rows { + Ok(rows) => rows.iter().map(|r| r.get::<_, String>(0)).collect(), + // 42P01 undefined_table / 3F000 invalid_schema_name: namespace not bootstrapped yet. + Err(e) + if e.code().map_or(false, |c| { + c == &tokio_postgres::error::SqlState::UNDEFINED_TABLE + || c == &tokio_postgres::error::SqlState::INVALID_SCHEMA_NAME + }) => + { + vec![] + } + Err(e) => { + return Err(Error::internal_err(format!( + "listing fork ducklake views in {metadata_schema}: {e}" + ))) + } + }; + let fork_tables = match table_rows { + Ok(rows) => rows.iter().map(|r| r.get::<_, String>(0)).collect(), + Err(e) + if e.code().map_or(false, |c| { + c == &tokio_postgres::error::SqlState::UNDEFINED_TABLE + || c == &tokio_postgres::error::SqlState::INVALID_SCHEMA_NAME + }) => + { + Default::default() + } + Err(e) => { + return Err(Error::internal_err(format!( + "listing fork ducklake tables in {metadata_schema}: {e}" + ))) + } + }; + + // Ancestors living in OTHER catalog databases (this fork's catalog drifted after forking): + // one connection per distinct catalog, best-effort — an unreachable ancestor catalog only + // disables that ancestor's defer. + for (identity, (resource, schemas)) in groups { + let checked = async { + let pg: crate::PgDatabase = serde_json::from_value(resource) + .map_err(|e| Error::internal_err(format!("not a postgres resource: {e}")))?; + let (client, connection) = pg.connect(Some(db)).await?; + let join_handle = tokio::spawn(async move { connection.await }); + let res = query_schemas(&client, schemas).await; + drop(client); + let _ = join_handle.await; + res.map_err(|e| Error::internal_err(format!("{e}"))) + } + .await; + match checked { + Ok(found) => existing_schemas.extend(found), + Err(e) => { + tracing::warn!( + "fork ancestor catalog `{identity}` unreachable while checking ducklake \ + namespaces ({e:#}); its ancestors' defer is disabled for this resolution" + ); + } + } + } + Ok((existing_schemas, fork_views, fork_tables)) +} + +/// Canonical identity of a lake's catalog database (`:`) — used +/// for the cleanup registry and for grouping ancestors by which catalog their namespace lives +/// in. Identifies the database *pointer*, not its (live-resolved) credentials. +pub fn ducklake_catalog_identity(catalog: &DucklakeCatalog) -> String { + format!( + "{}:{}", + catalog.resource_type.as_ref(), + catalog.resource_path + ) +} + +/// Record that a fork attached this lake at this physical location, so fork deletion knows +/// exactly which pg metadata schema and which storage prefixes to clean up. One row per +/// (lake, storage, data path) EVER attached — if the fork's lake settings drift, later +/// attaches add rows rather than replace them, so cleanup covers every prefix the fork wrote. +/// The once-cache is keyed on the full location for the same reason. Re-registering an +/// existing row resets its `schema_dropped` cleanup phase: attaching recreates the metadata +/// schema, so a stale "already dropped" marker would make the eventual cleanup skip a live +/// schema. +async fn register_fork_ducklake_namespace( + db: &DB, + w_id: &str, + name: &str, + metadata_schema: &str, + catalog: &str, + storage: Option<&str>, + data_path: &str, +) -> Result<()> { + // '' = default storage (the column is part of the PK, which cannot hold NULL). + let storage = storage.unwrap_or(""); + // Resolve the logical storage name to its identity NOW: cleanup must delete from the + // storage that was active when the data was written, not whatever the name points at by + // deletion time. '' = unresolvable (no LFS configured — the write itself will fail at the + // proxy, so nothing lands anywhere). + let storage_ref = fork_storage_ref(db, w_id, storage) + .await + .unwrap_or_default(); + let location = format!("{name}\0{catalog}\0{storage}\0{storage_ref}\0{data_path}"); + let now = chrono::Utc::now().timestamp(); + if FORK_DUCKLAKE_REGISTERED + .get(w_id) + .is_some_and(|(locations, exp)| exp > now && locations.contains(&location)) + { + return Ok(()); + } + sqlx::query!( + "INSERT INTO fork_ducklake_namespace + (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (workspace_id, ducklake_name, catalog, storage, storage_ref, data_path) + DO UPDATE SET schema_dropped = false", + w_id, + name, + metadata_schema, + catalog, + storage, + &storage_ref, + data_path, + ) + .execute(db) + .await + .map_err(|e| Error::internal_err(format!("registering fork ducklake namespace: {e:#}")))?; + let mut locations = FORK_DUCKLAKE_REGISTERED + .get(w_id) + .filter(|(_, exp)| *exp > now) + .map(|(locations, _)| locations) + .unwrap_or_default(); + locations.insert(location); + FORK_DUCKLAKE_REGISTERED.insert(w_id.to_string(), (locations, now + 60)); + Ok(()) +} + +/// Canonical identity of a workspace storage (`:`) as +/// configured RIGHT NOW — recorded in the fork namespace registry so cleanup targets the +/// storage the data was actually written to. `storage` = '' for the primary storage, else a +/// `secondary_storage` name. Returns None when no matching storage is configured. +async fn fork_storage_ref(db: &DB, w_id: &str, storage: &str) -> Option { + let lfs_json = sqlx::query_scalar!( + "SELECT large_file_storage FROM workspace_settings WHERE workspace_id = $1", + w_id + ) + .fetch_optional(db) + .await + .ok() + .flatten() + .flatten()?; + let entry = if storage.is_empty() || storage == "_default_" { + lfs_json.clone() + } else { + lfs_json.get("secondary_storage")?.get(storage)?.clone() + }; + lfs_entry_storage_ref(&entry) +} + +/// The pure part of [`fork_storage_ref`]: one `LargeFileStorage` JSON entry → its canonical +/// `:` descriptor. The `$res:` prefix on resource paths is normalized away (stored +/// configs are inconsistent about it); cleanup re-adds it when resolving. +pub fn lfs_entry_storage_ref(entry: &serde_json::Value) -> Option { + let typ = entry.get("type")?.as_str()?; + let path_field = match typ { + "S3Storage" | "S3AwsOidc" => "s3_resource_path", + "AzureBlobStorage" | "AzureWorkloadIdentity" => "azure_blob_resource_path", + "GoogleCloudStorage" => "gcs_resource_path", + "FilesystemStorage" => "root_path", + _ => return None, + }; + let path = entry.get(path_field)?.as_str()?; + let path = path.strip_prefix("$res:").unwrap_or(path); + Some(format!("{typ}:{path}")) +} + +/// Resolve a `$res:`/`$var:` reference tree to its concrete value (recursively, secrets +/// decrypted). No permission checks — trusted server-side callers only; never echo the result +/// to a user. +pub async fn transform_json_value_unchecked( + value: &serde_json::Value, + w_id: &str, + db: &DB, +) -> Result { + transform_json_unchecked(value, w_id, db).await } // This does not check for any permission. Should never be displayed to a user. @@ -1140,4 +1821,162 @@ mod tests { let long_id = format!("wm-fork-{}", "a".repeat(43)); assert!(validate_fork_workspace_id(&long_id).is_err()); } + + #[test] + fn test_fork_ducklake_metadata_schema_shape() { + let s = fork_ducklake_metadata_schema("wm-fork-my-feature-42", "main"); + assert!(s.starts_with(FORK_DUCKLAKE_SCHEMA_PREFIX), "{s}"); + assert!(s.len() <= 63, "pg schema name limit: {s}"); + assert!( + s.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'), + "{s}" + ); + // Deterministic (persisted view SQL / registry rows depend on it). + assert_eq!( + s, + fork_ducklake_metadata_schema("wm-fork-my-feature-42", "main") + ); + } + + #[test] + fn test_fork_ducklake_metadata_schema_injective_after_mangling() { + // `-` and `_` mangle to the same char; the hash suffix must keep them distinct. + let a = fork_ducklake_metadata_schema("wm-fork-a-b", "main"); + let b = fork_ducklake_metadata_schema("wm-fork-a_b", "main"); + assert_ne!(a, b); + // Lake-scoped: two lakes of one workspace may share a catalog database, so their fork + // namespaces must be distinct schemas. + assert_ne!( + fork_ducklake_metadata_schema("wm-fork-a-b", "lake_a"), + fork_ducklake_metadata_schema("wm-fork-a-b", "lake_b") + ); + // Long ids truncate to the same mangled prefix; hash must still differ. + let long_a = fork_ducklake_metadata_schema(&format!("wm-fork-{}x", "a".repeat(40)), "main"); + let long_b = fork_ducklake_metadata_schema(&format!("wm-fork-{}y", "a".repeat(40)), "main"); + assert_ne!(long_a, long_b); + let long_lake = + fork_ducklake_metadata_schema(&format!("wm-fork-{}", "a".repeat(42)), &"l".repeat(40)); + assert!(long_a.len() <= 63 && long_b.len() <= 63 && long_lake.len() <= 63); + } + + #[test] + fn test_fork_ducklake_ancestor_alias_valid_identifier() { + let a = fork_ducklake_ancestor_alias("analytics", "wm-fork-dev-1"); + assert!(a.starts_with("__wm_dl_"), "{a}"); + assert!( + a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_'), + "{a}" + ); + // Distinct per lake for the same ancestor (a script can attach several lakes). + assert_ne!(a, fork_ducklake_ancestor_alias("staging", "wm-fork-dev-1")); + assert_ne!( + a, + fork_ducklake_ancestor_alias("analytics", "wm-fork-dev-2") + ); + } + + #[test] + fn test_extract_metadata_schema_arg() { + assert_eq!( + extract_metadata_schema_arg("METADATA_SCHEMA 'lake_b_ns', ENCRYPTED true"), + Some("lake_b_ns".to_string()) + ); + assert_eq!( + extract_metadata_schema_arg("metadata_schema bare_ident"), + Some("bare_ident".to_string()) + ); + // Last occurrence wins (DuckDB duplicate-option semantics). + assert_eq!( + extract_metadata_schema_arg("METADATA_SCHEMA 'a', METADATA_SCHEMA 'b'"), + Some("b".to_string()) + ); + assert_eq!(extract_metadata_schema_arg("ENCRYPTED true"), None); + } + + #[test] + fn test_fork_data_path_prefix_isolation() { + // Fork/dev ids are git-branch-safe and may contain `/`: `wm-fork-a/b` is a valid id. + // Its data prefix must NOT nest inside `wm-fork-a`'s, or deleting `wm-fork-a` would + // sweep the sibling's files via the object-store prefix listing. + assert!(validate_fork_workspace_id("wm-fork-a/b").is_ok()); + let a = fork_data_path("lake", "wm-fork-a"); + let ab = fork_data_path("lake", "wm-fork-a/b"); + assert!( + !format!("{ab}/").starts_with(&format!( + "{FORK_DUCKLAKE_DATA_DIR}/{}/", + fork_data_dir_segment("wm-fork-a") + )), + "{ab} nests under {a}'s cleanup prefix" + ); + // Single path component: the segment itself contains no separator. + assert!(!fork_data_dir_segment("wm-fork-a/b").contains('/')); + // Injective after mangling (`/` and `_` both mangle to `_`). + assert_ne!( + fork_data_dir_segment("wm-fork-a/b"), + fork_data_dir_segment("wm-fork-a_b") + ); + // Deterministic (registry rows + cleanup guard recompute it). + assert_eq!(a, fork_data_path("lake", "wm-fork-a")); + // Empty base path still yields a well-formed prefix. + assert_eq!( + fork_data_path("", "wm-fork-a"), + format!( + "{FORK_DUCKLAKE_DATA_DIR}/{}", + fork_data_dir_segment("wm-fork-a") + ) + ); + } + + #[test] + fn test_lfs_entry_storage_ref() { + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "S3Storage", "s3_resource_path": "$res:u/admin/minio" + })), + Some("S3Storage:u/admin/minio".to_string()) + ); + // The `$res:` prefix is optional in stored configs; normalized either way. + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "AzureBlobStorage", "azure_blob_resource_path": "u/admin/az" + })), + Some("AzureBlobStorage:u/admin/az".to_string()) + ); + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({ + "type": "FilesystemStorage", "root_path": "/data/lfs" + })), + Some("FilesystemStorage:/data/lfs".to_string()) + ); + assert_eq!( + lfs_entry_storage_ref(&serde_json::json!({"type": "SomethingNew"})), + None + ); + assert_eq!(lfs_entry_storage_ref(&serde_json::json!({})), None); + } + + #[test] + fn test_strip_fork_reserved_attach_args() { + // Reserved options removed wherever they appear, others preserved. + assert_eq!( + strip_fork_reserved_attach_args("METADATA_SCHEMA 'main', ENCRYPTED true"), + "ENCRYPTED true" + ); + assert_eq!( + strip_fork_reserved_attach_args( + "ENCRYPTED true, DATA_PATH 's3://b/prod', OVERRIDE_DATA_PATH FALSE" + ), + "ENCRYPTED true" + ); + // Case-insensitive, and quoted values may contain commas/spaces. + assert_eq!( + strip_fork_reserved_attach_args("data_path 's3://b/x, y', SNAPSHOT_VERSION 3"), + "SNAPSHOT_VERSION 3" + ); + assert_eq!(strip_fork_reserved_attach_args(""), ""); + assert_eq!( + strip_fork_reserved_attach_args("METADATA_SCHEMA 'wm_fork_evil'"), + "" + ); + } } diff --git a/backend/windmill-queue/tests/schedule_push.rs b/backend/windmill-queue/tests/schedule_push.rs index 745b2685e9..e11ced23bc 100644 --- a/backend/windmill-queue/tests/schedule_push.rs +++ b/backend/windmill-queue/tests/schedule_push.rs @@ -1656,12 +1656,13 @@ mod schedule_push { tx.commit().await?; assert_eq!(count_queued_jobs(&db).await, 1); - let (kind, path, tag, raw_code) = sqlx::query_as::<_, (String, Option, String, Option)>( - "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code + let (kind, path, tag, raw_code) = + sqlx::query_as::<_, (String, Option, String, Option)>( + "SELECT j.kind::text, j.runnable_path, j.tag, j.raw_code FROM v2_job j JOIN v2_job_queue q ON j.id = q.id LIMIT 1", - ) - .fetch_one(&db) - .await?; + ) + .fetch_one(&db) + .await?; assert_eq!(kind, "preview"); assert_eq!(path.as_deref(), Some("f/ducklake_maintenance/legacy")); assert_eq!(tag, "duckdb"); @@ -1694,6 +1695,7 @@ mod schedule_push { }, storage: DucklakeStorage { storage: None, path: "legacy".to_string() }, extra_args: None, + fork_behavior: None, maintenance: Some(DucklakeMaintenance { enabled: maintenance_enabled, schedule: None, diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index 31db95b91a..817c86281c 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -14,7 +14,7 @@ use windmill_common::utils::sanitize_string_from_password; use windmill_common::worker::{get_memory, to_raw_value, Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{ get_datatable_resource_from_db_unchecked, get_ducklake_from_db_unchecked, - DucklakeCatalogResourceType, + strip_fork_reserved_attach_args, DucklakeCatalogResourceType, }; use windmill_common::PgDatabase; use windmill_object_store::S3_PROXY_LAST_ERRORS_CACHE; @@ -1155,6 +1155,7 @@ pub async fn do_duckdb( conn, &mut hidden_passwords, &job.workspace_id, + materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), ) .await? { @@ -1858,6 +1859,7 @@ async fn transform_attach_ducklake( conn: &Connection, hidden_passwords: &mut Arc>>, w_id: &str, + materialize_target: Option<&str>, ) -> Result>> { lazy_static::lazy_static! { static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'ducklake(://[^':]+)?'\s*AS\s+([^ ;]+)\s*(\([^)]*\))?").unwrap(); @@ -1867,15 +1869,28 @@ async fn transform_attach_ducklake( }; let name = cap.get(1).map(|m| &m.as_str()[3..]).unwrap_or("main"); let alias_name = cap.get(2).map(|m| m.as_str()).unwrap_or(""); - let extra_args = cap + let user_extra_args = cap .get(3) - .map(|m| format!(", {}", &m.as_str()[1..m.as_str().len() - 1])) - .unwrap_or("".to_string()); + .map(|m| m.as_str()[1..m.as_str().len() - 1].to_string()) + .unwrap_or_default(); let ducklake = match conn { Connection::Http(client) => get_ducklake_from_agent_http(client, name, w_id).await?, Connection::Sql(db) => get_ducklake_from_db_unchecked(name, w_id, db).await?, }; + // In a fork, METADATA_SCHEMA / DATA_PATH / OVERRIDE_DATA_PATH are owned by the fork + // resolution (DuckDB silently keeps the last occurrence of a duplicated option, so a + // user-supplied one would escape the fork namespace back to the parent's). + let user_extra_args = if ducklake.fork_defer.is_some() { + strip_fork_reserved_attach_args(&user_extra_args) + } else { + user_extra_args + }; + let extra_args = if user_extra_args.is_empty() { + String::new() + } else { + format!(", {}", user_extra_args) + }; let db_type = match ducklake.catalog.resource_type { DucklakeCatalogResourceType::Instance => "postgres", _ => ducklake.catalog.resource_type.as_ref(), @@ -1921,17 +1936,170 @@ async fn transform_attach_ducklake( } else { format!(", AUTOMATIC_MIGRATION TRUE{extra_args}") }; + // In a fork, re-emit the fork-owned DATA_PATH as the LAST option: DuckDB keeps the last + // occurrence of a duplicated option, so this wins over anything the arg stripping might + // not recognize (e.g. a dollar-quoted literal), regardless of literal syntax. The + // METADATA_SCHEMA injected by the fork resolution is already last within `extra_args`. + let extra_args = if ducklake.fork_defer.is_some() { + format!("{extra_args}, DATA_PATH 's3://{storage}/{data_path}'") + } else { + extra_args + }; let attach_str = format!( "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});", ); let install_db_ext_str = get_attach_db_install_str(db_type)?; - Ok(Some(vec![ + let mut statements = vec![ "INSTALL ducklake;".to_string(), install_db_ext_str.to_string(), attach_str, - ])) + ]; + if let Some(defer) = ducklake.fork_defer.as_ref() { + statements.extend(fork_defer_statements( + name, + alias_name, + defer, + materialize_target, + hidden_passwords, + )?); + } + Ok(Some(statements)) +} + +// Double-quote a possibly schema-qualified table reference, each dotted segment +// independently (local copy of sql_materialize's private helper). +fn quote_qualified_table(name: &str) -> String { + name.split('.') + .map(|id| format!("\"{}\"", id.replace('"', "\"\""))) + .collect::>() + .join(".") +} + +// `schema.table` → `schema.table_current`: the SCD2 companion view lives next to its table. +fn current_companion(table: &str) -> String { + match table.rsplit_once('.') { + Some((s, t)) => format!("{s}.{t}_current"), + None => format!("{table}_current"), + } +} + +/// Statements appended after a fork workspace's lake ATTACH: read-only attaches of the +/// ancestor namespaces plus `CREATE VIEW IF NOT EXISTS` defer views over the direct parent for +/// every table the fork has not materialized. When this job's managed materialize targets one +/// of the deferred tables, its defer view is dropped instead of created — the write must hit a +/// real fork table (`CREATE [OR REPLACE] TABLE` refuses to replace a view). +fn fork_defer_statements( + lake_name: &str, + alias_name: &str, + defer: &windmill_common::workspaces::DucklakeForkDefer, + materialize_target: Option<&str>, + hidden_passwords: &mut Arc>>, +) -> Result> { + let mut stmts = vec![]; + if defer.ancestors.is_empty() { + // Defer unavailable (an ancestor no longer defines this lake); the fork namespace + // still isolates writes. + return Ok(stmts); + } + for a in &defer.ancestors { + if let Some(pwd) = a.catalog_resource.get("password").and_then(|p| p.as_str()) { + hidden_passwords.lock().unwrap().push(pwd.to_string()); + } + let db_type = match a.catalog.resource_type { + DucklakeCatalogResourceType::Instance => "postgres", + _ => a.catalog.resource_type.as_ref(), + }; + stmts.push(get_attach_db_install_str(db_type)?.to_string()); + let conn_str = + format_attach_db_conn_str(a.catalog_resource.clone(), db_type)?.replace('\'', "''"); + let storage = a + .storage + .storage + .as_deref() + .unwrap_or(DEFAULT_STORAGE) + .replace('\'', "''"); + let data_path = a.storage.path.replace('\'', "''"); + let metadata_schema = a + .metadata_schema + .as_ref() + .map(|s| format!(", METADATA_SCHEMA '{}'", s.replace('\'', "''"))) + .unwrap_or_default(); + // The ancestor config's own non-reserved args (e.g. `ENCRYPTED true`) — an + // option-dependent lake wouldn't attach without them. Emitted FIRST: DuckDB keeps the + // last occurrence of a duplicated option, so the fork-owned DATA_PATH / READ_ONLY / + // METADATA_SCHEMA after them always win. + let extra_args = a + .extra_args + .as_ref() + .map(|e| format!("{e}, ")) + .unwrap_or_default(); + // READ_ONLY: a fork job must never write an ancestor namespace. No AUTOMATIC_MIGRATION + // / CREATE_IF_NOT_EXISTS: an ancestor lake that would need creating or migrating fails + // loudly rather than being mutated from a fork. IF NOT EXISTS: several ATTACH blocks in + // one script (e.g. the user's + the materialize synthetic one) emit the same ancestors. + stmts.push(format!( + "ATTACH IF NOT EXISTS 'ducklake:{db_type}:{conn_str}' AS {} ({extra_args}DATA_PATH 's3://{storage}/{data_path}', OVERRIDE_DATA_PATH TRUE, READ_ONLY{metadata_schema});", + a.alias + )); + } + let parent_alias = &defer.ancestors[0].alias; + let target_table = materialize_target.and_then(|ap| { + let (l, t) = ap.split_once('/')?; + (l == lake_name).then_some(t) + }); + let mut created_schemas = std::collections::HashSet::new(); + for dt in &defer.defer_tables { + if Some(dt.table.as_str()) == target_table { + continue; + } + // Each view targets the NEAREST ancestor that physically owns the table (a direct + // parent that itself defers has no copy to bind against). Out-of-range index (never + // produced by the resolver, but the field crosses the agent wire) falls back to the + // direct parent rather than panicking. + let owner_alias = defer + .ancestors + .get(dt.ancestor_idx as usize) + .map(|a| a.alias.as_str()) + .unwrap_or(parent_alias); + if let Some((schema, _)) = dt.table.rsplit_once('.') { + if created_schemas.insert(schema) { + stmts.push(format!( + "CREATE SCHEMA IF NOT EXISTS {alias_name}.{};", + quote_qualified_table(schema) + )); + } + } + let q = quote_qualified_table(&dt.table); + stmts.push(format!( + "CREATE VIEW IF NOT EXISTS {alias_name}.{q} AS SELECT * FROM {owner_alias}.{q};" + )); + if dt.with_current_view { + let qc = quote_qualified_table(¤t_companion(&dt.table)); + stmts.push(format!( + "CREATE VIEW IF NOT EXISTS {alias_name}.{qc} AS SELECT * FROM {owner_alias}.{qc};" + )); + } + } + if let Some(t) = target_table { + // View→table transition: replace the target's defer view(s) with the real table this + // job writes (`CREATE [OR REPLACE] TABLE` refuses to replace a view). Keyed on the + // catalog's ACTUAL live views — not on recorded materialization status, which after a + // failed run can't tell a defer view from a real table, and a mismatched DROP VIEW + // would wedge the asset. `_current` is dropped too when it is a view — SCD2 codegen + // recreates it with `IF NOT EXISTS`, which would otherwise silently keep a view over + // the parent. + for name in [t.to_string(), current_companion(t)] { + if defer.fork_views.iter().any(|v| v == &name) { + stmts.push(format!( + "DROP VIEW IF EXISTS {alias_name}.{};", + quote_qualified_table(&name) + )); + } + } + } + Ok(stmts) } async fn transform_attach_datatable( @@ -2036,6 +2204,198 @@ pub struct Arg { mod tests { use super::*; + fn test_ancestor(wid: &str) -> windmill_common::workspaces::DucklakeAncestorAttach { + windmill_common::workspaces::DucklakeAncestorAttach { + workspace_id: wid.to_string(), + alias: format!("__wm_dl_lake_{}_abcd1234", wid.replace('-', "_")), + catalog: windmill_common::workspaces::DucklakeCatalog { + resource_type: windmill_common::workspaces::DucklakeCatalogResourceType::Instance, + resource_path: "cat_db".to_string(), + }, + catalog_resource: serde_json::json!({ + "host": "h", "dbname": "cat_db", "user": "u", "password": "pw" + }), + storage: windmill_common::workspaces::DucklakeStorage { + storage: None, + path: "lake".to_string(), + }, + metadata_schema: None, + extra_args: None, + } + } + + fn test_fork_defer_chain( + ancestor_wids: Vec<&str>, + defer_tables: Vec<(&str, bool, u32)>, + fork_views: Vec<&str>, + ) -> windmill_common::workspaces::DucklakeForkDefer { + windmill_common::workspaces::DucklakeForkDefer { + ancestors: ancestor_wids.into_iter().map(test_ancestor).collect(), + defer_tables: defer_tables + .into_iter() + .map( + |(t, cur, idx)| windmill_common::materialization::ForkDeferTable { + table: t.to_string(), + with_current_view: cur, + ancestor_idx: idx, + }, + ) + .collect(), + fork_views: fork_views.into_iter().map(str::to_string).collect(), + } + } + + fn test_fork_defer( + defer_tables: Vec<(&str, bool)>, + fork_views: Vec<&str>, + ) -> windmill_common::workspaces::DucklakeForkDefer { + test_fork_defer_chain( + vec!["parent-ws"], + defer_tables.into_iter().map(|(t, c)| (t, c, 0)).collect(), + fork_views, + ) + } + + #[test] + fn test_fork_defer_statements_ancestor_extra_args() { + // The ancestor config's own args (e.g. ENCRYPTED) must ride on its read-only attach — + // BEFORE the fork-owned options, so DuckDB's last-occurrence-wins keeps DATA_PATH / + // READ_ONLY / METADATA_SCHEMA authoritative even if the args tried to override them. + let mut defer = test_fork_defer(vec![("orders", false)], vec![]); + defer.ancestors[0].extra_args = Some("ENCRYPTED true".to_string()); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let attach = stmts + .iter() + .find(|s| s.starts_with("ATTACH IF NOT EXISTS")) + .unwrap(); + assert!(attach.contains("(ENCRYPTED true, DATA_PATH "), "{attach}"); + assert!( + attach.find("ENCRYPTED true").unwrap() < attach.find("READ_ONLY").unwrap(), + "{attach}" + ); + } + + #[test] + fn test_fork_defer_statements_chained_ancestors() { + // fork → parent → root: `orders` was only materialized in the root (idx 1) — its view + // must target the ROOT alias (the parent has no physical copy); `daily` owned by the + // direct parent (idx 0) targets the parent alias. Out-of-range idx falls back to the + // direct parent instead of panicking. + let defer = test_fork_defer_chain( + vec!["parent-ws", "root-ws"], + vec![("orders", false, 1), ("daily", false, 0), ("oob", false, 9)], + vec![], + ); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"orders\" AS SELECT * FROM __wm_dl_lake_root_ws_abcd1234.\"orders\"" + ), + "{joined}" + ); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"daily\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"daily\"" + ), + "{joined}" + ); + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"oob\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"oob\"" + ), + "{joined}" + ); + // Both ancestors attached read-only. + assert_eq!( + joined.matches("ATTACH IF NOT EXISTS").count(), + 2, + "{joined}" + ); + } + + #[test] + fn test_fork_defer_statements_shape() { + let defer = test_fork_defer(vec![("orders", false), ("dim", true)], vec![]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + // Ancestor attach: read-only, idempotent, never auto-migrating or auto-creating. + assert!(joined.contains("ATTACH IF NOT EXISTS"), "{joined}"); + assert!(joined.contains("READ_ONLY"), "{joined}"); + assert!(!joined.contains("AUTOMATIC_MIGRATION"), "{joined}"); + assert!(!joined.contains("CREATE_IF_NOT_EXISTS"), "{joined}"); + // Ancestor catalog password is hidden from logs. + assert_eq!(hp.lock().unwrap().as_slice(), ["pw"]); + // Defer views over the parent alias, plus the SCD2 `_current` companion. + assert!( + joined.contains( + "CREATE VIEW IF NOT EXISTS dl.\"orders\" AS SELECT * FROM __wm_dl_lake_parent_ws_abcd1234.\"orders\"" + ), + "{joined}" + ); + assert!(joined.contains("dl.\"dim_current\""), "{joined}"); + // No target → no transition drops. + assert!(!joined.contains("DROP VIEW"), "{joined}"); + } + + #[test] + fn test_fork_defer_statements_target_transition() { + // Target currently a defer view → skip its CREATE, drop the view (+ companion). + let defer = test_fork_defer(vec![("orders", false)], vec!["orders", "orders_current"]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = + fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) + .unwrap(); + let joined = stmts.join("\n"); + assert!(!joined.contains("CREATE VIEW"), "{joined}"); + assert!( + joined.contains("DROP VIEW IF EXISTS _wm_target.\"orders\";"), + "{joined}" + ); + assert!( + joined.contains("DROP VIEW IF EXISTS _wm_target.\"orders_current\";"), + "{joined}" + ); + + // Target already a real table (NOT in fork_views, e.g. after a failed re-run whose + // status can't be trusted) → no DROP VIEW, or the job would wedge on a type mismatch. + let defer = test_fork_defer(vec![("orders", false)], vec![]); + let stmts = + fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp) + .unwrap(); + assert!(!stmts.join("\n").contains("DROP VIEW"), "{stmts:?}"); + + // Target in a different lake → this lake's defer views are untouched. + let defer = test_fork_defer(vec![("orders", false)], vec!["orders"]); + let stmts = + fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains("CREATE VIEW IF NOT EXISTS dl.\"orders\""), + "{joined}" + ); + assert!(!joined.contains("DROP VIEW"), "{joined}"); + } + + #[test] + fn test_fork_defer_statements_schema_qualified() { + let defer = test_fork_defer(vec![("staging.raw", false)], vec![]); + let mut hp = Arc::new(Mutex::new(vec![])); + let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap(); + let joined = stmts.join("\n"); + assert!( + joined.contains("CREATE SCHEMA IF NOT EXISTS dl.\"staging\";"), + "{joined}" + ); + assert!( + joined.contains("CREATE VIEW IF NOT EXISTS dl.\"staging\".\"raw\""), + "{joined}" + ); + } + fn mrow(name: &str, params: &str, body: &str, is_table: bool, provider: &str) -> MacroRow { MacroRow { name: name.to_string(), diff --git a/docs/pipeline-env-isolation.md b/docs/pipeline-env-isolation.md new file mode 100644 index 0000000000..2b6b87ba68 --- /dev/null +++ b/docs/pipeline-env-isolation.md @@ -0,0 +1,219 @@ +# Fork data environments for pipeline materialization ("dev data") + +Status: **implemented** (branch `pipeline-env-isolation`). Companion to +[`ducklake-materialization.md`](./ducklake-materialization.md). + +## The problem + +Workspace forks fork the CODE, but `// materialize ducklake:///
` targets are +absolute pointers into shared physical storage. A fork clones `workspace_settings.ducklake` +verbatim, so `transform_attach_ducklake` resolved a fork's `ducklake://main` to the **parent's +catalog database and S3 data path** — deploying or running a materializing pipeline inside a +fork wrote the production tables. dbt answers this with target schemas + `--defer`; datatables +already have a fork answer (`forked_datatables` → `wm_fork_*` databases + a drop endpoint); +ducklake had none. `wmill pipeline run --local` only covers preview runs, and previews still +resolve targets against the real workspace. + +## The design + +Everything hangs off two DuckLake ATTACH capabilities (verified in a spike, DuckDB 1.5.4 + +current ducklake extension): `METADATA_SCHEMA` (the catalog's metadata tables live in a chosen +pg schema of the same catalog DB — auto-created on first attach; default is `public`) and +`READ_ONLY`, plus catalog-stored views that may reference other attached catalogs and rebind +per session. + +### Fork-time choice: isolated (default) vs shared, per lake + +The fork-creation dialog offers a per-lake data-environment choice (the section renders only +when the parent workspace has lakes configured, mirroring the datatable section): + +- **Isolated** (default — also what unlisted lakes and API callers that omit the field get) — + the full behavior described below. +- **Shared** — explicit opt-out: the fork reads and writes the parent's lake directly through + its cloned config, like a non-fork workspace (e.g. a fork meant to run prod-equivalent + backfills). Stamped as `fork_behavior: "shared"` on the lake entry in the fork's own + `workspace_settings.ducklake` at creation (`shared_ducklakes` on the create-fork request); + the resolver, the graph indicator, and (trivially — nothing registers) cleanup all key off + it. Shared also lifts the mysql-catalog restriction, since nothing needs namespacing. + +Absent-field-means-isolated keeps every pre-existing fork and API caller on the safe path. +Sharing is a per-fork-creation choice, never inherited: the settings clone copies the source's +config verbatim, so fork creation strips any cloned `fork_behavior` stamps before applying its +own `shared_ducklakes` list — a fork of a shared fork defaults back to isolated. A +shared fork ancestor in a fork chain resolves like a root (its data lives at its config's +default location). Flipping a fork's lake to `shared` later (settings edit) orphans any +namespace it already materialized until fork deletion cleans it — the registry row survives +the flip. + +### Write redirect — one seam, no grammar changes + +`get_ducklake_from_db_unchecked` (windmill-common/src/workspaces.rs) is fork-aware: when the +resolved workspace has a `parent_workspace_id` (throwaway `wm-fork-*` fork or standing dev +workspace), the returned `DucklakeWithConnData` is namespaced **in its existing fields**: + +- `extra_args` += `METADATA_SCHEMA 'wm_fork___'` — + injective, ≤63 chars, stable across releases (persisted state depends on it), and + **lake-scoped**: two lakes of one workspace may share a catalog database, and a + per-workspace schema would merge their namespaces in the fork; +- `storage.path` = `__wm_forks//` — the segment is the + mangled+hashed workspace id collapsed to ONE path component (fork ids are only + git-branch-safe and may contain `/`; used raw, `wm-fork-a/b` would nest inside sibling + `wm-fork-a`'s cleanup prefix) — a **bucket-root prefix**, not a + sub-path of the parent's DATA_PATH, because lake maintenance + (`ducklake_delete_orphaned_files`, snapshot expiry) scans everything under the parent's path + and would delete live fork files nested there. + +Folding the redirect into existing fields means the EE agent-worker endpoint (which just +serializes this struct) needs **no changes**, and an agent worker running an older binary still +writes the fork namespace — it only misses the defer views (reads fail loudly; never a prod +write). Every ducklake access funnels through the same ATTACH transform — the managed +materialize's synthetic `_wm_target` attach, `materialize manual`, raw user SQL, and all UI +preview / History / backfill surfaces (they are `WM_INTERNAL_DB_*` marker preview jobs) — so a +fork cannot reach the parent namespace through the managed path at all. + +Hardening: in fork mode, user- and settings-supplied `METADATA_SCHEMA` / `DATA_PATH` / +`OVERRIDE_DATA_PATH` ATTACH options are stripped before injection (DuckDB silently keeps the +*last* occurrence of a duplicated option, so a duplicate would escape the namespace). +MySQL-catalog lakes error loudly in forks (no pg schemas to namespace with) rather than ever +writing the shared catalog. + +### Read-defer — iterate on one node without rebuilding upstream + +In a fork, `transform_attach_ducklake` additionally emits: + +1. `ATTACH IF NOT EXISTS … AS __wm_dl___ ([, ] + DATA_PATH , OVERRIDE_DATA_PATH TRUE, READ_ONLY[, METADATA_SCHEMA + ])` per fork-chain ancestor, resolved from the **ancestor's own settings** + (fork-side settings drift can't repoint what "parent" means). The ancestor config's + non-reserved `extra_args` (e.g. `ENCRYPTED true`) ride along — FIRST, so the fork-owned + options win under DuckDB's last-occurrence-wins. No + `AUTOMATIC_MIGRATION`/`CREATE_IF_NOT_EXISTS` — a fork job must never mutate an ancestor + lake. `READ_ONLY` makes prod writes *physically impossible* through ducklake, including in + `materialize manual` and raw SQL. Fork ancestors whose namespace was never bootstrapped are + skipped, with existence checked in each ancestor's OWN catalog database (a fork whose + catalog drifted away from an ancestor's must not misread that ancestor's namespace as + missing); an unreachable ancestor catalog only disables that ancestor's defer. +2. `CREATE VIEW IF NOT EXISTS .
AS SELECT * FROM .
` for every **deferred table**: (`materialized_partition` rows with + `status = 'materialized'` anywhere in the ancestor chain) MINUS (the fork's own rows). Each + view targets the NEAREST ancestor that physically owns the table + (`ForkDeferTable.ancestor_idx`): in a `fork → parent → root` chain where only the root + materialized a table, the parent has no copy (it defers too), so a view over the parent + would not bind. SCD2 owners (detected by the `is_current` column in the owning ancestor's + latest captured `materialized_asset_schema`) also get the `
_current` companion view. + So a consumer reading `dl.orders` transparently reads the nearest owning ancestor's current + data until the fork materializes `orders` itself. +3. When the running job's managed materialize targets a table that currently exists **as a + view** in the fork namespace, the view (and its `_current` companion) is dropped after the + target attach — the first fork materialize replaces the defer view with a real fork table + (`CREATE [OR REPLACE] TABLE` refuses to replace a view — verified — so without the drop the + transition fails; with it, `sql_materialize.rs` codegen needs no fork awareness at all). + The is-it-a-view decision keys on the catalog's ACTUAL live views (`ducklake_view`, read at + resolution time into `fork_defer.fork_views`), **not** on recorded materialization status: + after a failed run the status cannot distinguish a defer view from a real table, and a + mismatched `DROP VIEW` (or skipped drop) would wedge the asset on every retry — found live + in e2e when a storage failure flipped a fork asset's status to `Failed`. + +The graph shows the state per ducklake asset in a fork: an amber ↗ chip = *deferred* (some ancestor in the chain +owns the table), +an emerald fork chip = *fork-materialized* (`fork_materialization` on the `/assets/graph` +response, computed from fork + parent `materialized_partition`; the parent's rows are read on +the plain pool since fork membership doesn't imply parent membership). The details pane shows +the equivalent banner. + +### Lifecycle & cleanup + +A sidecar registry `fork_ducklake_namespace(workspace_id, ducklake_name, +metadata_schema, catalog, storage, storage_ref, data_path)` is upserted (behind a TTL'd in-process cache, keyed +per workspace and invalidated whenever a workspace's registry rows are cleaned up — a same-id fork recreated +within the TTL must re-register or its namespace would be orphaned at its own deletion) on fork +resolution, **one row per physical location ever attached** — its PK includes +`(catalog, storage, storage_ref, data_path)`, so if the fork's lake settings drift, later attaches add rows rather +than replace them and cleanup covers every location the fork wrote, not just the first — connecting to the +REGISTERED catalog identity and deleting files from the REGISTERED storage identity +(`storage_ref`, resolved from the logical storage name at attach time), never whatever the +fork's settings point at by deletion time. Explicit +`GRANT`s ship in the same migration; the registry is not cloned into sub-forks. + +The registry has deliberately **no FK to `workspace`**: rows are the durable cleanup ledger and +outlive the workspace row when physical cleanup fails after the delete commits (unreachable +catalog, storage outage) — each row is deleted only after its schema + data cleanup succeed. +Fork **creation** retries any leftover rows for the reused id and refuses to proceed while a +metadata schema still cannot be dropped, so a recreated same-id fork can never silently +reattach stale tables. Data-file leftovers alone don't block: a deleted fork's `$res:` storage +resource is gone forever, but once the schema is dropped the files are inert — the surviving +row keeps them tracked and the next successful cleanup of the same prefix (typically the +recreated fork's own deletion, with live credentials) sweeps them. + +Retries after the fork is gone get two aids: the row's `schema_dropped` phase flag (set when +the schema dropped but data cleanup failed; retries then skip the schema phase and need no +catalog credentials — registration resets it on re-attach, which recreates the schema), and +`$res:` fallback resolution against the workspace being forked (the deleted fork's resources +were clones of a parent's, so the new parent is the natural credential donor). + +`POST /w/{fork}/workspaces/drop_forked_ducklake_namespaces` (same permissions as +`delete_workspace` via a shared gate: fork owner or superadmin, plus parent-prod-admin when the +fork is an attached dev workspace) drops each registered pg metadata schema +(`DROP SCHEMA … CASCADE` on the catalog connection, hard-guarded on the `wm_fork_` prefix) and +deletes the `__wm_forks//…` objects in the workspace storage (hard-guarded on that +prefix; `parquet` feature). The fork-deletion UI calls it next to +`drop_forked_datatable_databases`, for the fork and for deleted child forks. Registry rows are +deleted per-lake only after both cleanups succeed, so a partial failure is retryable. + +## Semantics & caveats (by design) + +- **Orphaned forks stay isolated.** `parent_workspace_id` is `ON DELETE SET NULL`, so a + `wm-fork-*` workspace can outlive its parent. Its ancestor chain is then empty, but its cloned + config still points at the shared lake — so fork resolution keys on the `wm-fork-` prefix as + well as the chain (mirroring `workspace_is_fork`). An orphaned fork gets the write redirect, + registration and cleanup but no defer views; pre-orphaning defer views fail loudly at bind and + flip to real tables on first materialize. The same rule applies to a fork's *ancestors*: a + last-in-chain `wm-fork-*` ancestor is an orphaned fork (not a root) and its READ_ONLY attach + targets its fork namespace, not its cloned base config. Prefix-less dev workspaces cannot be + orphaned (deleting their prod is blocked while attached). +- **Defer is live, not snapshot-pinned.** A deferred read sees the parent's *current* table — + the same trade dbt `--defer` makes. Snapshot-pinned defer could later ride on the recorded + `snapshot_id`s. +- **Defer granularity is per-table.** Materializing one partition in the fork makes the fork's + table authoritative for *all* partitions (the rest are empty until backfilled in the fork). +- **Incremental strategies start empty in a fork.** `merge`/`append`/SCD2 create a fresh table + on first fork run — no parent rows are inherited, and fork SCD2 history diverges from the + parent's. Copy-on-fork seeding would be a follow-up opt-in. +- **Only tracked tables defer.** A parent table with no `materialized_partition` row (raw SQL + writes outside `// materialize`) or whose latest run `Failed` is not deferred — defer views + bind at CREATE against the parent catalog, and a view on a missing table would fail every + unrelated fork job. Such tables read as absent in the fork. +- **Transient race.** Between a fork materialize's commit and its state recording, a concurrent + fork job may still emit `CREATE VIEW IF NOT EXISTS` for that table — which silently yields to + an existing table (verified), so the race is a no-op. +- **Accident prevention, not a security boundary.** A fork holds the parent's cloned + credentials by design; a determined fork user can still `ATTACH 'postgres:…'` with the cloned + catalog resource or `COPY TO 's3://…'` directly. The isolation guarantees are scoped to the + managed ducklake path (where they are physical: READ_ONLY parent attach + separate metadata + schema + separate data prefix). +- **Fork chains** compose: each ancestor namespace is attached under a deterministic alias, and + a parent's own defer views (referencing *its* parent's alias) rebind correctly in the + grandchild's session because the whole chain is attached. +- **Datatables are out of scope** — the existing `forked_datatables` chooser at fork creation + covers them (`keep_original` remains their default, which shares the parent DB; picking + `schema_only` there is the datatable analog of this feature). + +## Key files + +- `backend/windmill-common/src/workspaces.rs` — fork resolution (`fork_scoped_ducklake`), + `fork_ancestor_chain`, naming (`fork_ducklake_metadata_schema`, + `fork_ducklake_ancestor_alias`, `fork_data_path`), option stripping, registry. +- `backend/windmill-common/src/materialization.rs` — `list_fork_defer_tables`. +- `backend/windmill-worker/src/duckdb_executor.rs` — `fork_defer_statements` (ancestor + attaches, defer views, target transition), user-arg stripping. +- `backend/windmill-api-assets/src/lib.rs` — `fork_materialization` on the graph. +- `backend/windmill-api-workspaces/src/workspaces_extra.rs` — + `drop_forked_ducklake_namespaces`. +- `backend/windmill-api-workspaces/src/workspaces.rs` — `clone_asset_usages_and_triggers`: + `asset` + `script_trigger` rows are deploy-derived and were never cloned into forks, so a + fork's pipeline graph had no asset nodes/edges and its dispatch cascade never fired (fixed + here because this feature is unusable without it). +- `backend/migrations/20260703170745_fork_ducklake_namespace.*` — registry + GRANTs. +- Frontend: `AssetGraph/{types.ts,AssetNode.svelte,AssetGraphCanvas.svelte, + AssetGraphDetailsPane.svelte}`, `pipeline/[folder]/+page.svelte`, + `sidebar/SidebarContent.svelte` (fork-delete wiring). diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 6352e6adc9..41f5ddc3f5 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -330,6 +330,7 @@ data: { asset_kind: a.kind, path: a.path, + fork_materialization: a.fork_materialization, onAddScript: onAddScriptForAsset, pathPrefix, defaultPathSuffix, diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte index c9b921be2e..095169e3b7 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte @@ -8,9 +8,11 @@ import { formatAssetKind, type AssetWithAltAccessType } from '$lib/components/assets/lib' import { AlertTriangle, + ArrowUpRight, Code2, ExternalLink, GitBranch, + GitFork, Loader2, PanelRightClose, Save, @@ -147,6 +149,11 @@ // `replace` producer). Forwarded to the Schema tab: version history when // true, a single fixed-schema view when false. Defaults to true (unknown). schemaCanEvolve?: boolean + // Fork workspaces: data-environment state of the selected ducklake asset + // ('fork' = materialized here, 'deferred' = reads the parent's current + // data). Pulled from the graph node by the parent page, like + // `schemaCanEvolve`. Undefined outside forks. + selectionForkMaterialization?: 'fork' | 'deferred' // Producer-side facts for the editor's live schema-contract diagnostics // (ignore suppression + scd2 `_current` fallback), built by the page from // the resolved graph and forwarded to ScriptEditor. @@ -259,6 +266,7 @@ selectionProducers = [], selectionColumnGraph, schemaCanEvolve = true, + selectionForkMaterialization = undefined, schemaContractContext = undefined, runsRefreshKey, runsPendingJobId, @@ -1065,6 +1073,31 @@ selected snapshot / tab instead of carrying state across. --> {#key selection.path}
+ {#if selectionForkMaterialization === 'deferred'} + +
+ + + Deferred to parent workspace — reads its current data (not + snapshot-pinned). Materialize this node in the fork to iterate on it. + +
+ {:else if selectionForkMaterialization === 'fork'} +
+ + + Materialized in this fork — reads and writes use the fork's isolated + namespace; the parent workspace's table is untouched. + +
+ {/if} {#if selectionColumnGraph && assetColumnNodes(selectionColumnGraph, selection.asset_kind, selection.path).length > 0}
{formatShortAssetPath(asset)} + + {#if data.fork_materialization === 'deferred'} + + + + {:else if data.fork_materialization === 'fork'} + + + + {/if}
{#if showActions} +{#if allDucklakes.current && allDucklakes.current.length > 0} +
` statements. + target_table: String, } // Fetch and validate a custom data-test script's body. v1 custom tests are @@ -148,11 +169,14 @@ fn build_materialized_query( .target_path .split_once('/') .unwrap_or((m.target_path.as_str(), "")); - let meta = MaterializeExec { + let mut meta = MaterializeExec { asset_kind: windmill_common::assets::AssetKind::Ducklake, asset_path: m.target_path.clone(), partition: partition.clone(), n_data_tests: ann.data_tests.len(), + on_schema_change: m.on_schema_change, + // Set below once the strategy is known (managed persist-and-mutate only). + sync_prepass: None, }; // `{partition}` → escaped SQL literal substitution, applied to the managed @@ -242,6 +266,36 @@ fn build_materialized_query( } } + let cg_probe = windmill_parser::sql_materialize::MaterializeCodegen { + target_qualified: "", + select_sql: "", + partition_col: "_wm_partition", + partition_value_sql: "", + partitioned, + strategy: strategy.clone(), + on_schema_change: m.on_schema_change, + }; + // `sync` needs a host round-trip: a pre-pass probe reads the SELECT's and the + // table's columns so the ALTER DDL can be computed, then spliced into the + // plan at the sentinel slot. Build the probe query here (setup + target + // ATTACH + the two column reads) while we still have the substituted SELECT + + // setup; the executor runs it through the same interpolation/ATTACH transform + // as the main query. Only for the positional persist-and-mutate strategies + // (scd2 / whole-table replace emit no sentinel). + if m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Sync + && cg_probe.is_persist_and_mutate() + { + meta.sync_prepass = Some(SyncPrepass { + probe_query: build_sync_probe_query( + &plan.setup, + &synthetic_attach, + &plan.output, + table, + ), + target_table: table.to_string(), + }); + } + let mat_plan = build_wrap_blocks( &plan, &synthetic_attach, @@ -251,6 +305,7 @@ fn build_materialized_query( &pval, partitioned, strategy, + m.on_schema_change, &resolved, ) .map_err(Error::ExecutionErr)?; @@ -263,6 +318,191 @@ fn build_materialized_query( Ok(Some((Some(blocks.join("\n")), meta))) } +// Build the `on_schema_change=sync` pre-pass probe: the substituted setup + the +// synthetic target ATTACH, then two column reads unioned — the SELECT's columns +// (via `DESCRIBE`) tagged `sel`, and the target table's (via information_schema, +// which yields zero rows when the table doesn't exist yet, so a first +// materialize is a no-drift skip rather than an error) tagged `tbl`. The managed +// `_wm_partition` column is filtered out of the table side. Carries +// `$args`/`{partition}` verbatim; the caller interpolates + ATTACH-transforms it +// like the main query so the DESCRIBE binds run-time args. +fn build_sync_probe_query( + setup: &[String], + synthetic_attach: &str, + select_sql: &str, + table: &str, +) -> String { + let (schema, tname) = match table.split_once('.') { + Some((s, t)) => (Some(s), t), + None => (None, table), + }; + let esc = |s: &str| s.replace('\'', "''"); + let schema_filter = schema + .map(|s| format!(" AND table_schema = '{}'", esc(s))) + .unwrap_or_default(); + let mut out = String::new(); + let mut push_stmt = |s: &str| { + let t = s.trim_end(); + if t.is_empty() { + return; + } + out.push_str(t); + if !t.ends_with(';') { + out.push(';'); + } + out.push('\n'); + }; + for s in setup { + push_stmt(s); + } + push_stmt(synthetic_attach); + // Read the target's columns from `information_schema.columns` scoped to the + // attached target catalog via `table_catalog` (DuckDB's catalog-qualified + // `.information_schema` does NOT exist; the unqualified view spans + // attached catalogs and distinguishes them by `table_catalog`). It yields + // zero rows for a not-yet-created table — a no-drift skip, not an error + // (unlike `pragma_table_info`/`DESCRIBE`). + out.push_str(&format!( + "SELECT 'sel' AS _wm_which, column_name AS _wm_name, column_type AS _wm_type \ + FROM (DESCRIBE SELECT * FROM ({select_sql})) \ + UNION ALL \ + SELECT 'tbl' AS _wm_which, column_name AS _wm_name, data_type AS _wm_type \ + FROM information_schema.columns \ + WHERE table_catalog = '{alias}' AND table_name = '{tname}'{schema_filter} \ + AND column_name <> '_wm_partition';", + alias = windmill_parser::sql_materialize::TARGET_ALIAS, + tname = esc(tname), + )); + out +} + +// Run the sync pre-pass probe (already interpolated + ATTACH-transformed by the +// caller) and turn its column diff into the `ALTER TABLE … ADD/DROP COLUMN` DDL +// to splice at the sentinel. Empty string when the table is fresh (no rows) or +// the SELECT already matches. The probe failing is fatal (no silent fallback). +async fn compute_sync_alter_ddl( + probe_blocks: Vec, + job_args: Vec, + target_table: String, + token: String, + base_internal_url: String, + w_id: String, + job_dir: String, +) -> Result { + let n = probe_blocks.len(); + let (result, _) = tokio::task::spawn_blocking(move || { + run_duckdb_ffi_safe( + probe_blocks.iter().map(String::as_str), + n, + job_args, + &token, + &base_internal_url, + &w_id, + &job_dir, + SqlResultCollectionStrategy::LastStatementAllRows, + ) + }) + .await + .map_err(|e| Error::from(to_anyhow(e))) + .and_then(|r| r) + .map_err(|e| { + Error::ExecutionErr(format!( + "on_schema_change=sync: the schema pre-pass probe failed ({e}); refusing to write" + )) + })?; + + let (added, removed) = parse_sync_drift(&result)?; + let tq = format!( + "{}.{}", + windmill_parser::sql_materialize::TARGET_ALIAS, + quote_qualified_table(&target_table) + ); + let mut stmts = Vec::new(); + for (name, typ) in &added { + if typ.is_empty() { + return Err(Error::ExecutionErr(format!( + "on_schema_change=sync: could not resolve a type for new column `{name}`" + ))); + } + // `typ` is DuckDB's own DESCRIBE type name (e.g. BIGINT, VARCHAR, + // DECIMAL(10,2)) — a valid type expression, injected verbatim. + stmts.push(format!( + "ALTER TABLE {tq} ADD COLUMN \"{}\" {typ};", + name.replace('"', "\"\"") + )); + } + for name in &removed { + stmts.push(format!( + "ALTER TABLE {tq} DROP COLUMN \"{}\";", + name.replace('"', "\"\"") + )); + } + Ok(stmts.join("\n")) +} + +// Parse the sync probe result rows (`_wm_which`/`_wm_name`/`_wm_type`) into the +// column diff: `added` = SELECT columns (with their types) absent from the +// table, `removed` = table columns absent from the SELECT. `_wm_partition` and +// empty names are ignored. An empty table side means the table doesn't exist +// yet — a no-op first materialize, so both lists are empty. +fn parse_sync_drift(result: &RawValue) -> Result<(Vec<(String, String)>, Vec)> { + let root: Value = serde_json::from_str(result.get()).map_err(|e| { + Error::ExecutionErr(format!("on_schema_change=sync: bad probe result json: {e}")) + })?; + // The FFI may surface the rows as an array, a single object, or a JSON + // string — normalize to a list of row objects. + let owned; + let rows: Vec<&Value> = match &root { + Value::Array(a) => a.iter().collect(), + Value::Object(_) => vec![&root], + Value::String(s) => { + owned = serde_json::from_str::(s).unwrap_or(Value::Null); + match &owned { + Value::Array(a) => a.iter().collect(), + Value::Object(_) => vec![&owned], + _ => vec![], + } + } + _ => vec![], + }; + let mut sel: Vec<(String, String)> = Vec::new(); + let mut tbl: std::collections::HashSet = std::collections::HashSet::new(); + for row in rows { + let Some(o) = row.as_object() else { continue }; + let name = o + .get("_wm_name") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + if name.is_empty() || name == "_wm_partition" { + continue; + } + let typ = o + .get("_wm_type") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + match o.get("_wm_which").and_then(|v| v.as_str()).unwrap_or("") { + "sel" => sel.push((name, typ)), + "tbl" => { + tbl.insert(name); + } + _ => {} + } + } + if tbl.is_empty() { + return Ok((Vec::new(), Vec::new())); + } + let sel_names: std::collections::HashSet<&String> = sel.iter().map(|(n, _)| n).collect(); + let added = sel + .iter() + .filter(|(n, _)| !tbl.contains(n)) + .cloned() + .collect(); + let removed = tbl.into_iter().filter(|n| !sel_names.contains(n)).collect(); + Ok((added, removed)) +} + // Fetch the deployed body of every `// data_test ` custom test declared in // `query`, keyed by path, so the sync `build_materialized_query` can splice them // in. The DB read is the only part of materialize codegen that needs a @@ -878,6 +1118,50 @@ fn extract_schema( } } +// Pull the `on_schema_change=warn` drift out of the materialize summary's +// `schema_drift` column: a `{added: [..], removed: [..]}` struct (or NULL when +// the SELECT matched the table). Like `output_schema`, the FFI may surface it +// nested or as a JSON string — accept both. `None` when there is no drift. +fn extract_schema_drift(result: &RawValue) -> Option<(Vec, Vec)> { + fn str_list(v: Option<&Value>) -> Vec { + match v { + Some(Value::Array(a)) => a + .iter() + .filter_map(|x| x.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } + } + fn find_field(v: &Value) -> Option<&Value> { + match v { + Value::Object(o) => o.get("schema_drift"), + Value::Array(a) => a.iter().find_map(find_field), + _ => None, + } + } + let root = serde_json::from_str::(result.get()).ok()?; + let field = find_field(&root)?; + let owned; + let obj = match field { + Value::Object(_) => field, + // FFI serialized the struct as a JSON string — parse it. + Value::String(s) => { + owned = serde_json::from_str::(s).ok()?; + &owned + } + // NULL / absent ⇒ no drift. + _ => return None, + }; + let o = obj.as_object()?; + let added = str_list(o.get("added")); + let removed = str_list(o.get("removed")); + if added.is_empty() && removed.is_empty() { + None + } else { + Some((added, removed)) + } +} + // Render the full pass/fail breakdown for a failed data-test run — every test, // not just the first failure, so the user sees the whole picture in one place. fn format_data_test_breakdown(asset_path: &str, tests: &[DataTestOutcome]) -> String { @@ -1092,7 +1376,27 @@ pub async fn do_duckdb( let (query, _) = &sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; - let query = transform_s3_uris(query).await?; + let mut query = transform_s3_uris(query).await?; + + // `on_schema_change=sync`: interpolate the pre-pass probe with the same + // sig / args / reserved vars as the main query (before `sig` is consumed + // and the arg map drained just below) so its `DESCRIBE` of the SELECT + // binds `$args`/`$partition` the same way. + let sync_probe_interpolated = match materialize + .as_ref() + .and_then(|(_, m)| m.sync_prepass.as_ref()) + { + Some(pp) => { + let (interp, _) = sanitize_and_interpolate_unsafe_sql_args( + &pp.probe_query, + &sig, + &job_args, + &reserved_variables, + )?; + Some(transform_s3_uris(&interp).await?) + } + None => None, + }; let job_args = { let mut m = Vec::new(); @@ -1127,6 +1431,96 @@ pub async fn do_duckdb( m }; + // `on_schema_change=sync` pre-pass: run the probe (through the same + // ATTACH transform as the main query), diff the SELECT's columns against + // the table's, and splice the `ALTER TABLE … ADD/DROP COLUMN` DDL into + // the plan at the sentinel slot (removing the sentinel when there is no + // drift). Runs before the main query so the ALTERs are in place when the + // BY NAME insert executes. + if let (Some(probe_sql), Some(pp)) = ( + &sync_probe_interpolated, + materialize + .as_ref() + .and_then(|(_, m)| m.sync_prepass.as_ref()), + ) { + let mut probe_blocks = vec![]; + // Held for the whole probe (through compute_sync_alter_ddl): a + // `TYPE bigquery` ATTACH needs GOOGLE_APPLICATION_CREDENTIALS + the + // temp creds file set before DuckDB binds the extension, exactly as + // the main query path does — otherwise the probe DESCRIBE fails. + let mut probe_bigquery_credentials = None; + for query_block in parse_sql_blocks(probe_sql, true).iter() { + let query_block = remove_comments(query_block); + if let Some(parsed) = parse_attach_db_resource(query_block) { + probe_blocks.extend( + transform_attach_db_resource_query( + &parsed, + &job.id, + client, + &mut hidden_passwords, + ) + .await?, + ); + if parsed.db_type == "bigquery" { + probe_bigquery_credentials = Some(UseBigQueryCredentialsFile::new( + job.id, + parsed.resource_path, + )?); + } + } else if let Some(q) = transform_attach_ducklake( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + materialize.as_ref().map(|(_, m)| m.asset_path.as_str()), + ) + .await? + { + probe_blocks.extend(q); + } else if let Some(q) = transform_attach_datatable( + &query_block, + conn, + &mut hidden_passwords, + &job.workspace_id, + ) + .await? + { + probe_blocks.extend(q); + } else { + probe_blocks.push(query_block.to_string()); + } + } + // The probe `DESCRIBE`s the same SELECT as the main query, so it + // needs the same workspace-macro / `// use` library definitions + // injected (post-ATTACH), or a macro-calling SELECT fails to bind + // here and the fatal probe makes `sync` unusable with macros. + let probe_blocks = inject_workspace_macros( + conn, + &job.workspace_id, + macro_ann.macros, + ¯o_ann.use_libs, + probe_blocks, + ) + .await?; + let alter_ddl = compute_sync_alter_ddl( + probe_blocks, + job_args.clone(), + pp.target_table.clone(), + token.clone(), + client.base_internal_url.clone(), + job.workspace_id.clone(), + job_dir.to_string(), + ) + .await?; + query = query.replace( + windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL, + &alter_ddl, + ); + // Kept alive until the probe finished; the main query path recreates + // its own credentials for its own ATTACH pass below. + drop(probe_bigquery_credentials); + } + let query_block_list = parse_sql_blocks(&query, true); // Replace custom ATTACH statements with the real instructions @@ -1283,6 +1677,37 @@ pub async fn do_duckdb( } else { None }; + // `on_schema_change=warn` (default): the summary carries a + // `schema_drift` column when the SELECT's columns diverged from the + // fixed table schema. The write already happened positionally (data + // may have landed in the wrong/old columns) — log it LOUDLY so the + // drift isn't silent. The drift also rides back in the job result + // (it's a column of the returned summary row). + if is_managed + && meta.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Warn + { + if let Some((added, removed)) = extract_schema_drift(&result) { + let fmt = |cols: &[String]| { + if cols.is_empty() { + "(none)".to_string() + } else { + cols.join(", ") + } + }; + let warning = format!( + "\n⚠️ SCHEMA DRIFT on {asset}: the SELECT's columns no longer match the \ + existing table.\n added (in SELECT, not in table): {added}\n removed \ + (in table, not in SELECT): {removed}\n The write proceeded POSITIONALLY \ + against the existing table schema — data may have landed in the wrong \ + columns. Set `on_schema_change=fail` to block this, or \ + `on_schema_change=sync` to migrate the table automatically.\n", + asset = meta.asset_path, + added = fmt(&added), + removed = fmt(&removed), + ); + append_logs(&job.id, &job.workspace_id, warning, conn).await; + } + } // Defense-in-depth: codegen embedded `n_data_tests` checks, so the // summary row must carry that many outcomes. Recovering fewer means // the `data_tests` column was dropped/reshaped before we read it — @@ -2984,6 +3409,134 @@ mod tests { ); } + // on_schema_change=fail on a persist-and-mutate strategy (merge) emits the + // drift guard inside the write, and no sync artifacts. + #[test] + fn materialize_fail_emits_drift_guard() { + let script = "-- materialize ducklake://main/dim key=id on_schema_change=fail\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("CAST(error(") && rewritten.contains("on_schema_change=fail"), + "fail emits the drift guard: {rewritten}" + ); + assert!(!rewritten.contains("BY NAME"), "fail writes positionally"); + assert!(meta.sync_prepass.is_none(), "fail needs no pre-pass probe"); + } + + // on_schema_change=sync writes BY NAME, emits the ALTER sentinel, and builds + // the pre-pass probe (DESCRIBE of the SELECT + information_schema read). + #[test] + fn materialize_sync_by_name_and_prepass() { + let script = "-- materialize ducklake://main/dim key=id on_schema_change=sync\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("INSERT INTO _wm_target.dim BY NAME"), + "sync insert is name-mapped: {rewritten}" + ); + assert!( + rewritten.contains(windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL), + "sync emits the ALTER injection sentinel" + ); + let pp = meta.sync_prepass.expect("sync builds a pre-pass probe"); + assert_eq!(pp.target_table, "dim"); + assert!(pp + .probe_query + .contains("DESCRIBE SELECT * FROM (SELECT id, name FROM dl.src)")); + assert!(pp.probe_query.contains("FROM information_schema.columns")); + assert!(pp + .probe_query + .contains("table_catalog = '_wm_target' AND table_name = 'dim'")); + assert!(pp + .probe_query + .contains("ATTACH 'ducklake://main' AS _wm_target;")); + } + + // on_schema_change=warn (default) on a persist-and-mutate strategy folds the + // drift into the summary; no guard, no sync artifacts. + #[test] + fn materialize_warn_summary_carries_schema_drift() { + let script = "-- materialize ducklake://main/dim key=id\n\ + SELECT id, name FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("AS schema_drift"), + "warn folds drift into the summary: {rewritten}" + ); + assert!(!rewritten.contains("CAST(error(")); + assert!(!rewritten.contains("BY NAME")); + assert!(meta.sync_prepass.is_none()); + } + + // Whole-table replace (unpartitioned, no key/append) is not persist-and- + // mutate: even with on_schema_change=fail there is no guard / sentinel — the + // CREATE OR REPLACE already rebuilds the schema each run. + #[test] + fn materialize_whole_table_replace_ignores_guardrail() { + let script = "-- materialize ducklake://main/t on_schema_change=fail\n\ + SELECT a, b FROM dl.src"; + let (rewritten, meta) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!(rewritten.contains("CREATE OR REPLACE TABLE _wm_target.t")); + assert!(!rewritten.contains("on_schema_change=fail")); + assert!(!rewritten.contains(windmill_parser::sql_materialize::SYNC_ALTER_SENTINEL)); + assert!(meta.sync_prepass.is_none()); + } + + #[test] + fn extract_schema_drift_parses_struct_and_string_and_null() { + // nested struct form + let r = raw( + r#"{"materialized":"ducklake://main/dim","schema_drift":{"added":["c"],"removed":["b"]}}"#, + ); + let (added, removed) = extract_schema_drift(&r).expect("drift present"); + assert_eq!(added, vec!["c".to_string()]); + assert_eq!(removed, vec!["b".to_string()]); + // FFI string-encoded form + let r = raw(r#"{"schema_drift":"{\"added\":[\"x\"],\"removed\":[]}"}"#); + let (added, removed) = extract_schema_drift(&r).expect("drift present"); + assert_eq!(added, vec!["x".to_string()]); + assert!(removed.is_empty()); + // NULL / both-empty ⇒ no drift + assert!(extract_schema_drift(&raw(r#"{"schema_drift":null}"#)).is_none()); + assert!( + extract_schema_drift(&raw(r#"{"schema_drift":{"added":[],"removed":[]}}"#)).is_none() + ); + assert!(extract_schema_drift(&raw(r#"{"rows":3}"#)).is_none()); + } + + #[test] + fn parse_sync_drift_computes_added_removed_and_skips_fresh_table() { + // table (a, b) vs SELECT (a, c): add c, drop b; _wm_partition ignored + let r = raw(r#"[{"_wm_which":"sel","_wm_name":"a","_wm_type":"BIGINT"}, + {"_wm_which":"sel","_wm_name":"c","_wm_type":"VARCHAR"}, + {"_wm_which":"tbl","_wm_name":"a","_wm_type":"BIGINT"}, + {"_wm_which":"tbl","_wm_name":"b","_wm_type":"BIGINT"}]"#); + let (added, removed) = parse_sync_drift(&r).unwrap(); + assert_eq!(added, vec![("c".to_string(), "VARCHAR".to_string())]); + assert_eq!(removed, vec!["b".to_string()]); + // no `tbl` rows ⇒ table doesn't exist yet ⇒ no migration + let r = raw(r#"[{"_wm_which":"sel","_wm_name":"a","_wm_type":"BIGINT"}]"#); + let (added, removed) = parse_sync_drift(&r).unwrap(); + assert!(added.is_empty() && removed.is_empty()); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { From 5769b6036cf14b0cb424c5b3d9d878c600a5652e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 17:10:06 +0200 Subject: [PATCH 12/34] fix(pipelines): activity-axis label clarity + select failed node on cascade failure (#9931) * fix(pipelines): clarify activity-window axis label + select failed node on cascade failure Co-Authored-By: Claude Fable 5 * fix(pipelines): clear active draft so failed-node focus is not masked PipelineGraphEditor gives an open draft (activeDraftPath) priority over selection via openScriptPath, so the cascade-failure focus set only selection and stayed masked while a draft pane was open. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Fable 5 --- .../assets/AssetGraph/ActivityHistogram.svelte | 5 +++-- .../(logged)/pipeline/[folder]/+page.svelte | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte index fcfd95a5d3..cd9548de59 100644 --- a/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte @@ -147,9 +147,10 @@ > {/if} - +
- {fmtTime(from)} + from {fmtTime(from)} now
diff --git a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte index 0889d53a01..167aaa8f25 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte @@ -1480,6 +1480,14 @@ } else { const failed = [...res.statuses.entries()].filter(([, s]) => s.status === 'failure') const skipped = [...res.statuses.values()].filter((s) => s.status === 'skipped').length + // Surface the failed node's error in the details pane. Clear the + // active draft too — it has pane priority over `selection` + // (openScriptPath), so a bare selection would stay masked while a + // draft is open. + if (failed.length > 0) { + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: failed[0][0] } + } sendUserToast( `Chain run failed at ${failed.map(([p]) => p).join(', ')}` + (skipped > 0 ? ` — ${skipped} downstream skipped` : ''), @@ -1616,6 +1624,14 @@ } else { const failed = [...res.statuses.entries()].filter(([, s]) => s.status === 'failure') const skipped = [...res.statuses.values()].filter((s) => s.status === 'skipped').length + // Surface the failed node's error in the details pane. Clear the + // active draft too — it has pane priority over `selection` + // (openScriptPath), so a bare selection would stay masked while a + // draft is open. + if (failed.length > 0) { + pe.activeDraftPath = undefined + pe.selection = { kind: 'runnable', runnable_kind: 'script', path: failed[0][0] } + } sendUserToast( `Bounded run failed at ${failed.map(([p]) => p).join(', ')}` + (skipped > 0 ? ` — ${skipped} downstream skipped` : ''), From 182b10b2ad084cce28be3f25fdded44b9bf51f9e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 18:01:20 +0200 Subject: [PATCH 13/34] render FFI errors with real newlines and no stray quoting (#9932) * fix(duckdb): render FFI errors with real newlines and no stray quoting The DuckDB FFI returns errors as ERROR , so the executor was surfacing the serde_json-escaped form (wrapping quotes, literal \\n). Multi-line errors like the write-audit-publish data-test breakdown were unreadable. Decode the JSON string back to the raw message at both FFI error sites. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: tighten decode_ffi_error comment to the invariant Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-worker/src/duckdb_executor.rs | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index d7436c579c..e51c59cb98 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2004,6 +2004,15 @@ fn cgroup_bytes_to_duckdb_memory_limit(bytes: i64) -> Option { } // Read backend/windmill-duckdb-ffi-internal/README_DEV.md for details about why we use FFI +// The FFI returns errors as `ERROR ` (the message is +// serde_json::to_string'd). Decode the JSON string back so multi-line errors +// render with real newlines, not escaped `\n` inside wrapping quotes; fall back +// to the raw slice if it is somehow not a JSON string. +fn decode_ffi_error(result_str: &str) -> String { + let raw = result_str.strip_prefix("ERROR ").unwrap_or(result_str); + serde_json::from_str::(raw).unwrap_or_else(|_| raw.to_string()) +} + fn run_duckdb_ffi_safe<'a>( query_block_list: impl Iterator, query_block_list_count: usize, @@ -2069,7 +2078,7 @@ fn run_duckdb_ffi_safe<'a>( }; if result_str.starts_with("ERROR") { - Err(Error::ExecutionErr(result_str[6..].to_string())) + Err(Error::ExecutionErr(decode_ffi_error(&result_str))) } else { let result = if collection_strategy == SqlResultCollectionStrategy::AllStatementsAllRows { // Avoid parsing JSON @@ -2133,7 +2142,7 @@ fn prepare_duckdb_ffi_safe<'a>( }; if result_str.starts_with("ERROR") { - Err(Error::ExecutionErr(result_str[6..].to_string())) + Err(Error::ExecutionErr(decode_ffi_error(&result_str))) } else { Ok(serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?) } @@ -2629,6 +2638,31 @@ pub struct Arg { mod tests { use super::*; + #[test] + fn decode_ffi_error_unescapes_multiline_and_strips_quotes() { + // Mirror the FFI: JSON-encode the raw DuckDB message, prefix "ERROR ". + let raw_msg = "Invalid Input Error: data tests failed on main/raw_orders \ + (1/3 failed) — write rolled back, previous version left live:\n \ + ✓ not_null(order_id)\n ✗ accepted_values(status) — 12 violating row(s)"; + let ffi = format!("ERROR {}", serde_json::to_string(raw_msg).unwrap()); + let decoded = decode_ffi_error(&ffi); + assert_eq!(decoded, raw_msg); + // real newlines, no literal `\n`, no wrapping quotes + assert!(decoded.contains('\n') && !decoded.contains("\\n")); + assert!(!decoded.starts_with('"')); + // single-line error with inner quotes round-trips unescaped + let single = format!( + "ERROR {}", + serde_json::to_string("Binder Error: column \"x\" not found").unwrap() + ); + assert_eq!( + decode_ffi_error(&single), + "Binder Error: column \"x\" not found" + ); + // non-JSON payload falls back to the raw slice + assert_eq!(decode_ffi_error("ERROR not json"), "not json"); + } + fn test_ancestor(wid: &str) -> windmill_common::workspaces::DucklakeAncestorAttach { windmill_common::workspaces::DucklakeAncestorAttach { workspace_id: wid.to_string(), From 799b9e3b7c1998c1264c17cafba858bf192b8cb8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 18:23:51 +0200 Subject: [PATCH 14/34] chore(main): release 1.748.0 (#9914) * chore(main): release 1.748.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 23 ++ backend/Cargo.lock | 211 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 ++-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 174 insertions(+), 140 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5342e3024..ec10dc8990 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [1.748.0](https://github.com/windmill-labs/windmill/compare/v1.747.0...v1.748.0) (2026-07-05) + + +### Features + +* **ai-agent:** support reasoning effort in AI agent workflow steps ([#9886](https://github.com/windmill-labs/windmill/issues/9886)) ([a368d49](https://github.com/windmill-labs/windmill/commit/a368d49bd8786a2dca6771f2051f1d44d1b2363d)) +* **ducklake:** scheduled lake maintenance (expiry, compaction, orphan cleanup) ([#9916](https://github.com/windmill-labs/windmill/issues/9916)) ([3352150](https://github.com/windmill-labs/windmill/commit/33521505dbc34f22b575d21fda1cc76d698a8840)) +* **pipelines:** asset freshness — fresh/stale badge (CE) + watchdog (EE) ([#9909](https://github.com/windmill-labs/windmill/issues/9909)) ([5d7fb6d](https://github.com/windmill-labs/windmill/commit/5d7fb6deca3e02e89d77e5d3856483beb8b8bfeb)) +* **pipelines:** capture violating-row samples for data tests ([#9919](https://github.com/windmill-labs/windmill/issues/9919)) ([d4b4374](https://github.com/windmill-labs/windmill/commit/d4b4374de8f8a7875b050c16d1236fcd0355812b)) +* **pipelines:** fork data environments for ducklake materialization (dev data) ([#9915](https://github.com/windmill-labs/windmill/issues/9915)) ([39eb9de](https://github.com/windmill-labs/windmill/commit/39eb9de1bce400109c130a081807e40e995ae068)) +* **pipelines:** on_schema_change write guardrails + data_test deploy validation ([#9930](https://github.com/windmill-labs/windmill/issues/9930)) ([377c02e](https://github.com/windmill-labs/windmill/commit/377c02ec47389e64b7ef5cbbae0de05df648266d)) +* **pipelines:** record upstream snapshot ids on cascade-dispatched jobs ([#9910](https://github.com/windmill-labs/windmill/issues/9910)) ([af36498](https://github.com/windmill-labs/windmill/commit/af36498432e643108308e1c03b5d986d0f0f8888)) +* **pipelines:** schema contracts — save-time consumer checks vs captured schemas ([#9917](https://github.com/windmill-labs/windmill/issues/9917)) ([42e11c6](https://github.com/windmill-labs/windmill/commit/42e11c6570b62ffaa86598438fa8ddf462c4035f)) +* **pipeline:** write-audit-publish for materialization data tests ([#9911](https://github.com/windmill-labs/windmill/issues/9911)) ([dce247c](https://github.com/windmill-labs/windmill/commit/dce247c6d2678a2c95bd728027e17ae3965638e2)) +* **sdk:** enforce s3:// URIs for string S3 params + ingestion (EL) docs ([#9912](https://github.com/windmill-labs/windmill/issues/9912)) ([5ad2de9](https://github.com/windmill-labs/windmill/commit/5ad2de91a26b312bf27124ceca16ef331621bde8)) + + +### Bug Fixes + +* **cli:** pipeline + workspace UX batch (init/bind stub, run errors, macro libs, lock-job report, upgrade errors) ([#9929](https://github.com/windmill-labs/windmill/issues/9929)) ([28a6b08](https://github.com/windmill-labs/windmill/commit/28a6b086c842105298f236baa0a61868f71a5eb1)) +* **cli:** publish all windmill-parser-wasm-* deps so local pipeline graph keeps write edges ([#9926](https://github.com/windmill-labs/windmill/issues/9926)) ([744a759](https://github.com/windmill-labs/windmill/commit/744a7597edaf3ca9a7fd2b21a34fb33457913a64)) +* **pipelines:** activity-axis label clarity + select failed node on cascade failure ([#9931](https://github.com/windmill-labs/windmill/issues/9931)) ([5769b60](https://github.com/windmill-labs/windmill/commit/5769b6036cf14b0cb424c5b3d9d878c600a5652e)) + ## [1.747.0](https://github.com/windmill-labs/windmill/compare/v1.746.0...v1.747.0) (2026-07-03) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1153916767..cd780be48c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2057,9 +2057,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.65" +version = "1.2.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" dependencies = [ "find-msvc-tools", "jobserver", @@ -5072,9 +5072,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -7563,9 +7565,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", @@ -8361,9 +8363,9 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pest" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0848c601009d37dfa3430c4666e147e49cdcf1b92ecd3e63657d8a5f19da662" +checksum = "47627dd7305c6a2d6c8c6bcd24c5a4c17dbbf425f4f9c5313e724b38fc9782e9" dependencies = [ "memchr", "ucd-trie", @@ -8371,9 +8373,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f486f1ea21e6c10ed15d5a7c77165d0ee443402f0780849d1768e7d9d6fe77" +checksum = "4b4254325ecad416ab689e27ba51da03ba01a9632bc6e108f5fe7c3c4ad29d58" dependencies = [ "pest", "pest_generator", @@ -8381,9 +8383,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8040c4647b13b210a963c1ed407c1ff4fdfa01c31d6d2a098218702e6664f94f" +checksum = "6c4c0e91ead7a8f7acecbca6f003fc2e8282b1dbe2dd9c9d2f16aba42995e0a7" dependencies = [ "pest", "pest_meta", @@ -8394,12 +8396,11 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.8.6" +version = "2.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89815c69d36021a140146f26659a81d6c2afa33d216d736dd4be5381a7362220" +checksum = "f9744bc48116fee06334924bb5f2bad41eed5e89bd26e29b0b799f9a3f82c210" dependencies = [ "pest", - "sha2 0.10.9", ] [[package]] @@ -8995,15 +8996,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.4", + "getrandom 0.4.3", "lru-slab", - "rand 0.9.0", + "rand 0.10.2", + "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", "rustls 0.23.35", @@ -9017,16 +9019,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" dependencies = [ "cfg_aliases", "libc", "once_cell", "socket2 0.6.4", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9184,6 +9186,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -13322,9 +13333,9 @@ checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" [[package]] name = "utf8-width" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" +checksum = "159a7cadce548703edd50d24069bc294c5415ecab0a480e0cd1ca06d112dc94a" [[package]] name = "utf8_iter" @@ -13735,7 +13746,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -13817,7 +13828,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.747.0" +version = "1.748.0" dependencies = [ "async-stream", "async-trait", @@ -13850,7 +13861,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13863,7 +13874,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "argon2", @@ -14001,7 +14012,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14024,7 +14035,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14039,7 +14050,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14065,7 +14076,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.747.0" +version = "1.748.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14075,7 +14086,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14092,7 +14103,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14114,7 +14125,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14137,7 +14148,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14153,7 +14164,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14174,7 +14185,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14195,7 +14206,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14209,7 +14220,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -14244,7 +14255,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14269,7 +14280,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14287,7 +14298,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14309,7 +14320,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14329,7 +14340,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14366,7 +14377,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14394,7 +14405,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.747.0" +version = "1.748.0" dependencies = [ "lazy_static", "serde", @@ -14406,7 +14417,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.747.0" +version = "1.748.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14431,7 +14442,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14445,7 +14456,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.747.0" +version = "1.748.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14479,7 +14490,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.747.0" +version = "1.748.0" dependencies = [ "chrono", "lazy_static", @@ -14493,7 +14504,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14512,7 +14523,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.747.0" +version = "1.748.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14614,7 +14625,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.747.0" +version = "1.748.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14633,7 +14644,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.747.0" +version = "1.748.0" dependencies = [ "regex", "serde", @@ -14648,7 +14659,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14672,7 +14683,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "futures", @@ -14689,7 +14700,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.747.0" +version = "1.748.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14705,7 +14716,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -14726,7 +14737,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -14757,7 +14768,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "arc-swap", @@ -14782,7 +14793,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-stream", @@ -14816,7 +14827,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "futures", @@ -14834,7 +14845,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.747.0" +version = "1.748.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14843,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14855,7 +14866,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14867,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "gosyn", @@ -14879,7 +14890,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14891,7 +14902,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14903,7 +14914,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "nu-parser", @@ -14914,7 +14925,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14925,7 +14936,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14937,7 +14948,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14948,7 +14959,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -14970,7 +14981,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -14982,7 +14993,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -14996,7 +15007,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15013,7 +15024,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -15026,7 +15037,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -15038,7 +15049,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -15056,7 +15067,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15072,7 +15083,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15088,7 +15099,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -15099,7 +15110,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -15138,7 +15149,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "const_format", @@ -15177,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.747.0" +version = "1.748.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15188,7 +15199,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -15222,7 +15233,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15246,7 +15257,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15279,7 +15290,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15312,7 +15323,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15332,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15366,7 +15377,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15402,7 +15413,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15425,7 +15436,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15449,7 +15460,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-nats", @@ -15473,7 +15484,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15508,7 +15519,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15536,7 +15547,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-trait", @@ -15561,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15580,7 +15591,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-once-cell", @@ -15690,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.747.0" +version = "1.748.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a91518dc38..e80fbc89db 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.747.0" +version = "1.748.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.747.0" +version = "1.748.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 815d62644f..a9caa82ede 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.747.0" +version = "1.748.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.747.0" +version = "1.748.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.747.0" +version = "1.748.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.747.0" +version = "1.748.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 9111ca8ccb..521fb7a132 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.747.0" +version = "1.748.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0c080b0a2a..afe8cbc0b6 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.747.0 + version: 1.748.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 4b342d93bd..37132a3b9e 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.747.0"; +export const VERSION = "v1.748.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 9e2110814d..f01ea9c68b 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.747.0"; +export const VERSION = "1.748.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 3b9c1b8134..f0fd64f6fc 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 7f0ed5e1d9..adfd48cad4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.747.0", + "version": "1.748.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 482c95db43..d2b78d3e8f 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.747.0" +wmill = ">=1.748.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index b0508b2719..32f13a4f0c 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.747.0 + version: 1.748.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 0b3006323a..f99fb3832d 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.747.0' + ModuleVersion = '1.748.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index dda29ba2b4..3219756afa 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.747.0" +version = "1.748.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index afcb96626d..191bab2bf6 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.747.0", + "version": "1.748.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index d8058153be..02a1bbb060 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.747.0", + "version": "1.748.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 8eff60cc0e..a16d7ab553 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.747.0 +1.748.0 From 22452ce54034a9bea8f7d48946818fd148b938c0 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:20:55 +0200 Subject: [PATCH 15/34] fix(object-storage): remove 20-file bucket-browser listing cap in CE (#9935) * fix(object-storage): remove 20-file bucket-browser cap in CE The Community Edition build rejected the object-storage `list_stored_files` endpoint with an error once a workspace bucket held more than 20 objects, making the bucket browser unusable on larger buckets. The listing already collects up to `max_keys` objects, so the hard cap was purely a gate. Drops the CE listing cap (in the EE-symlinked `job_helpers_ee.rs`, tracked in the companion windmill-ee-private PR) and removes the now-inaccurate sentence from the workspace object-storage settings banner. The 10 GiB total-storage write quota remains as the intentional CE limit. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 4bd7b73dcef1b77dc2866bc58e0c86962559495c This commit updates the EE repository reference after PR #648 was merged in windmill-ee-private. Previous ee-repo-ref: aa14d0724216030948c2f575bcc19c0e6e0476a7 New ee-repo-ref: 4bd7b73dcef1b77dc2866bc58e0c86962559495c Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../lib/components/workspaceSettings/StorageSettings.svelte | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c3ae39c767..cc9c36f5a5 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -fa7ac11c1e0ab39e84a0c18973ba427a240933ca +4bd7b73dcef1b77dc2866bc58e0c86962559495c diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte index be49e3c564..1e6c990819 100644 --- a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -190,8 +190,7 @@ {#if !$enterpriseLicense} Total workspace storage is capped at {quotaDisplay} in the Community Edition: writes that would exceed - the quota are rejected. The bucket browser will also not work for buckets containing more than 20 - files. Consider upgrading to Windmill EE for unlimited workspace storage. + the quota are rejected. Consider upgrading to Windmill EE for unlimited workspace storage. {:else} From 52ce805f619747af4f998cde7819a164c754205a Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:24:41 +0200 Subject: [PATCH 16/34] fix(pipelines): dedup guard for keyed merge + deploy-time SCD2 validation (#9936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two correctness/validation improvements to managed materialization: 1. A keyed `merge` (`key=`) is delete-by-key + insert-all and does NOT deduplicate its source, so two incoming rows sharing a key both landed under that key — silently breaking the one-row-per-key contract. Codegen now emits an in-transaction guard (same `error(...)` shape as the schema -drift guard) that fails the run when the SELECT returns more than one row for a non-NULL key, naming the key. Authors deduplicate in the SELECT or switch to `append`. NULL keys are exempt, matching the delete's `IN (...)` scope. 2. The two SCD2 misconfigurations that were only caught at run time — `history` without `key=`, and `history` + `// partitioned` — now fail fast at deploy via a shared `MaterializeSpec::validate`, called from `create_script_internal`. The DuckDB executor keeps the same check as a safety net for preview/test runs that never deploy (shared message, no drift). Adds unit tests for the merge guard codegen and for `validate` (all four cases), and updates docs/ducklake-materialization.md and docs/pipelines-vs-dbt.md. Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-parser/src/asset_parser.rs | 79 +++++++++++++++++++ .../windmill-parser/src/sql_materialize.rs | 64 +++++++++++++++ backend/windmill-api-scripts/src/scripts.rs | 7 ++ .../windmill-worker/src/duckdb_executor.rs | 18 +++-- docs/ducklake-materialization.md | 19 ++++- docs/pipelines-vs-dbt.md | 11 ++- 6 files changed, 185 insertions(+), 13 deletions(-) diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4ef30246a6..d95234a297 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -333,6 +333,40 @@ impl OnSchemaChange { } } +impl MaterializeSpec { + /// Deploy-time validation of the option combination against the script's + /// partitioning, returning a human-facing error for combinations the + /// runtime cannot honor. Called at save (`create_script_internal`) so a + /// misconfigured script is rejected up front, and again in the DuckDB + /// executor as a safety net for preview/test runs that never deploy. Both + /// checks are SCD2-specific and inert for `manual` mode (which owns its DDL + /// and ignores the reconciliation strategy). `partitioned` is whether the + /// script declares `// partitioned`. + pub fn validate(&self, partitioned: bool) -> Result<(), String> { + if self.manual || !self.scd2 { + return Ok(()); + } + // SCD2 needs a natural key to identify an entity across versions. + if self.unique_key.as_deref().map_or(true, str::is_empty) { + return Err( + "materialize scd2: requires a natural key — add `key=` (e.g. \ + `// materialize ducklake:///
key=id history`)" + .to_string(), + ); + } + // SCD2's diff/close/open shape has no partition-scoped form in v1. + if partitioned { + return Err( + "materialize scd2: `// partitioned` is not supported with scd2 in v1 — remove \ + `// partitioned`, or drop `history`/`scd2` to materialize the partition without \ + history" + .to_string(), + ); + } + Ok(()) + } +} + // `// data_test …` — a data-quality assertion run against the // freshly-materialized asset (post DELETE+INSERT), failing the run on // violation. The first extensible annotation family: the parser turns a @@ -1721,6 +1755,51 @@ mod pipeline_annotation_tests { assert!(!out.materialize.expect("materialize").close_deleted); } + #[test] + fn materialize_validate_scd2_requires_key() { + // scd2 without `key=` is rejected at deploy (was a run-time error). + let m = parse_pipeline_annotations("// materialize ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.scd2 && m.unique_key.is_none()); + let err = m.validate(false).expect_err("scd2 without key must fail"); + assert!(err.contains("requires a natural key")); + // with a key it validates + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_scd2_rejects_partitioned() { + // scd2 + `// partitioned` has no v1 form — rejected at deploy. + let m = parse_pipeline_annotations("// materialize ducklake://a/dim key=id history") + .materialize + .expect("materialize"); + let err = m.validate(true).expect_err("scd2 + partitioned must fail"); + assert!(err.contains("`// partitioned` is not supported with scd2")); + // unpartitioned scd2 is fine + assert!(m.validate(false).is_ok()); + } + + #[test] + fn materialize_validate_non_scd2_and_manual_are_inert() { + // Non-scd2 strategies are unconstrained by these checks, partitioned or not. + let m = parse_pipeline_annotations("// materialize ducklake://a/orders key=id") + .materialize + .expect("materialize"); + assert!(m.validate(true).is_ok()); + assert!(m.validate(false).is_ok()); + // `manual` owns its DDL and ignores the strategy — never rejected here, + // even with a partitioned scd2-looking combo. + let m = parse_pipeline_annotations("// materialize manual ducklake://a/dim history") + .materialize + .expect("materialize"); + assert!(m.manual && m.scd2); + assert!(m.validate(true).is_ok()); + } + #[test] fn materialize_on_schema_change_opt() { let out = parse_pipeline_annotations( diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index fff036b22c..dccf8ec313 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -514,6 +514,11 @@ impl<'a> MaterializeCodegen<'a> { // same write shape as `replace`, which is reliable. The DELETE is // scoped to the current partition when partitioned so it stays // slice-local (a key present in another partition is untouched). + // + // Guard first: the DELETE+INSERT does not dedup the source, so + // two incoming rows sharing a key would both persist under it. + // Raise instead of silently double-writing (see the helper). + out.push(duplicate_source_key_guard_sql(sel, unique_key)); let scope = if self.partitioned { format!("{pcol} = {pval} AND ") } else { @@ -746,6 +751,29 @@ fn schema_drift_guard_sql(sel_sql: &str, target_qualified: &str, partition_col: ) } +/// In-transaction guard for the keyed `merge` strategy: raises via DuckDB +/// `error(...)` when the source SELECT holds more than one row for the same +/// non-NULL `unique_key`. A keyed merge is delete-by-key + insert-all (it does +/// NOT deduplicate the source), so duplicate source keys would land every +/// duplicate row under one key — the exact silent double-write this guards +/// against. Erroring keeps the semantics explicit: the author must deduplicate +/// in the SELECT (or use `append`). NULL keys are excluded to match the delete's +/// `key IN (...)` scope, which never matches NULL. `unique_key` is embedded raw +/// in identifier position (matching the merge's own DELETE/IN) and doubled-quote +/// escaped where it lands inside the error string literal. +fn duplicate_source_key_guard_sql(sel_sql: &str, unique_key: &str) -> String { + let key_lit = unique_key.replace('\'', "''"); + format!( + "SELECT CASE WHEN _wm_dup_keys > 0 THEN CAST(error('managed materialize: keyed merge on \ + `{key_lit}` blocked — the source has ' || _wm_dup_keys || ' key value(s) with more than \ + one row. A keyed merge keeps one row per key and does not deduplicate; deduplicate in the \ + SELECT (e.g. QUALIFY row_number() OVER (PARTITION BY {key_lit} ORDER BY …) = 1) or use \ + `append`.') AS VARCHAR) ELSE 'ok' END FROM (SELECT count(*) AS _wm_dup_keys FROM (SELECT \ + {unique_key} FROM ({sel_sql}) WHERE {unique_key} IS NOT NULL GROUP BY {unique_key} HAVING \ + count(*) > 1));" + ) +} + /// The `warn`-mode summary column: a `schema_drift` struct `{added, removed}` /// when the SELECT drifted from the table, else NULL. Appended to the /// materialize summary row so the executor can log it and fold it into the job @@ -1555,6 +1583,42 @@ mod tests { .any(|s| s.starts_with("INSERT INTO _wm_target.orders_daily SELECT *, '2026-06-19'"))); } + #[test] + fn codegen_merge_emits_duplicate_source_key_guard() { + // A keyed merge does not dedup its source, so codegen must emit a guard + // that fails the write when the SELECT has >1 row per key — otherwise + // duplicate source keys silently double-write. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.orders_daily", + select_sql: "SELECT order_id, amount FROM dl.orders", + partition_col: "_wm_partition", + partition_value_sql: "'2026-06-19'", + partitioned: false, + strategy: MaterializeStrategy::Merge { unique_key: "order_id".to_string() }, + on_schema_change: OnSchemaChange::Warn, + }; + let st = cg.statements(); + let guard = st + .iter() + .find(|s| s.contains("_wm_dup_keys")) + .expect("duplicate-key guard stmt"); + // raises via error(), counts non-NULL keys appearing more than once + assert!(guard.contains("error('managed materialize: keyed merge on `order_id` blocked")); + assert!(guard.contains("GROUP BY order_id HAVING count(*) > 1")); + assert!(guard.contains("WHERE order_id IS NOT NULL")); + // must run before the mutations so a violation aborts before any write + let guard_pos = st.iter().position(|s| s.contains("_wm_dup_keys")).unwrap(); + let del_pos = st + .iter() + .position(|s| s.starts_with("DELETE FROM")) + .unwrap(); + let ins_pos = st + .iter() + .position(|s| s.starts_with("INSERT INTO")) + .unwrap(); + assert!(guard_pos < del_pos && guard_pos < ins_pos); + } + #[test] fn codegen_append_inserts_only() { let cg = MaterializeCodegen { diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index c3d01dad13..0ae16b9d51 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -1307,6 +1307,13 @@ async fn create_script_internal<'c>( ns.path ); } + // SCD2 (`key=history` / `scd2`) option-combination checks the + // runtime cannot honor: reject at deploy with a clear message rather + // than letting the script deploy and fail on its first run. Mirrors the + // executor's safety-net `MaterializeSpec::validate` (duckdb_executor). + if let Err(e) = m.validate(pipeline_annotations.partition.is_some()) { + return Err(Error::BadRequest(e)); + } // `manual` materialize never captures a schema (no wrap codegen), so // there is no contract for `on_schema_change` to mute downstream. if m.manual && m.on_schema_change == windmill_parser::asset_parser::OnSchemaChange::Ignore { diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e51c59cb98..051aa77488 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -219,19 +219,18 @@ fn build_materialized_query( for s in plan.setup.iter_mut() { *s = substitute(s); } + // Deploy (`create_script_internal`) already rejects the invalid SCD2 combos, + // but preview/test runs reach the executor without a deploy — re-check so a + // bad combo fails with the same clear message instead of a raw DuckDB error. + m.validate(partitioned).map_err(Error::ExecutionErr)?; let strategy = if m.scd2 { - // SCD2 needs a natural key to identify an entity across versions, and its - // diff/close/open shape has no partition-scoped form in v1. + // SCD2 needs a natural key to identify an entity across versions (checked + // by `validate` above, which guarantees a non-empty key here). let key = m.unique_key.clone().ok_or_else(|| { Error::ExecutionErr( "materialize scd2: requires a natural key — add `key=`".to_string(), ) })?; - if partitioned { - return Err(Error::ExecutionErr( - "materialize scd2: `// partitioned` is not supported with scd2 in v1".to_string(), - )); - } MaterializeStrategy::Scd2 { key, track: m.track.clone(), close_deleted: m.close_deleted } } else if m.append { MaterializeStrategy::Append @@ -3510,7 +3509,10 @@ mod tests { rewritten.contains("AS schema_drift"), "warn folds drift into the summary: {rewritten}" ); - assert!(!rewritten.contains("CAST(error(")); + // No schema-drift *fail* guard in warn mode (the keyed merge still emits + // its own duplicate-source-key guard, which is a different `error(...)`). + assert!(!rewritten.contains("on_schema_change=fail blocked")); + assert!(rewritten.contains("keyed merge on `id` blocked")); assert!(!rewritten.contains("BY NAME")); assert!(meta.sync_prepass.is_none()); } diff --git a/docs/ducklake-materialization.md b/docs/ducklake-materialization.md index 8f99b4c2c9..38c38a01f2 100644 --- a/docs/ducklake-materialization.md +++ b/docs/ducklake-materialization.md @@ -62,9 +62,18 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). error pointing to the `wmll.ducklake` helpers). - **`manual`** — escape hatch: the script writes its own DDL; Windmill only records state (no snapshot capture, no idempotency guarantee). Rare; explicit. -- **`key=`** → MERGE (dedup within slice, SCD type 1 — overwrites history); - **`append`** → INSERT-only; neither → DELETE-by-partition + INSERT (replace). - `append` wins over `key` if both are given (deploy warning). +- **`key=`** → MERGE (upsert-by-key within the slice, SCD type 1 — + overwrites history); **`append`** → INSERT-only; neither → DELETE-by-partition + + INSERT (replace). `append` wins over `key` if both are given (deploy warning). + - **Source must have unique keys.** The merge is delete-by-key + insert-all: it + reconciles the incoming rows against the *target* (rows whose key is in the + SELECT are replaced), but it does **not** deduplicate the *source*. Two + incoming rows sharing a key would therefore both land under that key, silently + breaking the "one row per key" contract. The managed write guards against this + — the run **fails** with a clear error when the SELECT returns more than one + row for a non-NULL key (`NULL` keys are exempt, matching the insert-only path). + Deduplicate in the SELECT (e.g. `QUALIFY row_number() OVER (PARTITION BY + ORDER BY DESC) = 1`) or use `append` if duplicates are intended. - **`key=history [track=] [deletes=close]`** → upgrades the keyed merge to managed SCD **type 2** history (the leading keyword `scd2` is an alias). The SELECT is the *current snapshot* (one row per `key`), and the runtime adds @@ -73,7 +82,9 @@ stays separate because it is cross-cutting (cascade + scheduling + materialize). history. Diff → close-old (`UPDATE`) → open-new (`INSERT`) in one transaction; the effective timestamp is the transaction clock (`now()`), so a run is self-consistent. v1 is **non-partitioned only** (`// partitioned` + history is - rejected). Unlike `manual`, it is managed, so `// data_test` and schema capture + rejected **at deploy**), and it **requires `key=`** (also rejected at deploy). + Both misconfigurations fail fast at save with a clear message instead of on the + first run. Unlike `manual`, it is managed, so `// data_test` and schema capture work. - **Deletes.** By default a key that disappears from the snapshot stays current (soft delete — dbt's `hard_deletes=ignore`). `deletes=close` opts into diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md index 126a4ac215..d00a78aa7f 100644 --- a/docs/pipelines-vs-dbt.md +++ b/docs/pipelines-vs-dbt.md @@ -137,7 +137,9 @@ schema capture, and each run (re)creates a `_current` view; consumers get effective-dated joins via native `ASOF JOIN … >= valid_from`. Deletes follow dbt: soft by default (absent keys stay current), or `deletes=close` for `hard_deletes=close`. v1 is non-partitioned; partitioned history is a follow-up. -See `ducklake-materialization.md`. +Both misconfigurations — `history` without `key=`, and `history` + `// +partitioned` — are rejected **at deploy** (fail-fast at save, not on the first +run). See `ducklake-materialization.md`. ### 5. Selective execution grammar @@ -434,6 +436,13 @@ DELETE-by-partition + INSERT works only if you reprocess from a source-of-truth source. If you're consuming amendments and need dedup *within* the slice, you need MERGE on `order_id`, not DELETE on partition. +Note the boundary: the keyed merge reconciles the incoming rows against the +*target* (delete-by-key + insert), but it does not deduplicate the *source*. +"Latest version per `order_id`" is still your SELECT's job — pick one row per +key (e.g. `QUALIFY row_number() OVER (PARTITION BY order_id ORDER BY updated_at +DESC) = 1`). A source that still has two rows for one key is rejected at run +time rather than double-written (see `ducklake-materialization.md`). + This is what dbt's `unique_key` does. Orthogonal to partitioning: `partitioned` answers "which slice?"; `unique_key` answers "how do I dedup inside the slice?" From 0ad174fa490e17eeb26280b5bdfd62956dfed9ff Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:29:35 +0200 Subject: [PATCH 17/34] feat(pipelines): self-teaching custom data_test errors + scaffold (#9937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Custom `// data_test ` scripts must be a single SELECT reading the freshly-materialized target via the internal `_wm_target.
` alias — neither was documented or scaffolded. Make the codegen errors name the exact violation (multi-statement, non-SELECT, wrong alias, empty) and append a copyable `SELECT * FROM _wm_target.
WHERE ` example. Add a DuckDB-only 'Data test' pipeline output kind that scaffolds that starter body. Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-parser/src/sql_materialize.rs | 136 ++++++++++++++++-- .../pipelineTemplates.dataTest.test.ts | 56 ++++++++ .../assets/AssetGraph/pipelineTemplates.ts | 48 ++++++- 3 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index dccf8ec313..1b24a5bccd 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -1248,6 +1248,28 @@ fn sample_star(ctx: &DataTestCtx, qualifier: Option<&str>) -> String { } } +// Self-teaching tail appended to every malformed-custom-test error. It states +// the two rules that aren't documented or scaffolded anywhere else — the body +// is a single SELECT, and it reads the freshly-materialized target through the +// internal `_wm_target.
` alias — and doubles that alias into a copyable +// one-line example. `target_qualified` is already `_wm_target.
`. +fn custom_test_hint(target_qualified: &str) -> String { + format!( + "Write a single SELECT against `{target_qualified}` returning the offending rows, e.g. \ + `SELECT * FROM {target_qualified} WHERE ` — an empty result means the test \ + passes." + ) +} + +// Whether a custom-test statement reads the materialized target through the +// reserved `_wm_target` alias (the only handle the runtime attaches it under). +// SQL identifiers are case-insensitive, so match case-insensitively; +// `split_statements` has already stripped comments, so a match here is a real +// reference, not one buried in a comment. `TARGET_ALIAS` is lowercase. +fn references_target(stmt: &str) -> bool { + stmt.to_lowercase().contains(TARGET_ALIAS) +} + /// Compile resolved data tests into ATTACH statements + per-test checks for /// `ctx`'s target. Pure: returns SQL text, executes nothing. Errors carry an /// actionable message (e.g. a relationships target that isn't an attachable @@ -1379,24 +1401,44 @@ pub fn build_data_test_checks( } DataTestResolved::Custom { path, body } => { // dbt singular-test convention: the body is a *single* SELECT - // (or CTE) returning the violating rows. It is embedded as a - // subquery (`FROM ()`), so a multi-statement body would - // produce invalid SQL — validate up front with an actionable - // error. It runs in the target's connection (can read - // `_wm_target` + the user's attaches); partition substitution is - // already applied by the worker. + // (or CTE) returning the violating rows, reading the + // freshly-materialized target through the internal `_wm_target` + // schema. It is embedded as a subquery (`FROM ()`), so a + // multi-statement or non-SELECT body would produce invalid SQL. + // Neither rule is documented or scaffolded elsewhere, so the + // errors are self-teaching: they name the exact violation and + // append a correct one-line example. It runs in the target's + // connection (can read `_wm_target` + the user's attaches); + // partition substitution is already applied by the worker. + let hint = custom_test_hint(t); let stmts = split_statements(body); if stmts.is_empty() { - return Err(format!("data_test custom `{path}`: empty test body")); + return Err(format!( + "data_test custom `{path}`: empty test body. {hint}" + )); } if stmts.len() > 1 { return Err(format!( - "data_test custom `{path}`: must be a single SELECT returning the \ - violating rows (found {} statements)", + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + but found {} statements. {hint}", stmts.len() )); } - push_check(&mut out, format!("custom({path})"), stmts[0].to_string()); + let stmt = &stmts[0]; + if classify_block(stmt) != BlockClass::Output { + return Err(format!( + "data_test custom `{path}`: a custom data test must be a single SELECT, \ + not a write or DDL statement. {hint}" + )); + } + if !references_target(stmt) { + return Err(format!( + "data_test custom `{path}`: the test never reads the freshly-materialized \ + target — reference it through the internal `{TARGET_ALIAS}` schema (as \ + `{t}`), not the table name on its own. {hint}" + )); + } + push_check(&mut out, format!("custom({path})"), stmt.to_string()); } } } @@ -2343,13 +2385,85 @@ mod tests { #[test] fn data_test_custom_rejects_multi_statement_body() { // The body is embedded as a subquery, so a setup-then-SELECT body would - // produce invalid SQL — reject it up front with an actionable error. + // produce invalid SQL — reject it up front with a self-teaching error + // that names the violation and shows the correct single-SELECT shape. let tests = vec![DataTestResolved::Custom { path: "f/tests/amount".into(), body: "SET threads = 1; SELECT * FROM _wm_target.orders WHERE amount < 0".into(), }]; let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("found 2 statements"), + "unexpected error: {err}" + ); + // the copyable example points at the internal target alias. + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_non_select_body() { + // A write/DDL body can't be embedded as `FROM ()`; the error must + // say so and teach the single-SELECT convention. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "DELETE FROM _wm_target.orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("single SELECT"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_rejects_wrong_target_alias() { + // Referencing the target by its bare table name (not `_wm_target.
`) + // is the most common custom-test mistake — the runtime only attaches the + // freshly-materialized target under `_wm_target`, so the query would fail + // at runtime. Catch it at codegen with a self-teaching error. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "SELECT * FROM orders WHERE amount < 0".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("_wm_target"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); + } + + #[test] + fn data_test_custom_accepts_from_first_and_uppercased_alias() { + // DuckDB's FROM-first syntax is a valid Output, and the alias match is + // case-insensitive (SQL identifiers are), so this passes. + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: "FROM _WM_TARGET.orders WHERE amount < 0".into(), + }]; + let sql = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap(); + assert!(sql.checks[0] + .probe + .contains("FROM (FROM _WM_TARGET.orders WHERE amount < 0) _wm_v")); + } + + #[test] + fn data_test_custom_empty_body_teaches_shape() { + let tests = vec![DataTestResolved::Custom { + path: "f/tests/amount".into(), + body: " \n-- just a comment\n".into(), + }]; + let err = build_data_test_checks(&tests, &ctx_unpartitioned()).unwrap_err(); + assert!(err.contains("empty test body"), "unexpected error: {err}"); + assert!( + err.contains("SELECT * FROM _wm_target.orders WHERE "), + "unexpected error: {err}" + ); } #[test] diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts new file mode 100644 index 0000000000..2acdf2abfb --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.dataTest.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from 'vitest' +import { + autoOutputAsset, + compatibleOutputKinds, + generatePipelineDraft, + PIPELINE_OUTPUT_KINDS +} from './pipelineTemplates' + +// The `data_test` output kind scaffolds a *custom* (singular) data test: a +// standalone DuckDB script referenced from a materialize script's +// `-- data_test ` line. It must be a single SELECT that reads the +// freshly-materialized target through the internal `_wm_target` schema — the +// two rules the backend's self-teaching errors enforce. +describe('data_test scaffold', () => { + it('is a DuckDB-only output kind exposed in the picker', () => { + expect(compatibleOutputKinds('duckdb')).toContain('data_test') + expect(compatibleOutputKinds('python3')).not.toContain('data_test') + expect(PIPELINE_OUTPUT_KINDS.map((k) => k.id)).toContain('data_test') + }) + + it('produces no output asset (it asserts against an existing target)', () => { + expect(autoOutputAsset('data_test', 'folder', 'duckdb')).toBeUndefined() + }) + + it('scaffolds a single SELECT against `_wm_target.
`', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + triggers: [] + }) + // starter body is a single SELECT against the internal target alias. + expect(src).toContain('SELECT * FROM _wm_target.your_table WHERE your_condition;') + // exactly one SQL statement (single SELECT) — count statement lines, not + // the word "SELECT" that also appears in the guidance comment. + const stmtLines = src.split('\n').filter((l) => /^\s*SELECT\b/i.test(l)) + expect(stmtLines).toHaveLength(1) + // no `-- materialize ` output annotation — a data test declares no + // asset (the word still appears in the guidance comment, which is fine). + expect(src).not.toMatch(/^--\s*materialize\s/m) + // teaches how to wire it up + the offending-rows convention. + expect(src).toContain('-- data_test ') + expect(src).toContain('offending rows') + }) + + it('seeds the table name from an upstream ducklake asset when present', () => { + const src = generatePipelineDraft({ + language: 'duckdb', + outputKind: 'data_test', + input: { kind: 'ducklake', path: 'analytics/orders' }, + triggers: [] + }) + expect(src).toContain('SELECT * FROM _wm_target.orders WHERE your_condition;') + // no ATTACH of the input — the runtime attaches the target as `_wm_target`. + expect(src).not.toContain('ATTACH') + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts index 11077d3609..f81d1a330d 100644 --- a/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts +++ b/frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts @@ -17,6 +17,7 @@ export type PipelineOutputKind = | 'datatable' | 'ducklake' | 'materialize' + | 'data_test' | 's3_parquet' | 's3_object' | 'macros' @@ -37,6 +38,11 @@ export const PIPELINE_OUTPUT_KINDS: PipelineOutputKindMeta[] = [ label: 'Materialized table', description: 'Managed DuckLake table — idempotent, versioned, tracked' }, + { + id: 'data_test', + label: 'Data test', + description: 'Custom assertion — a single SELECT returning offending rows (empty = pass)' + }, { id: 'datatable', label: 'Data table', @@ -83,7 +89,16 @@ const LANG_COMPATIBILITY: Record = { // single SELECT. The Python/TS `wmll.ducklake` helper currently takes a SQL // SELECT (not in-memory rows), so a polyglot managed materialize is a // separate follow-up — those langs keep the `ducklake` raw-write kind. - duckdb: ['materialize', 'datatable', 'ducklake', 's3_parquet', 's3_object', 'macros', 'none'], + duckdb: [ + 'materialize', + 'data_test', + 'datatable', + 'ducklake', + 's3_parquet', + 's3_object', + 'macros', + 'none' + ], postgresql: ['datatable', 'none'], mysql: ['none'], mssql: ['none'], @@ -183,8 +198,10 @@ export function autoOutputAsset( } } // A macro library produces no asset — its "output" is the registry - // entries the deploy records. + // entries the deploy records. A custom data test produces no asset + // either — it asserts against an existing materialized target. case 'macros': + case 'data_test': case 'none': return undefined } @@ -316,6 +333,17 @@ export type TemplateContext = { function header(ctx: TemplateContext): string { const { language, triggers, output, outputKind } = ctx const p = commentPrefix(language) + // A custom data test is a standalone script, not a graph node that produces + // an asset — so it gets no `// pipeline` / output annotation. Instead, tell + // the author how to wire it up (the `data_test` reference) and the two rules + // that aren't obvious: single SELECT, returning the offending rows. + if (outputKind === 'data_test') { + return [ + `${p} Custom data test — reference it from a materialize script with \`${p} data_test \`.`, + `${p} It must be a single SELECT returning the offending rows; the run fails if any row comes back.`, + '' + ].join('\n') + } const lines = triggers.map((t) => { switch (t.kind) { case 'asset': @@ -544,6 +572,22 @@ function bodyPython(ctx: TemplateContext): string { function bodyDuckdb(ctx: TemplateContext): string { const { input, output, outputKind } = ctx const dataUpload = isDataUpload(ctx.triggers) + if (outputKind === 'data_test') { + // Standalone custom data test: it reads ONLY the freshly-materialized + // target, which the runtime attaches under the internal `_wm_target` + // schema — so it emits no ATTACH / input load of its own. When the test + // was created off a ducklake asset, seed its table name; otherwise a + // clear placeholder. This exact shape (single SELECT vs `_wm_target`) is + // what the backend's self-teaching errors ask for. + const testTable = input?.kind === 'ducklake' ? catalogTableRef(input.path) : 'your_table' + return [ + '', + `-- Return the rows that VIOLATE your assertion; an empty result means the test passes.`, + `-- \`_wm_target\` is the freshly-materialized target, attached by the runtime.`, + `SELECT * FROM _wm_target.${testTable} WHERE your_condition;`, + '' + ].join('\n') + } const lines: string[] = [] if (dataUpload) { // `(s3object)` param declaration → the run form renders the S3 picker From 46be39dfb7fbfb2b70e61819d6065b45810c41c9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 5 Jul 2026 22:34:58 +0200 Subject: [PATCH 18/34] fix(pipelines): order data_test relationships refs before the tested script in a cascade (#9934) * fix(pipelines): order data_test relationships refs before the tested script in a cascade Co-Authored-By: Claude Opus 4.8 (1M context) * fix(pipelines): key custom-test reads by (usage_kind, path) to avoid same-path flow collisions Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- backend/windmill-api-assets/src/lib.rs | 118 ++++++++++++++++++ backend/windmill-api/openapi.yaml | 32 +++++ docs/ducklake-materialization.md | 34 +++++ .../assets/AssetGraph/AssetGraphCanvas.svelte | 27 ++++ .../assets/AssetGraph/boundedCascade.test.ts | 71 ++++++++++- .../assets/AssetGraph/boundedCascade.ts | 14 +++ .../lib/components/assets/AssetGraph/types.ts | 17 +++ 7 files changed, 311 insertions(+), 2 deletions(-) diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 2cebf9ed13..c0a93c039b 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -797,6 +797,25 @@ enum TriggerEdge { }, } +// Ordering-only "must-run-after" edge: `runnable_path`'s data test reads +// `asset` (a `// data_test relationships` ref, or a custom test whose body +// reads a known pipeline asset), so the asset's in-pipeline producer must +// materialize before `runnable_path` runs. NOT a data-consumption edge — the +// tested script doesn't ingest the asset's rows, it only needs the table to +// exist at test time. Rendered dashed (like macro edges) and fed into the +// cascade topo-sort so a cold cascade orders the referenced dimension first. +// Only emitted when the referenced asset has a producer in the graph; an +// external table (no producer) adds no edge — the runtime error stands. +#[derive(Serialize, Debug)] +struct TestEdge { + producer_kind: AssetUsageKind, + producer_path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + asset_kind: AssetKind, + asset_path: String, +} + #[derive(Serialize, Debug)] struct AssetGraphResponse { assets: Vec, @@ -805,6 +824,8 @@ struct AssetGraphResponse { triggers: Vec, #[serde(skip_serializing_if = "Vec::is_empty", default)] macro_edges: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + test_edges: Vec, } async fn asset_graph( @@ -1257,6 +1278,102 @@ async fn asset_graph( runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); } + // Data-test ordering edges. A `// data_test relationships -> ` + // (and, best-effort, a custom `// data_test + {#if data.fork_materialization === 'deferred'} - + + parent {:else if data.fork_materialization === 'fork'} - + + fork {/if} +
+ +
+ {/if} {#if showActions} + + {/if} {#if mode === 'edit' && saveErrors.size > 0} + {#if calendarPicker && materializeTarget?.kind === 'ducklake' && value} + {#if selectedMaterialized} + + + {value} is already materialized — running replaces it. + + {:else} + + + {value} not materialized yet — running creates it. + + {/if} + {/if} + + + {#if upstreamMissing} + + + + No upstream data for {value} yet — this run may materialize an + empty partition. + + + {/if} + + + {#if recentlyMissing.length > 0} +
+ + {recentlyMissing.length} of the last {recentWindow(spec.kind)} + {spec.kind} partitions are not materialized: + +
+ {#each recentlyMissing.slice(0, MAX_MISSING_CHIPS) as b (b)} + + {/each} + {#if recentlyMissing.length > MAX_MISSING_CHIPS} + + +{recentlyMissing.length - MAX_MISSING_CHIPS} more + + {/if} +
+
+ {/if} + diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte index fc5e7ad945..4384f125ba 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineRunForm.svelte @@ -1,6 +1,9 @@ - +
+ {#if partitionSpec} + args.partition, (v) => (args.partition = v)} + {workspace} + {materializeTarget} + {upstreamAssets} + /> + {/if} + +
diff --git a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte index cadd3c6a94..64e24e0ad2 100644 --- a/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte @@ -1,11 +1,13 @@ + +{#if show} +
+
+ +
+

Finish setting up pipelines

+

+ Pipelines materialize data into DuckLake tables backed by object storage. Configure the + following before your first pipeline can run. +

+
+
+ +
    + {#each steps as step (step.title)} + {@const Icon = step.icon} +
  • + {#if step.done === true} + + {:else if step.done === false} + + {:else} + + {/if} +
    + {step.title} + {step.description} +
    + {#if step.done !== true} + + {step.cta} + + + {:else} + Configured + {/if} +
    • + {/each} +
+
+{/if} diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts new file mode 100644 index 0000000000..a8a47af259 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.test.ts @@ -0,0 +1,161 @@ +import { describe, expect, it } from 'vitest' +import { + bucketFor, + bucketFromInputValue, + defaultBucket, + inputValueFromBucket, + isBeforeStart, + isValidStart, + isValidTimeZone, + partitionInputType, + partitionMetadataError, + recentBuckets, + startBucketOf, + usesCalendarPicker +} from './partitionBuckets' +import type { PartitionSpec } from './parsePipelineAnnotations' + +const spec = (kind: PartitionSpec['kind'], extra: Partial = {}): PartitionSpec => + ({ kind, ...extra }) as PartitionSpec + +// A fixed UTC instant: 2026-07-05 14:37Z (a Sunday — ISO week 27 of 2026). +// Explicit UTC so the assertions are independent of the test runner's TZ, and +// they exercise the same UTC default the backend uses when `spec.tz` is absent. +const at = new Date(Date.UTC(2026, 6, 5, 14, 37, 0)) + +describe('bucketFor mirrors backend default_format (UTC)', () => { + it('daily -> %Y-%m-%d', () => { + expect(bucketFor(spec('daily'), at)).toBe('2026-07-05') + }) + it('hourly -> %Y-%m-%dT%H', () => { + expect(bucketFor(spec('hourly'), at)).toBe('2026-07-05T14') + }) + it('monthly -> %Y-%m', () => { + expect(bucketFor(spec('monthly'), at)).toBe('2026-07') + }) + it('weekly -> ISO %G-W%V', () => { + expect(bucketFor(spec('weekly'), at)).toBe('2026-W27') + }) +}) + +describe('bucketFor honours spec.tz', () => { + // 2026-07-05 02:30Z is still 2026-07-04 in America/New_York (UTC-4 in July). + const nearMidnight = new Date(Date.UTC(2026, 6, 5, 2, 30, 0)) + it('shifts the day boundary by the producer tz', () => { + expect(bucketFor(spec('daily'), nearMidnight)).toBe('2026-07-05') // UTC default + expect(bucketFor(spec('daily', { tz: 'America/New_York' }), nearMidnight)).toBe('2026-07-04') + }) + it('shifts the hour bucket by the producer tz', () => { + // 02:30Z -> 22 (previous day) in New York. + expect(bucketFor(spec('hourly', { tz: 'America/New_York' }), nearMidnight)).toBe( + '2026-07-04T22' + ) + }) +}) + +describe('defaultBucket honours the start= anchor', () => { + // `at` is 2026-07-05. + it('seeds the current bucket when at or after start', () => { + expect(defaultBucket(spec('daily'), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-01-01' }), at)).toBe('2026-07-05') + expect(defaultBucket(spec('daily', { start: '2026-07-05' }), at)).toBe('2026-07-05') // == start, not before + }) + it('seeds the start bucket (never a pre-start one) when before start', () => { + expect(defaultBucket(spec('daily', { start: '2026-08-01' }), at)).toBe('2026-08-01') + expect(defaultBucket(spec('monthly', { start: '2026-08-01' }), at)).toBe('2026-08') + expect(defaultBucket(spec('hourly', { start: '2026-08-01' }), at)).toBe('2026-08-01T00') + }) + it('isBeforeStart mirrors the backend date comparison', () => { + expect(isBeforeStart(spec('daily', { start: '2026-08-01' }), at)).toBe(true) + expect(isBeforeStart(spec('daily', { start: '2026-07-05' }), at)).toBe(false) + expect(isBeforeStart(spec('daily'), at)).toBe(false) + }) + it('startBucketOf renders the anchor in the cadence, undefined when unset', () => { + expect(startBucketOf(spec('daily', { start: '2026-08-01' }))).toBe('2026-08-01') + expect(startBucketOf(spec('monthly', { start: '2026-08-15' }))).toBe('2026-08') + expect(startBucketOf(spec('hourly', { start: '2026-08-01' }))).toBe('2026-08-01T00') + expect(startBucketOf(spec('daily'))).toBeUndefined() + }) +}) + +describe('malformed metadata fails safe (parity with backend validation)', () => { + it('isValidTimeZone rejects garbage, accepts real zones and absence', () => { + expect(isValidTimeZone(undefined)).toBe(true) + expect(isValidTimeZone('UTC')).toBe(true) + expect(isValidTimeZone('America/New_York')).toBe(true) + expect(isValidTimeZone('Not/AZone')).toBe(false) + expect(isValidTimeZone('garbage')).toBe(false) + }) + it('isValidStart rejects malformed and JS-normalized dates', () => { + expect(isValidStart(undefined)).toBe(true) + expect(isValidStart('2026-08-01')).toBe(true) + expect(isValidStart('2026-02-31')).toBe(false) // JS would roll to Mar 3 + expect(isValidStart('2026-13-01')).toBe(false) + expect(isValidStart('08/01/2026')).toBe(false) + }) + it('partitionMetadataError reports the first problem, else undefined', () => { + expect(partitionMetadataError(spec('daily'))).toBeUndefined() + expect( + partitionMetadataError(spec('daily', { tz: 'America/New_York', start: '2026-08-01' })) + ).toBeUndefined() + expect(partitionMetadataError(spec('daily', { tz: 'Not/AZone' }))).toContain('timezone') + expect(partitionMetadataError(spec('daily', { start: '2026-02-31' }))).toContain('start date') + }) + it('bucketFor never throws on an invalid tz (falls back to UTC)', () => { + expect(bucketFor(spec('daily', { tz: 'Not/AZone' }), at)).toBe('2026-07-05') + }) + it('an invalid start is treated as no anchor (isBeforeStart/startBucketOf)', () => { + expect(isBeforeStart(spec('daily', { start: '2026-02-31' }), at)).toBe(false) + expect(startBucketOf(spec('daily', { start: '2026-02-31' }))).toBeUndefined() + }) +}) + +describe('partitionInputType', () => { + it('maps each calendar kind to its native input', () => { + expect(partitionInputType(spec('daily'))).toBe('date') + expect(partitionInputType(spec('hourly'))).toBe('datetime-local') + expect(partitionInputType(spec('weekly'))).toBe('week') + expect(partitionInputType(spec('monthly'))).toBe('month') + }) + it('falls back to text for dynamic and custom-format specs', () => { + expect(partitionInputType(spec('dynamic', { key: '$.tenant' }))).toBe('text') + expect(partitionInputType(spec('daily', { format: '%Y/%m/%d' }))).toBe('text') + expect(usesCalendarPicker(spec('dynamic', { key: '$.tenant' }))).toBe(false) + expect(usesCalendarPicker(spec('daily', { format: '%Y/%m/%d' }))).toBe(false) + }) +}) + +describe('native input <-> bucket round-trip', () => { + it('hourly truncates the datetime-local minutes and restores :00', () => { + expect(bucketFromInputValue(spec('hourly'), '2026-07-05T14:37')).toBe('2026-07-05T14') + expect(inputValueFromBucket(spec('hourly'), '2026-07-05T14')).toBe('2026-07-05T14:00') + }) + it('non-hourly kinds pass through unchanged', () => { + expect(bucketFromInputValue(spec('daily'), '2026-07-05')).toBe('2026-07-05') + expect(inputValueFromBucket(spec('weekly'), '2026-W27')).toBe('2026-W27') + }) + it('empty stays empty', () => { + expect(bucketFromInputValue(spec('hourly'), '')).toBe('') + expect(inputValueFromBucket(spec('daily'), '')).toBe('') + }) +}) + +describe('recentBuckets (UTC)', () => { + it('walks back day by day, most recent first', () => { + expect(recentBuckets(spec('daily'), at, 3)).toEqual(['2026-07-05', '2026-07-04', '2026-07-03']) + }) + it('walks back hour by hour', () => { + expect(recentBuckets(spec('hourly'), at, 3)).toEqual([ + '2026-07-05T14', + '2026-07-05T13', + '2026-07-05T12' + ]) + }) + it('walks back month by month across a year boundary without day roll-over', () => { + const jan31 = new Date(Date.UTC(2026, 0, 31, 0, 0, 0)) + expect(recentBuckets(spec('monthly'), jan31, 3)).toEqual(['2026-01', '2025-12', '2025-11']) + }) + it('walks back week by week', () => { + expect(recentBuckets(spec('weekly'), at, 3)).toEqual(['2026-W27', '2026-W26', '2026-W25']) + }) +}) diff --git a/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts new file mode 100644 index 0000000000..401ef82cae --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/partitionBuckets.ts @@ -0,0 +1,261 @@ +// Client-side partition-bucket math for the run form's partition picker. +// Produces the same canonical bucket strings the backend renders in +// `windmill-common/src/partition_ee.rs` (`resolve_time_partition` / +// `default_format`): the instant is localized to the spec's timezone +// (`spec.tz`, defaulting to UTC — NOT the browser's zone) and then formatted. +// Getting the zone right matters — a browser in a non-UTC zone would otherwise +// seed a default bucket, and compare against materialized rows, off by a +// day/hour near every boundary and always for an explicit `tz=` spec. +// +// daily %Y-%m-%d -> 2026-07-05 +// hourly %Y-%m-%dT%H -> 2026-07-05T14 +// weekly %G-W%V (ISO) -> 2026-W27 +// monthly %Y-%m -> 2026-07 +// +// Pure module (no Svelte runes) so the mapping is unit-testable. + +import type { PartitionSpec } from './parsePipelineAnnotations' + +export type PartitionInputType = 'date' | 'month' | 'week' | 'datetime-local' | 'text' + +// A spec carrying a custom strftime `format` can't be reproduced by the native +// date pickers (arbitrary strftime), and `dynamic` partitions are a free-form +// key extracted from the payload — both fall back to a plain text input. +export function usesCalendarPicker(spec: PartitionSpec): boolean { + return spec.kind !== 'dynamic' && !spec.format +} + +export function partitionInputType(spec: PartitionSpec): PartitionInputType { + if (!usesCalendarPicker(spec)) return 'text' + switch (spec.kind) { + case 'monthly': + return 'month' + case 'weekly': + return 'week' + case 'hourly': + return 'datetime-local' + default: + return 'date' + } +} + +function pad(n: number): string { + return String(n).padStart(2, '0') +} + +const ZONED_PARTS_OPTS: Intl.DateTimeFormatOptions = { + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + second: '2-digit', + hourCycle: 'h23' +} + +// Re-express `at` as a Date whose UTC fields equal the wall-clock in `tz`, so +// all downstream field reads / calendar arithmetic can use the UTC getters and +// stay in the producer's zone. `hourCycle: 'h23'` keeps hours 00–23. A +// malformed `tz` (rejected by `Intl`) falls back to UTC rather than throwing — +// the backend validates `tz=` and is the source of truth for the error; the +// picker must never crash the graph view (`partitionMetadataError` gates +// auto-seeding so a bogus bucket isn't silently sent). +function zonedAsUtc(at: Date, tz: string): Date { + let parts: Intl.DateTimeFormatPart[] + try { + parts = new Intl.DateTimeFormat('en-US', { ...ZONED_PARTS_OPTS, timeZone: tz }).formatToParts( + at + ) + } catch { + parts = new Intl.DateTimeFormat('en-US', { + ...ZONED_PARTS_OPTS, + timeZone: 'UTC' + }).formatToParts(at) + } + const g = (t: string) => Number(parts.find((p) => p.type === t)?.value) + return new Date( + Date.UTC(g('year'), g('month') - 1, g('day'), g('hour'), g('minute'), g('second')) + ) +} + +// `tz=` is valid iff `Intl` accepts it (absent === UTC === valid). +export function isValidTimeZone(tz?: string): boolean { + if (!tz) return true + try { + new Intl.DateTimeFormat('en-US', { timeZone: tz }) + return true + } catch { + return false + } +} + +// `start=` is valid iff it's a real `YYYY-MM-DD` calendar date. The round-trip +// check rejects dates JS would silently normalize (e.g. `2026-02-31` → Mar 3), +// which the backend's `NaiveDate::parse_from_str` also rejects. +export function isValidStart(start?: string): boolean { + if (!start) return true + const m = start.match(/^(\d{4})-(\d{2})-(\d{2})$/) + if (!m) return false + const y = Number(m[1]) + const mo = Number(m[2]) + const d = Number(m[3]) + const dt = new Date(Date.UTC(y, mo - 1, d)) + return dt.getUTCFullYear() === y && dt.getUTCMonth() === mo - 1 && dt.getUTCDate() === d +} + +// A human-readable reason the `// partitioned` metadata is unusable, or +// undefined when it's sound. Mirrors the backend's `tz=` / `start=` validation +// so the picker can refuse to auto-seed a bucket the backend would reject. +export function partitionMetadataError(spec: PartitionSpec): string | undefined { + if (!isValidTimeZone(spec.tz)) return `invalid timezone "${spec.tz}"` + if (!isValidStart(spec.start)) return `invalid start date "${spec.start}" (want YYYY-MM-DD)` + return undefined +} + +// ISO 8601 week-numbering year + week (chrono's %G / %V) of a UTC-substituted +// date. Standard "nearest Thursday" algorithm, all in UTC. +function isoWeekOf(d: Date): { isoYear: number; week: number } { + const date = new Date(Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate())) + const dayNum = (date.getUTCDay() + 6) % 7 // Mon=0 … Sun=6 + date.setUTCDate(date.getUTCDate() - dayNum + 3) // Thursday of this week + const isoYear = date.getUTCFullYear() + const firstThursday = new Date(Date.UTC(isoYear, 0, 4)) + const fdNum = (firstThursday.getUTCDay() + 6) % 7 + firstThursday.setUTCDate(firstThursday.getUTCDate() - fdNum + 3) + const week = 1 + Math.round((date.getTime() - firstThursday.getTime()) / (7 * 86400000)) + return { isoYear, week } +} + +// Render a UTC-substituted date into its canonical bucket for the cadence. +function fmtBucket(kind: PartitionSpec['kind'], d: Date): string { + const y = d.getUTCFullYear() + const m = d.getUTCMonth() + 1 + const day = d.getUTCDate() + const h = d.getUTCHours() + switch (kind) { + case 'hourly': + return `${y}-${pad(m)}-${pad(day)}T${pad(h)}` + case 'monthly': + return `${y}-${pad(m)}` + case 'weekly': { + const { isoYear, week } = isoWeekOf(d) + return `${isoYear}-W${pad(week)}` + } + default: + return `${y}-${pad(m)}-${pad(day)}` + } +} + +export function bucketFor(spec: PartitionSpec, at: Date): string { + return fmtBucket(spec.kind, zonedAsUtc(at, spec.tz ?? 'UTC')) +} + +// The zoned start date (`spec.start`, `YYYY-MM-DD`) as a UTC-substituted Date at +// 00:00, or undefined if unset/malformed. `start` is a plain date in the +// producer's tz — the backend parses it as a NaiveDate and compares by date. +function startDate(spec: PartitionSpec): Date | undefined { + if (!spec.start || !isValidStart(spec.start)) return undefined + const m = spec.start.match(/^(\d{4})-(\d{2})-(\d{2})$/)! + return new Date(Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]))) +} + +// True when `at` (localized to the producer tz) falls on a date before the +// `start=` anchor — exactly the backend's `local.date_naive() < start_date` +// check (`resolve_time_partition`), which resolves such an instant to no +// partition. +export function isBeforeStart(spec: PartitionSpec, at: Date): boolean { + const start = startDate(spec) + if (!start) return false + const zoned = zonedAsUtc(at, spec.tz ?? 'UTC') + const zonedDate = Date.UTC(zoned.getUTCFullYear(), zoned.getUTCMonth(), zoned.getUTCDate()) + return zonedDate < start.getTime() +} + +// The canonical bucket of the `start=` anchor (its date at 00:00), or undefined +// if unset. Buckets sort lexicographically within a cadence, so callers can +// compare against it to drop pre-start buckets. +export function startBucketOf(spec: PartitionSpec): string | undefined { + const start = startDate(spec) + return start ? fmtBucket(spec.kind, start) : undefined +} + +// The bucket to pre-fill the picker with. Normally the current bucket (matching +// the backend's "absent partition arg -> current bucket" resolution), but when +// the current instant is before the `start=` anchor the backend would resolve +// to NO partition — so default to the first valid bucket (the start) rather +// than a pre-start one the worker would take verbatim and materialize early. +export function defaultBucket(spec: PartitionSpec, at: Date): string { + if (isBeforeStart(spec, at)) { + const start = startDate(spec) + if (start) return fmtBucket(spec.kind, start) + } + return bucketFor(spec, at) +} + +// Native input value -> canonical bucket. Only hourly differs: datetime-local +// carries a minute component the hourly bucket drops. The picked wall-clock is +// taken verbatim as the bucket (the user picks in the producer's frame), so no +// timezone conversion happens here. +export function bucketFromInputValue(spec: PartitionSpec, inputValue: string): string { + if (!inputValue) return '' + if (spec.kind === 'hourly') { + const m = inputValue.match(/^(\d{4}-\d{2}-\d{2}T\d{2})/) + return m ? m[1] : inputValue + } + return inputValue +} + +// Canonical bucket -> native input value. Only hourly differs: datetime-local +// needs a minute component the bucket omits. +export function inputValueFromBucket(spec: PartitionSpec, bucket: string): string { + if (!bucket) return '' + if (spec.kind === 'hourly') { + return /T\d{2}$/.test(bucket) ? `${bucket}:00` : bucket + } + return bucket +} + +// The last `count` buckets ending at (and including) `now`, most-recent first, +// localized to `spec.tz`. Arithmetic walks calendar units in the zoned frame, +// so it's exact across DST (no ±1 drift). Undefined for non-calendar specs +// (the caller guards on `usesCalendarPicker`). +export function recentBuckets(spec: PartitionSpec, now: Date, count: number): string[] { + const base = zonedAsUtc(now, spec.tz ?? 'UTC') + const out: string[] = [] + for (let i = 0; i < count; i++) { + const d = new Date(base) + switch (spec.kind) { + case 'hourly': + d.setUTCHours(d.getUTCHours() - i) + break + case 'weekly': + d.setUTCDate(d.getUTCDate() - 7 * i) + break + case 'monthly': + // Normalize to the 1st first so subtracting months can't roll over + // a short target month (e.g. Mar 31 − 1mo → Mar 3). + d.setUTCDate(1) + d.setUTCMonth(d.getUTCMonth() - i) + break + default: + d.setUTCDate(d.getUTCDate() - i) + } + out.push(fmtBucket(spec.kind, d)) + } + return out +} + +// How many recent buckets the "missing partitions" hint scans, per kind — a +// window that reads as "recent" for each cadence without flooding the hint. +export function recentWindow(kind: PartitionSpec['kind']): number { + switch (kind) { + case 'hourly': + return 24 + case 'weekly': + return 8 + case 'monthly': + return 6 + default: + return 14 + } +} diff --git a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte index ab36b4c1f7..e15c3abdf7 100644 --- a/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/pipeline/+page.svelte @@ -1,6 +1,7 @@ +{#snippet circleStack()} + + + {#each visibleUsers as u, i (i)} + + {initials(u)} + + {/each} + {#if overflowCount > 0} + + +{overflowCount} + + {/if} + +{/snippet} + {#if showBadge} @@ -204,32 +233,23 @@ bind:isOpen={popoverOpen} > {#snippet trigger()} - - {#if orderedUsers.length > 0} - - - {#each visibleUsers as u, i (i)} - - {initials(u)} - - {/each} - {#if overflowCount > 0} - - +{overflowCount} - - {/if} - - {/if} - {draft_only ? 'Draft only' : 'Draft'} - + {#if iconOnly} + + + + {#if orderedUsers.length > 0} + {@render circleStack()} + {/if} + + {:else} + + {#if orderedUsers.length > 0} + {@render circleStack()} + {/if} + {draft_only ? 'Draft only' : 'Draft'} + + {/if} {/snippet} {#snippet content()}
diff --git a/frontend/src/lib/components/common/badge/Badge.svelte b/frontend/src/lib/components/common/badge/Badge.svelte index cf7a7cad66..c57d41029c 100644 --- a/frontend/src/lib/components/common/badge/Badge.svelte +++ b/frontend/src/lib/components/common/badge/Badge.svelte @@ -90,7 +90,7 @@ const hovers: Partial> = { gray: 'hover:bg-surface-hover', - blue: 'hover:bg-blue-200 dark:hover:bg-blue-700/40', + blue: 'hover:bg-blue-100 dark:hover:bg-blue-700/60', red: 'hover:bg-red-200 dark:hover:bg-red-500/25', green: 'hover:bg-green-200 dark:hover:bg-green-500/25', yellow: 'hover:bg-yellow-200 dark:hover:bg-yellow-500/25', diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 5f5835bd13..c67bcd17b5 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -44,6 +44,9 @@ import { buildSummaryMessageContent } from './compactionPrompt' import { dfs } from '$lib/components/flows/previousResults' +import { SvelteSet } from 'svelte/reactivity' +import type { UserDraftItemKind } from '$lib/gen' +import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' import { chatDraft, expanded } from './chatDraft' @@ -343,6 +346,75 @@ export class AIChatManager { // session rather than the UI-active one — keeps backgrounded sessions isolated. sessionId: string | undefined = undefined + // Fired whenever the active chat id changes away from the one the consumer + // knows (a "/clear" rotation or a history switch). Session runtimes wire this + // to keep the session record's chatId aligned — the compare-page handoff + // (`from_session`) reads it, and a stale id would preselect the previous + // chat's items. Set here (not imported) to avoid a copilot→sessions cycle. + onChatRotated: ((chatId: string) => void) | undefined = undefined + + // Workspace items the CURRENT chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys (see modifiedItemsMask.ts). + // undefined = untracked: the global side-panel chat (never initialised) and + // loaded legacy chats with no stored mask, both of which fall back to the + // show-all bar. A SvelteSet (even empty) = tracked. Reactive so the session + // bar updates as tools record mid-turn. + modifiedItems = $state | undefined>(undefined) + + // Start tracking for a brand-new session chat (empty = "tracked, nothing yet"). + initModifiedItemsTracking() { + this.modifiedItems = new SvelteSet() + } + + // Record an item an AI tool call created/edited/deleted. No-op when untracked + // (the global singleton never initialises the set), so it stays unaffected. + recordModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + this.modifiedItems?.add(maskKey(itemKind, storagePath)) + } + + // Un-record an item whose chat-made change was discarded — without this the + // still-existing deployed item would keep reading as this chat's "Deployed" + // edit. Persisted immediately: unlike recordModifiedItem (whose persistence + // rides on the turn's saveChat), a discard can fire from the review dock + // outside any turn, and waiting would resurrect the entry on reload. + async removeModifiedItem(itemKind: UserDraftItemKind, storagePath: string) { + if (!this.modifiedItems?.delete(maskKey(itemKind, storagePath))) return + await this.#persistModifiedItems() + } + + // Move a mask entry to the path a draft actually deployed to. A draft-only + // flow/app parks at a synthetic `draft_{uuid}` storage path and deploys to + // its chosen path — without the move, the existence check at the synthetic + // path fails after reload and the deployed row vanishes from the dock. + async renameModifiedItem(itemKind: UserDraftItemKind, fromPath: string, toPath: string) { + if (fromPath === toPath) return + if (!this.modifiedItems?.delete(maskKey(itemKind, fromPath))) return + this.modifiedItems.add(maskKey(itemKind, toPath)) + await this.#persistModifiedItems() + } + + // Serialized, snapshot-at-write-time persistence: two rapid dock actions + // would otherwise race their saveChat writes, and the earlier (staler) + // snapshot could land last — dropping the later mutation until the next + // turn-end save. + #maskPersistQueue: Promise = Promise.resolve() + #persistModifiedItems(): Promise { + this.#maskPersistQueue = this.#maskPersistQueue.then(() => + this.historyManager + .saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) + // Swallow (and log) a failed write so it can't wedge the queue as a + // rejected link — the next persist snapshots the full current set, so + // a lost write self-heals on the next mutation or turn-end save. + .catch((e) => console.error('Failed to persist modified-items mask', e)) + ) + return this.#maskPersistQueue + } + // Workspace AI skills (name + description) advertised in the GLOBAL system // prompt and surfaced as slash commands in session chat. Loaded // asynchronously when entering GLOBAL mode; the system message is rebuilt @@ -659,7 +731,12 @@ export class AIChatManager { ) switch (result) { case 'ok': - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) sendUserToast('Conversation compacted.') break case 'empty': @@ -1636,7 +1713,12 @@ export class AIChatManager { const projectedContextTokens = this.contextTokens + this.estimateMessagesTokens([userMessage]) this.messages.push(userMessage) - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.currentReply = '' this.currentReasoning = '' @@ -1672,7 +1754,12 @@ export class AIChatManager { this.contextUsage = Math.max(0, this.contextUsage - freed) } } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } } // Rollback anchors for restoreUnsentTurn: captured after compaction so @@ -1758,7 +1845,10 @@ export class AIChatManager { }, requestConfirmation: this.requestConfirmation, shouldAutoAcceptToolConfirmations: () => this.autoAcceptToolConfirmationsActive, - requestUserQuestion: this.requestUserQuestion + requestUserQuestion: this.requestUserQuestion, + onItemModified: (kind, path) => this.recordModifiedItem(kind, path), + onItemDeployed: (kind, from, to) => void this.renameModifiedItem(kind, from, to), + onItemDiscarded: (kind, path) => void this.removeModifiedItem(kind, path) } } @@ -1794,7 +1884,12 @@ export class AIChatManager { this.contextUsage = result?.lastIterationUsage ? result.lastIterationUsage.prompt + result.lastIterationUsage.completion : undefined - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Still counts as the saved first turn — skipping the hook here would // permanently miss it (the next turn isn't "first" anymore). if (isFirstUserTurn && this.afterFirstTurnSaved) { @@ -1824,7 +1919,12 @@ export class AIChatManager { // user message on reload. Remove it instead. this.historyManager.deletePastChat(this.historyManager.getCurrentChatId()) } else { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } if (!wasAborted) { sendUserToast('The model returned no response — your message was restored to the input.') @@ -1843,7 +1943,12 @@ export class AIChatManager { if (this.autoAcceptEditsActive) { this.acceptPendingFlowEdits() } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) // Only this branch is a clean send: the queued-message flush below // auto-sends the next message after it (set after saveChat so a // persistence failure falls through to the restore path instead). @@ -1867,7 +1972,12 @@ export class AIChatManager { // compaction on the next send instead of failing the same way again. this.contextUsage = undefined try { - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.saveChat( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) } catch (saveErr) { console.error('Failed to persist partial chat after error', saveErr) } @@ -1996,15 +2106,25 @@ export class AIChatManager { // Drop any message queued in this conversation so it can't auto-send into // the fresh chat or linger as a card across the switch. this.queuedMessage = '' - await this.historyManager.save(this.displayMessages, this.messages, this.contextUsage) + await this.historyManager.save( + this.displayMessages, + this.messages, + this.contextUsage, + this.modifiedItems ? [...this.modifiedItems] : undefined + ) this.displayMessages = [] this.messages = [] this.contextUsage = undefined + // The mask belongs to the conversation just saved — the fresh chat starts + // its own (empty) tracking; carrying entries over would claim the previous + // conversation's edits for the new one. Untracked chats stay untracked. + if (this.modifiedItems) this.modifiedItems = new SvelteSet() // In an AI session, linked files are session-scoped: they persist across conversations // (cleared only when the session is deleted). The ephemeral global side-panel chat has no // session, so "New chat" must clear them — otherwise the next, unrelated conversation // would still get the previous file roster and could read/search it. if (!this.isSessionChat) this.attachedFiles.clear() + this.onChatRotated?.(this.historyManager.getCurrentChatId()) } loadPastChat = async (id: string) => { @@ -2019,7 +2139,16 @@ export class AIChatManager { this.displayMessages = chat.displayMessages this.messages = chat.actualMessages this.contextUsage = normalizeContextUsage(chat.contextUsage) + // Seed the modified-items mask from the stored chat. A stored array + // (even empty) → tracked; a legacy chat with no field stays untracked + // (undefined) so the session bar falls back to showing all drafts. The + // global side-panel chat never tracks, so leave it untouched there. + if (this.isSessionChat) { + const stored = this.historyManager.getModifiedItems(id) + this.modifiedItems = stored !== undefined ? new SvelteSet(stored) : undefined + } this.#automaticScroll = true + this.onChatRotated?.(id) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index d23cbf96a1..a415377a7b 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -693,7 +693,9 @@ describe('AIChatManager context compaction', () => { expect(manager.messages[0]).toMatchObject({ role: 'user', content: 'c'.repeat(400) }) // Mid-turn, the report is debited by the freed estimate (visible in the // compaction-time save) so a rolled-back turn keeps a consistent value - expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000) + // 4th arg: the modified-items mask rides on every save (undefined here — + // this bare manager never initialised tracking). + expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000, undefined) // At commit, the no-report turn clears the stored value; the readable // number falls back to estimating the now-tiny compacted history expect(manager.contextUsage).toBeUndefined() diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 9173e4c441..0db0751293 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -25,6 +25,12 @@ interface ChatSchema extends IDBSchema { // New writes store the plain reported token count; chats persisted by // earlier versions may still hold the legacy anchor object. contextUsage?: PersistedContextUsage + // Workspace items this chat modified via AI tool calls, as + // `${UserDraftItemKind}:${storagePath}` keys. Persisted out-of-band from + // the message arrays so it survives compaction. Absent (undefined) on + // chats predating this feature → consumers fall back to showing all + // workspace drafts; a defined array (even empty) means "tracked". + modifiedItems?: string[] } } } @@ -80,6 +86,29 @@ export function __resetLegacyChatClaimForTesting(): void { legacyChatClaim = undefined } +// Read a chat's modified-items mask by chatId WITHOUT mounting an AIChatManager, +// for the standalone /forks/compare route. Returns undefined for a legacy chat +// (no field) so the page falls back to selecting all items; a defined array +// (even empty) narrows the preselection. Opens a throwaway user-scoped handle; +// the `get` is O(1) on the `id` keyPath. +export async function readChatModifiedItems(chatId: string): Promise { + const dbh = userScopedDb(DB_NAME, { + version: 1, + upgrade: createChatStore, + migrate: migrateLegacyChatDb + }) + try { + const db = await dbh.whenReady() + const chat = await db?.get('chats', chatId) + return chat?.modifiedItems + } catch (err) { + console.error('Could not read chat modified items', err) + return undefined + } finally { + dbh.close() + } +} + export default class HistoryManager { // Per-instance handle to the shared per-user DB lifecycle. There is one // HistoryManager per AIChatManager (the singleton + one per session runtime), @@ -100,6 +129,7 @@ export default class HistoryManager { lastModified: number sessionId?: string contextUsage?: PersistedContextUsage + modifiedItems?: string[] } > = $state({}) @@ -173,10 +203,15 @@ export default class HistoryManager { return Object.values(this.savedChats) } + getModifiedItems(id: string): string[] | undefined { + return this.savedChats[id]?.modifiedItems + } + async saveChat( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { if (displayMessages.length > 0) { // Compaction replaces the original first message with a summary boundary. @@ -203,7 +238,18 @@ export default class HistoryManager { id: this.currentChatId, lastModified: Date.now(), ...(this.sessionId ? { sessionId: this.sessionId } : {}), - ...(contextUsage !== undefined ? { contextUsage } : {}) + ...(contextUsage !== undefined ? { contextUsage } : {}), + // Only persist when the caller passes a defined array. Loaded legacy + // chats keep their accumulator undefined, so we never retroactively + // stamp them with [] (which would flip them to the filtered view). + // But since `put` replaces the whole record, a caller that omits the + // argument must not ERASE a tracked chat's stored mask — fall back to + // the previously saved field. + ...(modifiedItems !== undefined + ? { modifiedItems } + : this.savedChats[this.currentChatId]?.modifiedItems !== undefined + ? { modifiedItems: this.savedChats[this.currentChatId].modifiedItems } + : {}) } this.savedChats = { ...this.savedChats, @@ -218,9 +264,10 @@ export default class HistoryManager { async save( displayMessages: DisplayMessage[], messages: ChatCompletionMessageParam[], - contextUsage?: number + contextUsage?: number, + modifiedItems?: string[] ) { - await this.saveChat(displayMessages, messages, contextUsage) + await this.saveChat(displayMessages, messages, contextUsage, modifiedItems) this.currentChatId = createLongHash() } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 38fb7ecf0d..540dfaf72e 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -141,3 +141,30 @@ describe('HistoryManager title across compaction', () => { expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('original first question') }) }) + +describe('HistoryManager modified-items mask persistence', () => { + const msgs = [{ role: 'user', content: 'hello', index: 0 }] as DisplayMessage[] + + it('a save without the argument preserves a previously stored mask', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[], undefined, ['script:u/a/x']) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + + // e.g. manual compaction re-saving the transcript: the whole record is + // rewritten, but the tracked mask must survive. + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toEqual(['script:u/a/x']) + }) + + it('never retroactively stamps an untracked chat', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + await hm.saveChat(msgs, [] as ChatCompletionMessageParam[]) + expect(hm.getModifiedItems(id)).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 90dcb655f4..24045280d0 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -114,6 +114,7 @@ import { getEphemeralSecretVariableDraftValue, getGlobalDraft, getGlobalDraftStoragePath, + itemKindFor, listGlobalDrafts, persistGlobalDraft, readGlobalDraftValue, @@ -2779,6 +2780,7 @@ function finishAppDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const { content, message } = onSaved() ctx.toolCallbacks.setToolStatus(ctx.toolId, { content, result: 'Saved as draft' }) return JSON.stringify({ success: true, message }, null, 2) @@ -2791,6 +2793,7 @@ function finishDraftWrite( ): string { const failure = draftWriteFailure(result, ctx) if (failure) return failure + ctx.toolCallbacks.onItemModified?.(result.itemKind, result.storagePath) const stored = result.item const verb = existed ? 'Updated' : 'Created' // Don't echo the flow value back: the model just sent it in the write call, @@ -3898,6 +3901,16 @@ async function discardLocalDraft( await deleteGlobalDraft(workspace, type, path, triggerKind) + // The chat's touch on the item is undone — drop it from the mask so a + // pre-existing deployed item doesn't keep reading as this chat's edit. + const discardedKind = itemKindFor(type, triggerKind) + if (discardedKind) { + toolCallbacks.onItemDiscarded?.( + discardedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Discarded ${type} "${path}" draft`, result: 'Draft discarded' @@ -4237,6 +4250,9 @@ async function deployDraft( }) let actions: ToolDisplayAction[] | undefined + // Where the deploy actually lands — the app branch can resolve a different + // target from the draft's own path fields; the mask rename below must track it. + let deployedPath = path if (type === 'script' || type === 'flow') { // Promote the full persisted draft via the shared deploy module — the same @@ -4429,6 +4445,7 @@ async function deployDraft( throw e } } + deployedPath = targetPath if (await AppService.existsApp({ workspace, path: targetPath })) { // Omit custom_path on update for now. The backend preserves it when absent, while // sending it requires admin privileges; this chat deploy path does not yet mirror @@ -4478,6 +4495,18 @@ async function deployDraft( await deleteGlobalDraft(workspace, type, path, triggerKind, { preserveLiveDraft: true }) + // Move the chat's mask entry to the deployed path: a draft-only item's + // synthetic storage key never exists deployed, so the entry would otherwise + // stop matching anything after the draft is gone. + const deployedKind = itemKindFor(type, triggerKind) + if (deployedKind) { + toolCallbacks.onItemDeployed?.( + deployedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind), + deployedPath + ) + } + // Reload the session preview if it's open on the deployed item. Map the // deploy type to the preview kind — a raw app deploys under 'app' but the // preview addresses it as 'raw_app'; non-previewable types map to undefined. @@ -4548,6 +4577,17 @@ async function deleteWorkspaceItem( await deleteGlobalDraft(workspace, type, path, triggerKind) + // Record the deletion in the chat's modified-items mask. In a fork this leaves a + // reviewable "removed" diff vs the parent that stays scoped to this chat. Keyed + // by the same (itemKind, storagePath) as writes so it joins the draft/fork lists. + const deletedKind = itemKindFor(type, triggerKind) + if (deletedKind) { + toolCallbacks.onItemModified?.( + deletedKind, + getGlobalDraftStoragePath(workspace, type, path, triggerKind) + ) + } + toolCallbacks.setToolStatus(toolId, { content: `Deleted ${type} "${path}"`, result: 'Deleted' diff --git a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts index 87ca767904..2717d41608 100644 --- a/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts +++ b/frontend/src/lib/components/copilot/chat/global/userDraftAdapter.ts @@ -108,7 +108,7 @@ function clearEphemeralSecretVariableDraftValues(workspace: string): void { secretVariableDraftValues.delete(workspace) } -function itemKindFor( +export function itemKindFor( type: WorkspaceItemType, triggerKind?: TriggerKind ): UserDraftItemKind | undefined { @@ -377,10 +377,25 @@ export async function readGlobalDraftValue( return (await fetchBackendDraftValue(workspace, itemKind, storagePath)) as V | undefined } +// `itemKind` + `storagePath` are the canonical identity of the persisted draft +// (NOT item.path, which is the friendly display path). Callers use them to record +// the chat's modified-items mask. export type DraftPersistResult = - | { status: 'saved'; item: WorkspaceItem } - | { status: 'conflict'; item: WorkspaceItem; serverTimestamp?: string } - | { status: 'error'; item: WorkspaceItem; message: string } + | { status: 'saved'; item: WorkspaceItem; itemKind: UserDraftItemKind; storagePath: string } + | { + status: 'conflict' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + serverTimestamp?: string + } + | { + status: 'error' + item: WorkspaceItem + itemKind: UserDraftItemKind + storagePath: string + message: string + } // Persist a built draft value. `UserDraft.seed` reflects it into an open editor's // cell WITHOUT a double-POST (no-ops if no cell; its seedNextWrite suppresses the @@ -417,14 +432,20 @@ export async function persistGlobalDraft( // the chat "saved" while the DB-backed source of truth was never updated. const saveState = UserDraftDbSyncer.getState({ workspace, itemKind, path: storagePath }) if (saveState.state === 'failed') { - return { status: 'error', item, message: saveState.failureMessage ?? 'Draft save failed' } + return { + status: 'error', + item, + itemKind, + storagePath, + message: saveState.failureMessage ?? 'Draft save failed' + } } const conflict = opts.force ? undefined : UserDraftDbSyncer.getConflict({ workspace, itemKind, path: storagePath }).conflict return conflict - ? { status: 'conflict', item, serverTimestamp: conflict.serverTimestamp } - : { status: 'saved', item } + ? { status: 'conflict', item, itemKind, storagePath, serverTimestamp: conflict.serverTimestamp } + : { status: 'saved', item, itemKind, storagePath } } export async function getGlobalDraft( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 7abdc75168..c454f15fe6 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -3,6 +3,7 @@ import type { ChatCompletionMessageFunctionToolCall, ChatCompletionMessageParam } from 'openai/resources/chat/completions.mjs' +import type { UserDraftItemKind } from '$lib/gen' /** * Special module IDs used throughout the flow system @@ -759,6 +760,15 @@ export interface ToolCallbacks { toolId: string, question: UserQuestionDisplay ) => Promise + /** Records a workspace item the tool call created/edited/deleted, by its + * canonical (itemKind, storagePath). Session chats wire this to accumulate the + * chat's modified-items mask; the global side-panel chat omits it (no-op). */ + onItemModified?: (itemKind: UserDraftItemKind, storagePath: string) => void + /** A tool deployed a draft: the mask entry moves from the draft's storage path + * to the deployed path (they differ for synthetic draft-only storage keys). */ + onItemDeployed?: (itemKind: UserDraftItemKind, storagePath: string, deployedPath: string) => void + /** A tool discarded a draft: the chat's touch on the item is undone. */ + onItemDiscarded?: (itemKind: UserDraftItemKind, storagePath: string) => void } export function createToolDef( diff --git a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte b/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte deleted file mode 100644 index 0b529b2056..0000000000 --- a/frontend/src/lib/components/sessions/DraftDiffDrawer.svelte +++ /dev/null @@ -1,204 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, workspaceId)} -> - {#snippet titleExtra()} -
- - {ws?.name ?? workspaceId} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte b/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte deleted file mode 100644 index e03fc5a49c..0000000000 --- a/frontend/src/lib/components/sessions/ForkDiffDrawer.svelte +++ /dev/null @@ -1,125 +0,0 @@ - - - buildEditUrl(d as unknown as WorkspaceItemDiff, forkWorkspaceId)} -> - {#snippet titleExtra()} -
- - {forkWs?.name ?? forkWorkspaceId} - - {parentWs?.name ?? parentWorkspaceId} - {#if comparison} - - {comparison.summary.total_diffs} item{comparison.summary.total_diffs !== 1 ? 's' : ''} - - {#if comparison.summary.conflicts > 0} - - - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 ? 's' : ''} - - {/if} - {/if} -
- {/snippet} - diff --git a/frontend/src/lib/components/sessions/SessionChangesBar.svelte b/frontend/src/lib/components/sessions/SessionChangesBar.svelte new file mode 100644 index 0000000000..9f59d1f320 --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionChangesBar.svelte @@ -0,0 +1,264 @@ + + +{#snippet dock()} + +
+ {#if dockCounts.draft > 0} + diffDrawer?.open()}> + {dockCounts.draft} draft{dockCounts.draft === 1 ? '' : 's'} + + {/if} + {#if dockCounts.deployed > 0} + diffDrawer?.open()}> + {dockCounts.deployed} deployed + + {/if} + {#if deletionOnly && compareHref} + + + Review deletions on compare page + + + {/if} +
+{/snippet} + +{#if committedId && isUnavailable} + +
+
+ +
+ The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted + + Move this session to another workspace, or discard it. + {committedId} + +
+
+
+ onMove?.(workspaceId)} + onCreateFork={async (fork) => { + await onCreateForkAndMove?.(fork) + }} + createForkCaption="Created immediately and the session moved into it." + > + {#snippet trigger()} + + {/snippet} + + +
+
+{:else if showBar && committedId} + +
+
+ + Edits +
+ {@render dock()} +
+{/if} + + +{#if committedId && !isUnavailable} + + void runtime?.manager.renameModifiedItem(item.draftKind, item.path, item.displayPath)} + onItemDiscarded={(item) => void runtime?.manager.removeModifiedItem(item.draftKind, item.path)} + /> +{/if} diff --git a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte new file mode 100644 index 0000000000..1a6692048c --- /dev/null +++ b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte @@ -0,0 +1,107 @@ + + + + {#snippet titleExtra()} +
+ {#if isFork} + + + {ws?.name ?? workspaceId} + + + + {parentWs?.name ?? parentWorkspaceId} + + {:else} + + + {ws?.name ?? workspaceId} + + {/if} +
+ {/snippet} + diff --git a/frontend/src/lib/components/sessions/SessionDraftBar.svelte b/frontend/src/lib/components/sessions/SessionDraftBar.svelte deleted file mode 100644 index d23070151a..0000000000 --- a/frontend/src/lib/components/sessions/SessionDraftBar.svelte +++ /dev/null @@ -1,77 +0,0 @@ - - -{#if committedId && count > 0} -
-
- - - {count} draft{count === 1 ? '' : 's'} - {#snippet text()} - Tracks all unsaved draft changes in this workspace — including edits made outside this - chat (e.g. in the editor), not only changes made by the assistant. - {/snippet} - -
-
- drawer?.open()} /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionForkBar.svelte b/frontend/src/lib/components/sessions/SessionForkBar.svelte deleted file mode 100644 index a0aafee456..0000000000 --- a/frontend/src/lib/components/sessions/SessionForkBar.svelte +++ /dev/null @@ -1,212 +0,0 @@ - - -{#if committedId && isUnavailable} - -
-
- -
- The {committedIsFork ? 'fork' : 'workspace'} has been archived or deleted - - Move this session to another workspace, or discard it. - {committedId} - -
-
-
- onMove?.(workspaceId)} - onCreateFork={async (fork) => { - await onCreateForkAndMove?.(fork) - }} - createForkCaption="Created immediately and the session moved into it." - > - {#snippet trigger()} - - {/snippet} - - -
-
-{:else if forksAllowed && isFork && sessionWorkspace && parentWorkspace && parentWorkspaceId && committedId} - {@const StatusIcon = - forkStatus === 'ahead' - ? GitPullRequestArrow - : forkStatus === 'diverged' - ? GitCompareArrows - : GitFork} - {@const statusColor = - forkStatus === 'ahead' - ? 'text-blue-500' - : forkStatus === 'diverged' - ? 'text-amber-500' - : 'text-secondary'} - {@const statusTitle = - forkStatus === 'ahead' - ? 'Ahead of parent' - : forkStatus === 'diverged' - ? 'Diverged from parent' - : forkStatus === 'in_sync' - ? 'In sync with parent' - : 'Fork'} -
-
- - - - - {sessionWorkspace.name} - - - - {parentWorkspace.name} - -
-
- diffDrawer?.open()} - /> - -
-
- - -{/if} diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index aad3de26c3..0342fd53e1 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -21,7 +21,6 @@ import { slide } from 'svelte/transition' import { createSession, - deriveForkStatus, deleteSessionsForWorkspace, isForkSession, reconcileAfterWorkspaceChange, @@ -53,15 +52,6 @@ import { sendUserToast } from '$lib/toast' import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' - // Look up the cached fork comparison for a session through its runtime - // (if any). The deriveForkStatus helper handles the "no runtime yet" - // and "comparison not loaded" cases by returning undefined; we render - // a neutral fork icon in that interim, then upgrade to the proper - // status icon once the comparison lands. - function forkStatusFor(session: Session) { - return deriveForkStatus(session, $userWorkspaces, getRuntime(session.id)?.forkComparison.val) - } - function isForkFor(session: Session): boolean { return isForkSession(session, $userWorkspaces) } @@ -124,8 +114,8 @@ } // Flat list passing the archive + scope filters. Grouping for display happens - // in `sessionGroups`; this flat view drives the runtime / fork-comparison - // effects, the unread total, and keyboard navigation. + // in `sessionGroups`; this flat view drives the runtime effect, the unread + // total, and keyboard navigation. const visibleSessions = $derived( sessionState.sessions.filter((s) => { if (s.transient) return false @@ -211,23 +201,6 @@ } }) - // Pre-fetch the fork comparison for every visible fork session so the - // sidebar icons reflect the right ahead/diverged state without - // requiring the user to click into each session. Cheap enough at - // typical session counts; falls back to a plain dot until the - // fetch lands. - $effect(() => { - if (sectionCollapsed.val) return - for (const session of visibleSessions) { - if (!session.workspace_id) continue - const ws = $userWorkspaces.find((w) => w.id === session.workspace_id) - if (!ws?.parent_workspace_id) continue - const rt = getRuntime(session.id) - if (!rt) continue - void rt.ensureForkComparison(ws.parent_workspace_id, session.workspace_id) - } - }) - function isUnavailableFork(session: Session): boolean { return !!session.workspace_id && !$userWorkspaces.find((w) => w.id === session.workspace_id) } @@ -241,10 +214,6 @@ if (!isUnavailableFork(session)) { syncWorkspaceTo(session.workspace_id) } - // Refresh the fork diff count — users typically click back into a - // session after editing items elsewhere in the SPA, where neither - // the visibility-change nor the AI-loading signal would fire. - void getRuntime(session.id)?.refreshForkComparison() await goto(`/sessions?session_name=${encodeURIComponent(session.name)}`) if (restoreFocus) { // goto() resets focus to — put it back on the active session button @@ -436,7 +405,7 @@ {session.summary ?? 'Untitled session'} {#if draft || unread > 0} diff --git a/frontend/src/lib/components/sessions/SessionStatusDot.svelte b/frontend/src/lib/components/sessions/SessionStatusDot.svelte index b47816ab19..35028c80cc 100644 --- a/frontend/src/lib/components/sessions/SessionStatusDot.svelte +++ b/frontend/src/lib/components/sessions/SessionStatusDot.svelte @@ -3,19 +3,16 @@ AlertCircle, AlertTriangle, Building, - GitCompareArrows, GitFork, - GitPullRequestArrow, GitPullRequestClosed } from 'lucide-svelte' import type { SessionChatStatus } from './sessionRuntime.svelte' - import type { ForkStatus } from './sessionState.svelte' let { status, isFork, - forkStatus - }: { status: SessionChatStatus; isFork: boolean; forkStatus?: ForkStatus } = $props() + unavailable = false + }: { status: SessionChatStatus; isFork: boolean; unavailable?: boolean } = $props() const statusTooltip: Record = { idle: 'No chat activity', @@ -26,13 +23,6 @@ error: 'Last message had an error' } - const forkTooltip: Record = { - in_sync: 'Fork — in sync with parent', - ahead: 'Fork — ahead of parent', - diverged: 'Fork — diverged from parent', - unavailable: 'Fork — no longer available' - } - // Live chat signals take precedence over the persistent kind/fork // indicator: streaming, needs-confirmation, and error are time-critical // and warrant briefly hijacking the icon slot. @@ -41,7 +31,11 @@ ) const persistentTitle = $derived( - isFork ? (forkStatus ? forkTooltip[forkStatus] : 'Fork session') : 'Root workspace session' + isFork + ? unavailable + ? 'Fork — no longer available' + : 'Fork session' + : 'Root workspace session' ) const title = $derived(liveOverride ? statusTooltip[status] : persistentTitle) @@ -59,11 +53,7 @@ {:else if status === 'error'} {:else if isFork} - {#if forkStatus === 'ahead'} - - {:else if forkStatus === 'diverged'} - - {:else if forkStatus === 'unavailable'} + {#if unavailable} {:else} diff --git a/frontend/src/lib/components/sessions/SessionWrapper.svelte b/frontend/src/lib/components/sessions/SessionWrapper.svelte index c9fdb6454f..ee8cbed6e0 100644 --- a/frontend/src/lib/components/sessions/SessionWrapper.svelte +++ b/frontend/src/lib/components/sessions/SessionWrapper.svelte @@ -30,8 +30,7 @@ import RawAppEditorView from './RawAppEditorView.svelte' import PipelineEditorView from './PipelineEditorView.svelte' import SessionWorkspaceBar from './SessionWorkspaceBar.svelte' - import SessionForkBar from './SessionForkBar.svelte' - import SessionDraftBar from './SessionDraftBar.svelte' + import SessionChangesBar from './SessionChangesBar.svelte' import { createSession, deleteSessionsForWorkspace, @@ -235,7 +234,7 @@ // True when the session committed to a workspace that's no longer in // the user's list (deleted / archived / access revoked). The chat is - // disabled and SessionForkBar shows a move/discard banner. + // disabled and SessionChangesBar shows a move/discard banner. const isUnavailable = $derived( !!session?.workspace_id && !$userWorkspaces.find((w) => w.id === session!.workspace_id) ) @@ -274,16 +273,16 @@ {#if !hasFirstUserMessage} {/if} - +
{#if session.archived && !isUnavailable}
{/if} - moveAndActivate(workspaceId)} onCreateForkAndMove={(fork) => createForkAndMove(fork)} onArchive={() => archiveAndReset()} onDelete={() => (deleteConfirmOpen = true)} /> -
{/snippet} diff --git a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte index e6b4e5fd2f..50c6653b0a 100644 --- a/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/WorkspaceDiffDrawer.svelte @@ -1,45 +1,19 @@ - - searchableText(d)} + items={displayEntries} + bind:filteredItems={searchedEntries} + f={(e: DisplayEntry) => searchableText(e)} /> -{#snippet renderTreeNode(node: TreeNode, depth: number)} + +{#snippet rowBadge(item: DeployItem)} + {#if badgeOf(item) === 'draft'} + {#if model.staleOf(item.key)} + + + {#snippet trigger()} + + {/snippet} + {#snippet content()} +
+ Started from an older deployed version. A newer version was deployed after this draft + began. Review the latest deploy before deploying. +
+ {/snippet} + + {/if} + + {item.draftOnly ? 'Draft only' : 'Draft'} + + {:else} + + + + + + + {/if} +{/snippet} + + +{#snippet deployFailed(item: DeployItem)} + {@const s = model.statusOf(item.key)} + {#if s?.status === 'failed'} + + Failed + + + {/if} +{/snippet} + +{#snippet renderTreeNode(node: TreeNode, depth: number)} {#if node.type === 'folder'} {@const isUserScope = node.isScope && node.name.startsWith('u/')} {@const fkey = node.key} @@ -417,36 +613,62 @@ {@const isHl = fkey === highlightedKey}
(folderOpen[fkey] = (e.currentTarget as HTMLDetailsElement).open)} + ontoggle={(e) => { + // Record real user toggles only; skip the echo fired when the `open` + // attribute is driven by state (search force-open, expandApp). + const domOpen = (e.currentTarget as HTMLDetailsElement).open + if (domOpen !== isFolderOpen(fkey)) folderOpen[fkey] = domOpen + }} class="select-none" > {#if node.app} - {@const appSummary = summaries[node.app.summaryKey] ?? node.app.summary} - + {@const appItem = segmentItems.find((it) => it.key === node.app?.summaryKey)} + setHoverHighlight(fkey)} + onclick={(e) => { + e.preventDefault() + if (appItem) revealDiff(appItem, fkey) + }} title={node.fullPath} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" > - {appSummary ?? node.name} + {node.app.summary ?? node.name} - - + {#if appItem} + {@render rowBadge(staged[appItem.key] ?? appItem)} + {/if} + + {:else} setHoverHighlight(fkey)} - class="flex items-center gap-1.5 px-3 py-1.5 cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl + class="flex items-center gap-2 pl-3 pr-1 py-2 rounded-md cursor-pointer text-xs font-normal font-mono text-secondary hover:bg-surface-hover list-none [&::-webkit-details-marker]:hidden tree-summary {isHl ? 'bg-surface-hover' : ''}" style="padding-left: {depth * 12 + 8}px" @@ -465,14 +687,13 @@ {/if} {node.name} - - + + + + {/if}
-
- { - highlightedKey = key - scrollToDiff(d) - }} - onmouseenter={() => setHoverHighlight(key)} + +
+ {#if isSynthetic(d)} + void revealSynthetic(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + + {/snippet} + + {:else} + revealDiff(d, key)} + onmouseenter={() => setHoverHighlight(key)} + > + {#snippet extras()} + {@render rowBadge(staged[d.key] ?? d)} + {#if d.deployKind === 'raw_app'} + + { + e.stopPropagation() + expandApp(d) + }} + > + {#if loadedDiffs[d.key]?.state === 'loading'} + + {:else} + + {/if} + + {:else} + + {/if} + {/snippet} + + {/if} +
+ {/if} +{/snippet} + + +{#snippet diffBlock(item: DeployItem)} + {@const loaded = loadedDiffs[item.key]} + {#if !mountedRows[item.key]} +
+ + Diff loads on scroll… +
+ {:else if !loaded || loaded.state === 'loading'} +
+ + Loading diff… +
+ {:else if loaded.state === 'error'} +
{loaded.error}
+ {:else if item.deployKind === 'raw_app'} +
+ {#each rawAppItems(item, loaded) as sub (displayKey(sub))} + +
+
+ {sub.path} +
+ {#if sub.kind === 'raw_app_file'} + + {:else} + {@const runnable = sub as RawAppRunnableItem} + + {/if} +
+ {/each} +
+ {:else} + {/if} {/snippet} @@ -540,178 +869,215 @@ /> {/snippet} - {/snippet} -
- {#if diffs.length > 0} - - {/if} -
-
- {#if loading && diffs.length === 0} -
- - Loading comparison... +
+
+ {#if model.items.length > 0} + + {/if} +
+
+ {#if model.loading && model.items.length === 0} +
+ + Loading changes... +
+ {:else if model.error} +
{model.error}
+ {:else if model.items.length === 0} +
No changes.
+ {:else if orderedItems.length === 0} +
No files match.
+ {:else} +
+ {#each orderedItems as d (d.key)} + + {@const view = staged[d.key] ?? d} + {@const action = actionFor(view)} + {@const editUrl = editUrlFor?.(d)} + {@const status = model.statusOf(d.key)} +
- - -
- {#if editUrl} - + +
+ {#if editUrl} + + {d.displayPath} + + {:else} +
+ {d.displayPath} +
+ {/if} +
+
+ {@render rowBadge(view)} + {#if status?.status === 'failed'} + {@render deployFailed(d)} + {:else if action.op !== 'none'} + {#if action.secondary?.length} + + {/if} +
+ + {#if staged[d.key]} + +
+ +
+ {/if} +
+ {/if} +
+
+
+ + {#if view.done} +
- {dpath} - + Deployed — no pending changes. +
{:else} -
- {dpath} +
+ {@render diffBlock(view)}
{/if}
-
- {#if d.ahead && d.ahead > 0} - {d.ahead} ahead - {/if} - {#if d.behind && d.behind > 0} - {d.behind} behind - {/if} - - - {status} - -
- -
- {#if !mountedRows[key]} - -
- - Diff loads on scroll… -
- {:else if d.kind === 'raw_app_file'} - - {@const rawFile = d as RawAppFileItem} - - {:else if 'appPath' in d} - - {@const runnable = d as RawAppRunnableItem} - - {:else if !loaded || loaded.state === 'loading'} -
- - Loading diff… -
- {:else if loaded.state === 'error'} -
{loaded.error}
- {:else if loaded.state === 'ready'} - - {/if}
-
- {/each} -
- {/if} - + {/each} + + + + {/if} + + + + + {#if model.items.length > 0 && compareSessionHref} +
+ + Deploy all / request review on compare page + + +
+ {/if} + +