diff --git a/AGENTS.md b/AGENTS.md index aead237e59..ac00741c49 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,6 +24,7 @@ Open-source platform for internal tools, workflows, API integrations, background ## Dev Environment - **Backend**: `cargo run` from `backend/` (API at http://localhost:8000) +- **DuckDB local jobs**: before running DuckDB scripts locally, build the FFI shared library with `cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh`. Re-run it after clean builds or when `backend/target/debug/libwindmill_duckdb_ffi_internal.*` is missing. - **Frontend**: `REMOTE=http://localhost:8000 npm run dev` from `frontend/` (port 3000+) - **DB**: `psql postgres://postgres:changeme@localhost:5432/windmill` - **Login**: `admin@windmill.dev` / `changeme` diff --git a/CHANGELOG.md b/CHANGELOG.md index 361b6fbd91..668078da12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,89 @@ # Changelog +## [1.746.0](https://github.com/windmill-labs/windmill/compare/v1.745.0...v1.746.0) (2026-07-02) + + +### Features + +* **ai:** add Azure AI Foundry as a native AI provider ([#9879](https://github.com/windmill-labs/windmill/issues/9879)) ([d9b080f](https://github.com/windmill-labs/windmill/commit/d9b080f57fa0be144cefa773d39742c45b40f043)) +* **frontend:** group compare & deploy items by folder ([#9880](https://github.com/windmill-labs/windmill/issues/9880)) ([7b04820](https://github.com/windmill-labs/windmill/commit/7b04820f8ef8c7f02f79dd4239a877f667d23e6a)) +* **frontend:** pipelines index page and sql editor hint ([#9881](https://github.com/windmill-labs/windmill/issues/9881)) ([20351a6](https://github.com/windmill-labs/windmill/commit/20351a6b4c262184c5f815eeb5de007ab1eaf4a0)) +* **pipeline:** backfill a range of partitions from the asset drawer ([#9885](https://github.com/windmill-labs/windmill/issues/9885)) ([53bbb92](https://github.com/windmill-labs/windmill/commit/53bbb92953178eb6d0017818ef870f3cb2399dfd)) +* **pipelines:** workspace duckdb macro libraries (// macros / // use) ([#9890](https://github.com/windmill-labs/windmill/issues/9890)) ([84141ad](https://github.com/windmill-labs/windmill/commit/84141add1ddf35c2573e3c213366ce7c5f1f2258)) +* **s3:** replace CE 50MB upload cap with 10GiB workspace storage quota ([#9874](https://github.com/windmill-labs/windmill/issues/9874)) ([af01e90](https://github.com/windmill-labs/windmill/commit/af01e90b5c65d1b1cfacf4433f8cff7effe73768)) +* support workspace forks on cloud using parent workspace limits ([#9864](https://github.com/windmill-labs/windmill/issues/9864)) ([7c7d747](https://github.com/windmill-labs/windmill/commit/7c7d7474cc86a4052272032f281cc4d7a85db37b)) + + +### Bug Fixes + +* **duckdb:** auto-declare partition arg for `// partitioned` scripts ([#9878](https://github.com/windmill-labs/windmill/issues/9878)) ([b883adb](https://github.com/windmill-labs/windmill/commit/b883adbc0011073da592dc5b39e1b79db492c83c)) +* **frontend:** home New submenus fall back below, hugging the right edge ([#9894](https://github.com/windmill-labs/windmill/issues/9894)) ([186ac49](https://github.com/windmill-labs/windmill/commit/186ac4933b79aed57fce23ebcf3b525fcfd1c474)) +* **frontend:** show inline workspace name editor on general settings (Fixes GIT-911) ([#9892](https://github.com/windmill-labs/windmill/issues/9892)) ([a49c087](https://github.com/windmill-labs/windmill/commit/a49c0871d7ab2aaf78a7713b8a786ead937434da)) +* **frontend:** stack cron field and cron builder button on narrow screens ([#9871](https://github.com/windmill-labs/windmill/issues/9871)) ([7989795](https://github.com/windmill-labs/windmill/commit/79897950e7646b00d92a28a009174d91c705b251)) +* invalidate bun bundle cache on transitive relative-import changes ([#9891](https://github.com/windmill-labs/windmill/issues/9891)) ([d15033c](https://github.com/windmill-labs/windmill/commit/d15033cde6a474b548ebbaf18ff02223fc21f701)) +* make SMTP username and password optional in frontend validation ([#9895](https://github.com/windmill-labs/windmill/issues/9895)) ([37bb574](https://github.com/windmill-labs/windmill/commit/37bb57474e8336823bb31527f2a708ef41cd39c4)) +* **parsers:** infer py s3 assets from S3Object constructor and dict forms ([#9877](https://github.com/windmill-labs/windmill/issues/9877)) ([659642e](https://github.com/windmill-labs/windmill/commit/659642e4889361f86e8addb038cda62fc3471006)) +* pipeline dogfooding fixes — SCD2 data-test scope, --partition, s3object upload binding ([#9875](https://github.com/windmill-labs/windmill/issues/9875)) ([d65f58c](https://github.com/windmill-labs/windmill/commit/d65f58c388d88fff71cda22dfa21aecdae70c450)) +* polish pipeline graph view (layout, viewport, minimap, lineage, timestamps) ([#9883](https://github.com/windmill-labs/windmill/issues/9883)) ([b92a86b](https://github.com/windmill-labs/windmill/commit/b92a86b8b3a60b877540c3a7f0ffefe36ccbb053)) +* stale AI chat context picker after workspace item changes ([#9893](https://github.com/windmill-labs/windmill/issues/9893)) ([5af91a6](https://github.com/windmill-labs/windmill/commit/5af91a677cad88faccba702e3556fc4fb7b6e640)) +* **triggers:** retry transient websocket connect failures before disabling ([#9887](https://github.com/windmill-labs/windmill/issues/9887)) ([7894507](https://github.com/windmill-labs/windmill/commit/789450731b0a3c8dffa336f7bfc3f3de528c09fb)) + +## [1.745.0](https://github.com/windmill-labs/windmill/compare/v1.744.0...v1.745.0) (2026-07-01) + + +### Features + +* **forks:** partial-visibility deploy + surface hidden items ([#9868](https://github.com/windmill-labs/windmill/issues/9868)) ([20cd1a0](https://github.com/windmill-labs/windmill/commit/20cd1a02d582c0715bedacce52cc5c1e1e8d70ca)) +* **frontend:** add zoom and download to Mermaid graphs ([#9859](https://github.com/windmill-labs/windmill/issues/9859)) ([289017b](https://github.com/windmill-labs/windmill/commit/289017bcb28c049c8258b2ffd7da0ec3e6ef120b)) +* use derived username instead of email for non-member superadmins ([#9857](https://github.com/windmill-labs/windmill/issues/9857)) ([76a9523](https://github.com/windmill-labs/windmill/commit/76a95230095ca3f43c9dc9eecde0e9de6520242f)) + + +### Bug Fixes + +* **cli:** correct misleading delete-fork command description ([#9870](https://github.com/windmill-labs/windmill/issues/9870)) ([a73b14d](https://github.com/windmill-labs/windmill/commit/a73b14d902d759226d0af2f2faf9bdd6588e358c)) +* **folders:** allow dots and at-signs in folder owner validation ([#9856](https://github.com/windmill-labs/windmill/issues/9856)) ([383c705](https://github.com/windmill-labs/windmill/commit/383c70523bf81c5c07784a4379ef6b1c93ff86e5)) +* **forks:** require admin of both sides for the compare visibility guard ([#9869](https://github.com/windmill-labs/windmill/issues/9869)) ([7363d2c](https://github.com/windmill-labs/windmill/commit/7363d2c217cb04391f03b2f9958be70a9d0b5325)) +* **forks:** reset diff tally on trigger delete + guard compare visibility for admins ([#9866](https://github.com/windmill-labs/windmill/issues/9866)) ([6a6f129](https://github.com/windmill-labs/windmill/commit/6a6f12960e29c314d11ad541519c71412f42567b)) +* **jobs:** give flow dynselect a path and its worker tag, like scripts ([#9867](https://github.com/windmill-labs/windmill/issues/9867)) ([1a9debb](https://github.com/windmill-labs/windmill/commit/1a9debb689f756f38db085d1360c8fe7691ada48)) +* **offboarding:** make global reassignment per-workspace and optional ([#9863](https://github.com/windmill-labs/windmill/issues/9863)) ([3586164](https://github.com/windmill-labs/windmill/commit/35861641f807a02b5c205608fb592e20ee7cad7f)) + +## [1.744.0](https://github.com/windmill-labs/windmill/compare/v1.743.0...v1.744.0) (2026-07-01) + + +### Features + +* add copy-to-clipboard button to rendered Mermaid diagrams in AI chat ([#9838](https://github.com/windmill-labs/windmill/issues/9838)) ([a27e814](https://github.com/windmill-labs/windmill/commit/a27e814a03c615259381eaf684aa90d56569b0af)) +* add dev workspaces paired with a lockable prod workspace ([#9793](https://github.com/windmill-labs/windmill/issues/9793)) ([b4b0c6a](https://github.com/windmill-labs/windmill/commit/b4b0c6a93e52152251fadefe319773faf42549b2)) +* **ansible:** support repo-provided ansible.cfg in delegate_to_git_repo ([#9851](https://github.com/windmill-labs/windmill/issues/9851)) ([68bf0da](https://github.com/windmill-labs/windmill/commit/68bf0daf5815307cda6ce23214dd5159b6aa33b4)) +* **licensing:** enforce offline license seat cap ([#9845](https://github.com/windmill-labs/windmill/issues/9845)) ([83f3d7f](https://github.com/windmill-labs/windmill/commit/83f3d7f910b331c09f60cc9ff556728afa3dec07)) +* **object-store:** make GCS service account key optional for Workload Identity ([#9842](https://github.com/windmill-labs/windmill/issues/9842)) ([83ed011](https://github.com/windmill-labs/windmill/commit/83ed011e264f20ffa66a7bf933f2fe3615cf6b67)) +* **pipeline:** local development for data pipelines (CLI --local + pipeline dev preview) ([#9840](https://github.com/windmill-labs/windmill/issues/9840)) ([74f579e](https://github.com/windmill-labs/windmill/commit/74f579e6d9ef08e74460f904a4c22ed9d6a3b5b0)) +* **pipelines:** add managed SCD2 history materialize strategy ([#9850](https://github.com/windmill-labs/windmill/issues/9850)) ([5a66127](https://github.com/windmill-labs/windmill/commit/5a661279a3690e2393b9b16996f5d1a5a509259c)) + + +### Bug Fixes + +* **ai-chat:** replay anthropic turns verbatim to keep thinking valid ([#9843](https://github.com/windmill-labs/windmill/issues/9843)) ([a37a144](https://github.com/windmill-labs/windmill/commit/a37a144e81cf6b3de935688a617e9d0e1756004a)) +* grant dispatch_event table to windmill roles ([#9852](https://github.com/windmill-labs/windmill/issues/9852)) ([f05b50d](https://github.com/windmill-labs/windmill/commit/f05b50d29ac2fdbb808a97057fb92c8e425b4a2f)) +* grant workspace_diff, materialized_partition, debounce_stale_data to windmill roles ([#9853](https://github.com/windmill-labs/windmill/issues/9853)) ([293647d](https://github.com/windmill-labs/windmill/commit/293647de4c13cb8468cbd81ff1924cba90e164b4)) +* honor verify-ca/verify-full sslmode for postgres connections ([#9835](https://github.com/windmill-labs/windmill/issues/9835)) ([bf6be96](https://github.com/windmill-labs/windmill/commit/bf6be967fa8c74e1299cf63f813c1cfa34b97f3e)) +* **mcp:** stop double-escaping string query params in build_query_string ([#9855](https://github.com/windmill-labs/windmill/issues/9855)) ([1c46f89](https://github.com/windmill-labs/windmill/commit/1c46f899ca03edf62053f4f14d65b4eabff4255d)) +* **s3_proxy:** preserve URL-encoding on Hive-partition proxy writes ([#9848](https://github.com/windmill-labs/windmill/issues/9848)) ([6b79bdd](https://github.com/windmill-labs/windmill/commit/6b79bddd42fe55f891c17cb71a7e36ee31337bac)) +* validate workspace name length (max 50 chars) on create and fork ([#9854](https://github.com/windmill-labs/windmill/issues/9854)) ([b52972d](https://github.com/windmill-labs/windmill/commit/b52972d0de89004e98d18241d238ca028e4eecba)) + +## [1.743.0](https://github.com/windmill-labs/windmill/compare/v1.742.0...v1.743.0) (2026-06-29) + + +### Features + +* **home:** redesign create-new popover and home header ([#9827](https://github.com/windmill-labs/windmill/issues/9827)) ([2493eaf](https://github.com/windmill-labs/windmill/commit/2493eaf031f30072637a297398674e761f039005)) +* **pipeline:** AI-chat data-pipeline editor (route + in-session) + home surfacing ([#9805](https://github.com/windmill-labs/windmill/issues/9805)) ([c910278](https://github.com/windmill-labs/windmill/commit/c91027824be1f1f49cdd14148baf6aad092a1dd0)) + + +### Bug Fixes + +* **gcp:** require token verification for authenticated push delivery ([#9834](https://github.com/windmill-labs/windmill/issues/9834)) ([9b65161](https://github.com/windmill-labs/windmill/commit/9b65161c643bf3f120d2ebd82f786c17233a971b)) + ## [1.742.0](https://github.com/windmill-labs/windmill/compare/v1.741.0...v1.742.0) (2026-06-28) diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index 31b097867b..66e5fd3cb7 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -1223,6 +1223,66 @@ - places it in team_a (writable by this non-admin user) and not team_b (read-only) - leaves the result as a draft only +- id: global-test-pipeline-create-node + prompt: |- + Set up the first step of a data pipeline at `f/evals/global/orders_ingest`. + On a schedule, it should pull raw orders and land them in a managed DuckLake + table so later steps can build on it. Keep it as an AI draft only — don't + deploy or save it. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 10 + validate: + draftCountExactly: 1 + requiredDrafts: + - type: script + path: f/evals/global/orders_ingest + valueIncludes: + - pipeline + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - builds a data pipeline node as a script (not a flow) + - marks the script as a pipeline member with the pipeline annotation in the script's comment syntax (`-- pipeline` for a DuckDB/SQL node, not `// pipeline`) + - declares a schedule trigger and writes its output to a managed DuckLake table + - leaves the result as an AI draft and does not deploy or save it + +- id: global-test-pipeline-two-node-chain + prompt: |- + Build a small data pipeline in the `f/evals/global` folder: one step that + ingests orders into a DuckLake table, and a second step that reads that table + and writes a daily order-count rollup table. Wire the second step to run off + the first step's output. Keep everything as drafts — don't deploy. + initial: ai_evals/fixtures/frontend/global/initial/user_admin_evals_folder.json + runtime: + maxTurns: 14 + validate: + draftCountAtLeast: 2 + forbiddenDrafts: + - type: flow + pathStartsWith: f/evals/global/ + toolExpect: + requiredToolsUsed: + - write_script + forbiddenToolsUsed: + - write_flow + - deploy_workspace_item + - delete_workspace_item + judgeChecklist: + - creates two data pipeline nodes as scripts (not a flow) in f/evals/global + - both scripts carry the pipeline annotation in their comment syntax (`-- pipeline` for DuckDB/SQL nodes, not `// pipeline`) + - the first ingests orders into a DuckLake table + - the second reads that same table and writes a daily rollup, wired to the first step's output asset + - leaves both as AI drafts without deploying + - id: global-path5-create-folder-then-draft prompt: |- Create a new shared folder called "analytics" for our data work, then draft a diff --git a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json b/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json deleted file mode 100644 index cde17acf76..0000000000 --- a/backend/.sqlx/query-0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "0035bf99ce6fc00c7338bebfeb7e79bb9e7bc3d216b84279dee0018603965941" -} diff --git a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json b/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json deleted file mode 100644 index 5ca75fe782..0000000000 --- a/backend/.sqlx/query-02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM asset\n WHERE (workspace_id, path, kind) IN (\n SELECT workspace_id, path, kind FROM (\n SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[], \n $2::varchar[], \n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id \n AND a.path = limits.path \n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "VarcharArray", - "VarcharArray", - { - "Custom": { - "name": "asset_kind[]", - "kind": { - "Array": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - } - } - }, - "Int4Array" - ] - }, - "nullable": [] - }, - "hash": "02e526146f3584cd599dec708e1be48db3b0cd1c74adbfa2e4039377daa016f0" -} diff --git a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json similarity index 51% rename from backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json rename to backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json index 6fd7d38f69..036e8e4b57 100644 --- a/backend/.sqlx/query-40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e.json +++ b/backend/.sqlx/query-0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2", + "query": "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "40a8cf5e87bb489fd172689e9a6f0f1075b878f9916145929b3cd3b1a53b777e" + "hash": "0621faf69b1ef866a95f6310c9651875df409a2d2d72ada629bd71e8abdbbf8b" } diff --git a/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json b/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json new file mode 100644 index 0000000000..c43b9e53a1 --- /dev/null +++ b/backend/.sqlx/query-0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, archived, schema_validation, ws_error_handler_muted, deleted, extra_perms)\n VALUES ('test-workspace', 'f/restricted/item', 314159, 'def main(): return 1', '', '', 'python3', 'test-user', NOW(), false, false, false, false, '{}'::jsonb)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0ff0b7abad7717d7de398f629853b22cb831e91026dcd59d362a6d4382dc240b" +} diff --git a/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json b/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json new file mode 100644 index 0000000000..f2faa9f48a --- /dev/null +++ b/backend/.sqlx/query-12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, params, body, is_table_macro, provider_path FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "params", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "body", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "provider_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "12dec9fa086fd231cdba2ecaa4585b62e55f43d448332ff88d8f036443b1edd7" +} diff --git a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json b/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json deleted file mode 100644 index 911d6c3b07..0000000000 --- a/backend/.sqlx/query-1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n CASE WHEN q.running\n THEN $3::text::jsonb\n ELSE $4::text::jsonb\n END,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id = $1\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n VALUES ($5, $1, $2)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "x", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1437b432d2c23e30eb05443e83069cdb049f65ec299b0778ce14677728cf6346" -} diff --git a/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json new file mode 100644 index 0000000000..f548727abb --- /dev/null +++ b/backend/.sqlx/query-165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "165abf847773c7fe718fa6c832b259bf2dc531bc468d2ab27f76c9c653be0b9a" +} diff --git a/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json new file mode 100644 index 0000000000..058018a8ee --- /dev/null +++ b/backend/.sqlx/query-16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\", is_dev_workspace\n FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_dev_workspace", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "16c7838ecfcea5fd231f2a4766f691a9a11ca7bb9797b81444419d6a72883531" +} diff --git a/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json new file mode 100644 index 0000000000..708922a0cb --- /dev/null +++ b/backend/.sqlx/query-1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, storage, inflight_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, upload_id)\n DO UPDATE SET inflight_bytes = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b05728b33decc39766ccacca50464b5ff9f34e43fbcc38154939461aadfca9f" +} diff --git a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json b/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json deleted file mode 100644 index 75957a4f2f..0000000000 --- a/backend/.sqlx/query-1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - { - "Custom": { - "name": "asset_usage_kind", - "kind": { - "Enum": [ - "script", - "flow", - "job" - ] - } - } - } - ] - }, - "nullable": [] - }, - "hash": "1c5caaaa86e3488549cad179e992172315be5d53dcc266d713da01fd27f310b6" -} diff --git a/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json new file mode 100644 index 0000000000..b84cd8a14e --- /dev/null +++ b/backend/.sqlx/query-1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (workspace_id, name)\n DO UPDATE SET rules = EXCLUDED.rules,\n bypass_groups = EXCLUDED.bypass_groups,\n bypass_users = EXCLUDED.bypass_users\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int4", + "TextArray", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "1cd7c77e7a6a5c13c4ca521098bb07c1d805d21899fe0ebac22132b248ffd242" +} diff --git a/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json b/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json new file mode 100644 index 0000000000..1a3d7ef0e4 --- /dev/null +++ b/backend/.sqlx/query-21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT consumer_path AS \"consumer_path!\", macro_name AS \"macro_name!\"\n FROM macro_usage\n WHERE workspace_id = $1\n AND ($2::text IS NULL OR consumer_path LIKE $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "consumer_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "macro_name!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "21242859ee6b72466c6bfb9d826a7159de4488adccb9349cddfbdd572b4005e9" +} diff --git a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json index a53d131a3f..f9bc97d29d 100644 --- a/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json +++ b/backend/.sqlx/query-212553c83e4dcdc6d045eb2fe2dadbb2860ce52d37a56b2861de1215260ecff8.json @@ -38,7 +38,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } @@ -75,7 +76,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json b/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json new file mode 100644 index 0000000000..f828b906cd --- /dev/null +++ b/backend/.sqlx/query-246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "246e302feda95892d5eb1a43a29d4d4739fd663cf34f00d947f24b47c481f035" +} diff --git a/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json new file mode 100644 index 0000000000..508eb32ed6 --- /dev/null +++ b/backend/.sqlx/query-27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT COALESCE(MAX(depth), 0)::bigint AS \"depth!\" FROM chain\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "depth!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "27a131537cee699ae53088cfd710b70eb81b4e6b8f79d57db9d1d12d7f3ffde2" +} diff --git a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json b/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json deleted file mode 100644 index 642723decf..0000000000 --- a/backend/.sqlx/query-299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '10 minutes'\n RETURNING 1\n ) SELECT count(*) as \"c!\" FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "c!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "299b94a7972443267dd664c178a1704d195a7fc0d4e66e1014a18398e3a294f4" -} diff --git a/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json new file mode 100644 index 0000000000..3ad3666413 --- /dev/null +++ b/backend/.sqlx/query-29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5\n FROM workspace WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "29eb2c40e13d6e1ff7c37a05ab107829242f015a331bf15600986be3963878ee" +} diff --git a/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json new file mode 100644 index 0000000000..44d9f8f36d --- /dev/null +++ b/backend/.sqlx/query-2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS \"height!\" FROM tree\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "height!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "2eb746c4cc5c65e277d4c3a12e0bc9b6a776a4941e68d8e2b31904c677493e2d" +} diff --git a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json similarity index 71% rename from backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json rename to backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json index 50ed7549e8..6163a0f9fb 100644 --- a/backend/.sqlx/query-0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1.json +++ b/backend/.sqlx/query-2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + "query": "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff\n WHERE source_workspace_id = $1 AND fork_workspace_id = $2\n AND NOT EXISTS (\n SELECT 1 FROM ws_specific ws\n WHERE ws.path = workspace_diff.path\n AND ws.item_kind = workspace_diff.kind\n AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id)\n )", "describe": { "columns": [ { @@ -55,5 +55,5 @@ true ] }, - "hash": "0b8e5fe95f4a2855678ca041b50405b698a368626da42dd9f4ce9d0681d016a1" + "hash": "2ec9f88ad80d192a2066764222fdfed7c553de4df8e876ff8589738dea93d0cf" } diff --git a/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json new file mode 100644 index 0000000000..71615c6466 --- /dev/null +++ b/backend/.sqlx/query-3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'resource', $2::varchar\n WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3020d5477b4822f1b0e3b2e4f2947e24754b919f7ee1aa2e7c1cb8c36e9e94b1" +} diff --git a/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json b/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json new file mode 100644 index 0000000000..0624b86e0c --- /dev/null +++ b/backend/.sqlx/query-3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork)\n VALUES ('test-workspace', 'wm-fork-guard-test', 'f/restricted/item', 'script', 1, 0, true, true, true)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3211c7631f46975fc0127f106e84676237a3b37ffa59ff6c9134df79eed3b680" +} diff --git a/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json new file mode 100644 index 0000000000..1ca539c46d --- /dev/null +++ b/backend/.sqlx/query-34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"is_fork!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_fork!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "34b94da001dafbbe66b3b945e71e33128bbc0dcdba9850ccbe4dc278836513bd" +} diff --git a/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json b/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json new file mode 100644 index 0000000000..b814575234 --- /dev/null +++ b/backend/.sqlx/query-3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM skip_workspace_diff_tally WHERE workspace_id IN ('test-workspace', 'wm-fork-guard-test')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "3535965aed37a1cc853b229e062d4aac2dbb2ef7596d819db222fb87187d9c4b" +} diff --git a/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json new file mode 100644 index 0000000000..16063b9a3f --- /dev/null +++ b/backend/.sqlx/query-3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "3571fb1e1aee51850d5789e28025a055b6972eb00e4dc1818f7a68bcf04c1ba7" +} diff --git a/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json b/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json new file mode 100644 index 0000000000..d95cc6dc75 --- /dev/null +++ b/backend/.sqlx/query-36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name)\n SELECT $2, consumer_path, macro_name\n FROM macro_usage WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "36fb69f46ccfb559cd5a25db5fb35bd5a777effb1e61f530f3c1868b407e74e4" +} diff --git a/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json b/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json new file mode 100644 index 0000000000..dcb81f26b8 --- /dev/null +++ b/backend/.sqlx/query-3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 OR macro_name IN (SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "3740ea8dfc666c8b098e55784af185331f8b03eec083fa80c007f6f23019a5d2" +} diff --git a/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json new file mode 100644 index 0000000000..291dc00310 --- /dev/null +++ b/backend/.sqlx/query-3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2 AND created_at > now() - ($3::text)::interval)::bigint as \"other_reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "3a20f8f159b7185940639716b2ae0d5d1c3769d095352002daf8c11d10eac57b" +} diff --git a/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json new file mode 100644 index 0000000000..137af7638b --- /dev/null +++ b/backend/.sqlx/query-3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "3d4b21ca3f6dce2141b0a943d3b1bdb31f26e82fb0dc97bef7114d85959b6129" +} diff --git a/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json b/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json new file mode 100644 index 0000000000..667907e290 --- /dev/null +++ b/backend/.sqlx/query-3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro)\n SELECT $2, name, provider_path, params, body, is_table_macro\n FROM macro_definition WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "3e0c2268afa4ce21ae6da69d2a496e034c48d960930a2c0c340b28b5c44fe87f" +} diff --git a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json b/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json deleted file mode 100644 index 31c0ab4982..0000000000 --- a/backend/.sqlx/query-40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "40bcbfdcae9842c7919eb6dcfe44d844508304700b292059b24c3f74454a7cca" -} diff --git a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json similarity index 57% rename from backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json rename to backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json index 4fe298370b..c12e0cdac4 100644 --- a/backend/.sqlx/query-e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c.json +++ b/backend/.sqlx/query-41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id)\n VALUES ($1, $2, $3, $4)", + "query": "INSERT INTO workspace\n (id, name, owner, parent_workspace_id, is_dev_workspace)\n VALUES ($1, $2, $3, $4, $5)", "describe": { "columns": [], "parameters": { @@ -8,10 +8,11 @@ "Varchar", "Varchar", "Varchar", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "e947340c7d40f6b9536e7a24fa84bee393e24f6f39f67e0b9e20e9cb8f04244c" + "hash": "41ce08f45b09532cbab6fb039703f08d6476d00d85a02ea3b19aa559b9ec1056" } diff --git a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json similarity index 76% rename from backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json rename to backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json index 0265d7d2b5..76eab1d117 100644 --- a/backend/.sqlx/query-1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078.json +++ b/backend/.sqlx/query-42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ", + "query": "INSERT INTO usr\n (workspace_id, email, username, is_admin)\n SELECT $1, email, username, is_admin FROM usr\n WHERE workspace_id = $3 AND email = $2\n ON CONFLICT DO NOTHING\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "1a9f2ed5045016a3953db335957b26f41efc8a3cad7af7bc8fe97df6a5bf5078" + "hash": "42a0ba479ff164cc190c350927e13902ed94816142faf15446b4b9f19c3097d7" } diff --git a/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json b/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json new file mode 100644 index 0000000000..1146134a34 --- /dev/null +++ b/backend/.sqlx/query-447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro) VALUES ($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "447dd940e6ee86be8eab6982c22a7fd0fe229643573509bd1705a691ff7c4ba2" +} diff --git a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json b/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json deleted file mode 100644 index 9e7ffed082..0000000000 --- a/backend/.sqlx/query-451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO runnable_settings (hash, debouncing_settings, concurrency_settings)\n VALUES ($1, $2, $3)\n ON CONFLICT (hash)\n DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "451d9cde90d14071e21ffb5f615052b7ba7fc315fc301ed5c0ff50d9a3ab0d4a" -} diff --git a/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json new file mode 100644 index 0000000000..cb6bb20397 --- /dev/null +++ b/backend/.sqlx/query-453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT COUNT(DISTINCT id) AS \"count!\" FROM tree WHERE id != $1 AND NOT deleted\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "453c8dc05f4946a30fbbc517b5beaaee42e954e0b4ed0bd5ba6a5b308894d6ba" +} diff --git a/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json new file mode 100644 index 0000000000..18af751ca0 --- /dev/null +++ b/backend/.sqlx/query-472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtext('workspace_multipart_inflight'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "472d351f0bba2dc3404d83aec131cd50b297f80e5392715f54f32fab4bf346fc" +} diff --git a/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json new file mode 100644 index 0000000000..5f114c0156 --- /dev/null +++ b/backend/.sqlx/query-4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH potential AS (\n SELECT email, operator FROM usr WHERE is_service_account IS false\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user\n WHERE email NOT IN (SELECT email FROM password WHERE disabled IS true)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "authors!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "4865e22673a7886ff84cfe5ee1114d66b1678bfe3028df89dff4e343e2f4ab44" +} diff --git a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json b/backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json similarity index 74% rename from backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json rename to backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json index 3470285737..64330e91cf 100644 --- a/backend/.sqlx/query-394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db.json +++ b/backend/.sqlx/query-4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ORDER BY path, created_at DESC\n ", + "query": "\n SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND (content LIKE '%' || $2 || '%' OR path = ANY($3))\n ORDER BY path, created_at DESC\n ", "describe": { "columns": [ { @@ -17,7 +17,8 @@ "parameters": { "Left": [ "Text", - "Text" + "Text", + "TextArray" ] }, "nullable": [ @@ -25,5 +26,5 @@ false ] }, - "hash": "394e2598880aff8a7f4ee05c3fe748be58b6381f5fae5619d8376daefc3b21db" + "hash": "4add520575717d449a3758326777fee03012a3fa0fb943239e004cc4a7067501" } diff --git a/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json b/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json new file mode 100644 index 0000000000..06b8754d5f --- /dev/null +++ b/backend/.sqlx/query-4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO notify_event (channel, payload) VALUES ('notify_macro_registry_change', $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4b03b55595bbc02e63fc920cbaf4a1503df8811d2d212f1a97ffd00599aeb4c7" +} diff --git a/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json new file mode 100644 index 0000000000..8e9e6a8713 --- /dev/null +++ b/backend/.sqlx/query-4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4e4efea2b8d3b0bd2717b27d1895b00e8dba07aa817ee9bfbe2271d41c9b411a" +} diff --git a/backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json similarity index 66% rename from backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json rename to backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json index 1e39bfdaab..bdb39d134f 100644 --- a/backend/.sqlx/query-6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b.json +++ b/backend/.sqlx/query-4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n ORDER BY d.email NULLS LAST", + "query": "SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as \"username?\",\n d.created_at as \"draft_saved_at!\"\n FROM draft d\n LEFT JOIN usr u\n ON u.workspace_id = d.workspace_id\n AND u.email = d.email\n LEFT JOIN password p\n ON p.email = d.email\n AND p.super_admin = true\n WHERE d.workspace_id = $1\n AND d.path = $2\n AND d.typ = $3\n AND (d.email IS NULL OR d.email <> $4)\n ORDER BY d.email NULLS LAST", "describe": { "columns": [ { @@ -60,5 +60,5 @@ false ] }, - "hash": "6fbd5826ae1fafd7bbb218324e3e2d85231b889743de637eb35e756da29ad47b" + "hash": "4efaa4c3c14987fe95a508cc0cbac91ee793ad6f5ceed5b3eda9b48617c5434d" } diff --git a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json b/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json deleted file mode 100644 index d165e78ad2..0000000000 --- a/backend/.sqlx/query-5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM v2_job WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "5190528997a879981a87420ddf3d28c978c8a5876f5c1ac1613391e86ffb550f" -} diff --git a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json similarity index 57% rename from backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json rename to backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json index fce125c6d5..35b70a239c 100644 --- a/backend/.sqlx/query-9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9.json +++ b/backend/.sqlx/query-5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", + "query": "SELECT bool_and(operator) FROM (\n SELECT operator FROM usr WHERE email = $1 AND is_service_account IS false\n UNION ALL\n SELECT operator FROM workspace_invite WHERE email = $1\n ) t", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "9c85ba8d41bedbcb5466f44a7d4cf6b4946e1fd337f00d243f518283783833c9" + "hash": "5312b8db714139a94d7ff1c0794af063c36ac17e9d331cd9980b91b28d713c72" } diff --git a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json b/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json deleted file mode 100644 index f634fc2d4b..0000000000 --- a/backend/.sqlx/query-57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH mine AS (\n SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1\n ), claim_self AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE id = $1 AND consumed_at IS NULL\n RETURNING debounce_batch\n ), claim_rest AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)\n AND id <> $1 AND consumed_at IS NULL\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM mine) AS \"had_row!\",\n (SELECT debounce_batch FROM claim_self) AS claimed_batch,\n (SELECT consumed_by FROM mine) AS prev_consumed_by,\n ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "had_row!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "claimed_batch", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "prev_consumed_by", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "claimed_ids!", - "type_info": "UuidArray" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null, - null, - null, - null - ] - }, - "hash": "57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89" -} diff --git a/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json new file mode 100644 index 0000000000..cd2dd23319 --- /dev/null +++ b/backend/.sqlx/query-59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(inflight_bytes - target_existing_size, 0)), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3))::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "59c1137f718d442c4e99fc07365ebbe13e6298749b19570f9f8c4a0c7958b0ce" +} diff --git a/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json new file mode 100644 index 0000000000..be54b4a25d --- /dev/null +++ b/backend/.sqlx/query-5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "5cd47ddc6a0181c8e23998adae103ff9c96d613fb2d9b698361f4ede8031e5d2" +} diff --git a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json similarity index 63% rename from backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json rename to backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json index bb80e8d19a..a9cdc35dce 100644 --- a/backend/.sqlx/query-c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6.json +++ b/backend/.sqlx/query-63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id,\n workspace.is_dev_workspace,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings,\n usr.disabled\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -30,11 +30,16 @@ }, { "ordinal": 5, + "name": "is_dev_workspace", + "type_info": "Bool" + }, + { + "ordinal": 6, "name": "operator_settings", "type_info": "Jsonb" }, { - "ordinal": 6, + "ordinal": 7, "name": "disabled", "type_info": "Bool" } @@ -50,9 +55,10 @@ false, true, true, + false, null, false ] }, - "hash": "c095a9658c542efc9f0255a1b536d2fd8a25fe4cd57c223db7d744493f8470c6" + "hash": "63d323be5cacb7a02283d7d82c79bc408b9a33a228b80e88ec3f6432944a7c19" } diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json deleted file mode 100644 index cb20ec2ffb..0000000000 --- a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" -} diff --git a/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json b/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json new file mode 100644 index 0000000000..83e2711cf5 --- /dev/null +++ b/backend/.sqlx/query-689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name AS \"name!\", provider_path AS \"provider_path!\",\n params AS \"params!\", is_table_macro AS \"is_table_macro!\"\n FROM macro_definition\n WHERE workspace_id = $1\n ORDER BY provider_path, name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "provider_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "params!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "689c85d30f898f237c8a3a1e883440276ea6faee8589ff0febedf6917977a40e" +} diff --git a/backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json b/backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json similarity index 57% rename from backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json rename to backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json index 6ae3f60e49..4dfdcaab97 100644 --- a/backend/.sqlx/query-dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9.json +++ b/backend/.sqlx/query-6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT email FROM usr WHERE workspace_id = $1 AND username = $2", + "query": "SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path != $2", "describe": { "columns": [ { "ordinal": 0, - "name": "email", + "name": "name", "type_info": "Varchar" } ], @@ -19,5 +19,5 @@ false ] }, - "hash": "dadf78bae0299b24f6798e7a2cc86f0ecfcf63daa0b185994a410eb2fe41fad9" + "hash": "6939fbbaaacd5d5a34016ed9eb295ba3ef329059366632dd94262f2e1bdfed97" } diff --git a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json b/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json deleted file mode 100644 index e885c72039..0000000000 --- a/backend/.sqlx/query-70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'\n RETURNING 1\n ) SELECT count(*) FROM del", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18" -} diff --git a/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json new file mode 100644 index 0000000000..acfa08d0bf --- /dev/null +++ b/backend/.sqlx/query-71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE tree AS (\n SELECT id, 0 AS depth FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, tree.depth + 1 FROM workspace w\n JOIN tree ON w.parent_workspace_id = tree.id\n WHERE tree.depth < 20\n )\n SELECT id AS \"id!\" FROM tree WHERE id != $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "71ee328f016bac75a9050dc3c13dc898c0b40fe6f1c4e5c631b6f784f7e73a3d" +} diff --git a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json deleted file mode 100644 index c5266c66b5..0000000000 --- a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_path!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "kind!", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - null - ] - }, - "hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc" -} diff --git a/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json new file mode 100644 index 0000000000..f425d1eebd --- /dev/null +++ b/backend/.sqlx/query-7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT (parent_workspace_id IS NOT NULL) AS \"has_parent!\" FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_parent!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "7853a596a01884070455e68bbf8ab2afa79a5b5b8521f68ab19f0470f0265c7c" +} diff --git a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json b/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json deleted file mode 100644 index f7d3361d99..0000000000 --- a/backend/.sqlx/query-78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "78bb75578a880715fb482445883e0e762f289b7695f29bfd44fa23323c7e8523" -} diff --git a/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json new file mode 100644 index 0000000000..e1cf9bc17f --- /dev/null +++ b/backend/.sqlx/query-7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "7b950d49cb1cc7f9bf8032c9e7655c49027337a9eca1f1ded13dbae6475e3376" +} diff --git a/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json b/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json new file mode 100644 index 0000000000..f5fec50340 --- /dev/null +++ b/backend/.sqlx/query-7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE macro_usage SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "7e3d7f7516167c320c688c2406076aeebb9081dd36b45f609823dfbb4d3b8e9d" +} diff --git a/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json new file mode 100644 index 0000000000..e171d44af5 --- /dev/null +++ b/backend/.sqlx/query-7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7fa8b53615a7cb678f9d5caedbc44cfa928469ce3c1a53ec273ecc997f6e61f8" +} diff --git a/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json b/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json new file mode 100644 index 0000000000..384ea81940 --- /dev/null +++ b/backend/.sqlx/query-80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by)\n VALUES ('test-workspace', 'restricted', 'restricted', ARRAY['u/test-user']::varchar[], '{\"u/test-user\": true}'::jsonb, '', 'test-user')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "80f526a401d5f26021b5eb424e7481e28c360367f144f172b746cd11abfa67a2" +} diff --git a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json b/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json deleted file mode 100644 index 7a0c5578b0..0000000000 --- a/backend/.sqlx/query-8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO background_task_state (name, value)\n SELECT $1, jsonb_build_object(\n 'last_xmin', txid_snapshot_xmin(txid_current_snapshot())::bigint,\n 'last_ts', '1970-01-01T00:00:00+00:00')\n WHERE NOT EXISTS (SELECT 1 FROM global_settings WHERE name = $2)\n ON CONFLICT (name) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "8192986cd6106ed060b3d68dbc21e5bb34f5e68a5c6ac455a9d423188af77b23" -} diff --git a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json b/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json deleted file mode 100644 index c0573a8fab..0000000000 --- a/backend/.sqlx/query-82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace SELECT $1, $2, owner, false, premium FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "82185eb02e03e3dd1a4b5a3f22c3b60169989703ee33d14ab348301885c9d745" -} diff --git a/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json new file mode 100644 index 0000000000..673becc2ff --- /dev/null +++ b/backend/.sqlx/query-8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "deleted", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "8288b3916022adcbb28da2a0e4329cba84759b8a590546aea8f0ddab4f861c38" +} diff --git a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json deleted file mode 100644 index 96587ed1d0..0000000000 --- a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "runnable_settings_handle", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true - ] - }, - "hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2" -} diff --git a/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json new file mode 100644 index 0000000000..0859d3e071 --- /dev/null +++ b/backend/.sqlx/query-84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "84cbf9623a989dc16be1f8681c95eed9ae3e8d9c1552f396b6773767087bcfec" +} diff --git a/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json new file mode 100644 index 0000000000..7250aff598 --- /dev/null +++ b/backend/.sqlx/query-854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at < now() - ($2::text)::interval", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "854c2e57362df2b6907082c660f01d9c16a9d4b5b422159fc239ff5502a7b71f" +} diff --git a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json b/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json deleted file mode 100644 index f65ea6d2d4..0000000000 --- a/backend/.sqlx/query-8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "created_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "8fef781ecef8bb7e4f5172adbab1d1eefa5cf05ff6521ee605af7ac2b90fa60b" -} diff --git a/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json new file mode 100644 index 0000000000..3f159707b6 --- /dev/null +++ b/backend/.sqlx/query-90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ws_specific (workspace_id, item_kind, path)\n SELECT $1::varchar, 'variable', $2::varchar\n WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "90d21ee2276b41a4dd3dd8ed12f36e00b8d4fc27a2d72b74dff2dace900db75c" +} diff --git a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json b/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json deleted file mode 100644 index 569a5122ba..0000000000 --- a/backend/.sqlx/query-910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\"\n FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null - ] - }, - "hash": "910b9b8afb3df5e437e43ff4adc97d0a03f6c26f0b14fc92900f7600e70a7a8b" -} diff --git a/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json b/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json new file mode 100644 index 0000000000..34c18e2633 --- /dev/null +++ b/backend/.sqlx/query-919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag FROM flow WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "919bee5afcaba361f23760b5f64ce48f63e4d6d3e2d4fa9247b50963112f47ad" +} diff --git a/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json new file mode 100644 index 0000000000..c0207b6bc7 --- /dev/null +++ b/backend/.sqlx/query-93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_multipart_inflight\n (workspace_id, upload_id, part_id, storage, part_bytes, target_existing_size)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (workspace_id, upload_id, part_id)\n DO UPDATE SET part_bytes = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "93d95d25c6b2398faa416646f5daaff40a634fc5530e202f65fad6a241f2671a" +} diff --git a/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json new file mode 100644 index 0000000000..253fe79753 --- /dev/null +++ b/backend/.sqlx/query-954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n -- reservation of every OTHER in-flight upload\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id <> $2\n AND created_at > now() - ($4::text)::interval\n GROUP BY upload_id) t)::bigint as \"other_reserved!\",\n -- this upload's already-recorded parts, excluding the part being (re)uploaded\n (SELECT COALESCE(SUM(part_bytes), 0)\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2 AND part_id <> $3)::bigint as \"this_other_parts!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "other_reserved!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "this_other_parts!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "954ead4f28726b31e2cafbe04d3ed3e9625f54cd89cfd595665e4473c5f3ea6c" +} diff --git a/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json b/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json new file mode 100644 index 0000000000..07a14677e4 --- /dev/null +++ b/backend/.sqlx/query-9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9cfebb5bfee0d7d78f4e71c106a6aa1cb9aae18a6c3a1e5d6b36697ed063b50c" +} diff --git a/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json b/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json new file mode 100644 index 0000000000..64dfe64198 --- /dev/null +++ b/backend/.sqlx/query-9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM password WHERE username IS NOT NULL)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9ee6263223bd415d7ac21328a8ebc731d43e7331b9fd83282a87f8e2fd8fa770" +} diff --git a/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json b/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json new file mode 100644 index 0000000000..9ddc929622 --- /dev/null +++ b/backend/.sqlx/query-a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT name, provider_path FROM macro_definition WHERE workspace_id = $1 AND name = ANY($2) LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "provider_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a08e1e22ec6cc25bb8939e488a1c7cf162f2fa7c5f67f8a9e0071e5b7f238dcb" +} diff --git a/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json new file mode 100644 index 0000000000..2b9b0619a1 --- /dev/null +++ b/backend/.sqlx/query-a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a22611ed3b7a6caa76a218454494c96e1371956bb303eb8d3e3268139cd8fce0" +} diff --git a/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json new file mode 100644 index 0000000000..0c3fdb8d0a --- /dev/null +++ b/backend/.sqlx/query-a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_try_advisory_xact_lock(hashtext('workspace_storage_usage'), hashtext($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_try_advisory_xact_lock", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a488e5a6492562bbc5fa86ab348b610b028e88d5d2dbbea44377bf9dc416cd03" +} diff --git a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json b/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json deleted file mode 100644 index 819928ddc1..0000000000 --- a/backend/.sqlx/query-a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id)\n SELECT $1, $2, owner, false, premium,\n CASE WHEN $4 THEN parent_workspace_id ELSE NULL END\n FROM workspace WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "a54efa4a7466e61fd54d8fe293cb775225dcb430026cebe15ba4994ac636514d" -} diff --git a/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json b/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json new file mode 100644 index 0000000000..986638f649 --- /dev/null +++ b/backend/.sqlx/query-a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 OR consumer_path = $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6d54bb1e2ad153e0c89a4c71ade07eab23e2b35e58f812c69ba29de2d8e2fa5" +} diff --git a/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json new file mode 100644 index 0000000000..691016a610 --- /dev/null +++ b/backend/.sqlx/query-a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_object_keys(large_file_storage->'secondary_storage') as \"key!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a7017ba623279e614ef31a3026c5274749d7a436a753813da098758514a7493d" +} diff --git a/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json new file mode 100644 index 0000000000..0f1bcc2271 --- /dev/null +++ b/backend/.sqlx/query-a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT inflight_bytes, target_existing_size FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND upload_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "inflight_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "target_existing_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a8019c47b15baa9497ae44935456fea272ac289fc17ea9fc5aa237f77778b58f" +} diff --git a/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json new file mode 100644 index 0000000000..5d20f83f9d --- /dev/null +++ b/backend/.sqlx/query-a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a981f9b6424969e5fd72fb18c20e2910a138a848f3f6674dec58053572682077" +} diff --git a/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json new file mode 100644 index 0000000000..b022524bbb --- /dev/null +++ b/backend/.sqlx/query-a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "a999a5cf7b481d852222311a34959f38114926063718668907587d1b80dfc75e" +} diff --git a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json deleted file mode 100644 index d12d305b92..0000000000 --- a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b" -} diff --git a/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json new file mode 100644 index 0000000000..1148679230 --- /dev/null +++ b/backend/.sqlx/query-ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(SUM(bytes), 0)::bigint as \"total!\",\n COALESCE(MIN(computed_at) < now() - interval '10 minutes', true) as \"stale!\"\n FROM workspace_storage_usage WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ab0dca3f021243d71222643165548af40192832c07c1c7049def3a80057bb683" +} diff --git a/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json b/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json new file mode 100644 index 0000000000..e543db2a30 --- /dev/null +++ b/backend/.sqlx/query-ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ad07e40790b947d1e8da17dd4ae30365210bba4bff0b7e32d06c53ed13572360" +} diff --git a/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json b/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json new file mode 100644 index 0000000000..1b4082b18a --- /dev/null +++ b/backend/.sqlx/query-ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(\n (SELECT email FROM usr WHERE workspace_id = $1 AND username = $2),\n (SELECT email FROM password WHERE (username = $2 OR email = $2) AND super_admin = true)\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "ae2293c7d9d2b5dad61b27db3b29d5f87fc6b8d53fe6a073eef488533cbd3ed3" +} diff --git a/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json b/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json new file mode 100644 index 0000000000..545a9c161b --- /dev/null +++ b/backend/.sqlx/query-b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_definition WHERE workspace_id = $1 AND (provider_path = $2 OR provider_path = $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b3a9ef177aecfbcb31abba9f25e1292668a687ca01ae1c14c0ed3b5cf10fcb03" +} diff --git a/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json b/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json new file mode 100644 index 0000000000..c6022d3013 --- /dev/null +++ b/backend/.sqlx/query-b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_diff\n (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork)\n VALUES\n ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost', 'http_trigger', 1, 0, true, false, true),\n ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost_behind', 'http_trigger', 0, 1, true, true, false)", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "b55b0f11b8d73ff9fd19da3bc5555e8365dbd4766f273ef3f68e734972a96cd6" +} diff --git a/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json new file mode 100644 index 0000000000..94cf77ebc5 --- /dev/null +++ b/backend/.sqlx/query-b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via)\n SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via\n FROM usr WHERE workspace_id = $2\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "b98844926ff127c528ed3e7bc63bf1ebed0192be267983e8fbd18f79997e6142" +} diff --git a/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json new file mode 100644 index 0000000000..96608e1bc2 --- /dev/null +++ b/backend/.sqlx/query-bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, GREATEST($3::bigint, 0), to_timestamp(0))\n ON CONFLICT (workspace_id, storage)\n DO UPDATE SET bytes = GREATEST(workspace_storage_usage.bytes + $3::bigint, 0)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "bb8318ddc8e2235dce5c832ff2b0ebf972bcc7f821fbf1e0171266efd8d46616" +} diff --git a/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json new file mode 100644 index 0000000000..514024eb8d --- /dev/null +++ b/backend/.sqlx/query-bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH RECURSIVE chain AS (\n SELECT id, parent_workspace_id, 0 AS depth\n FROM workspace WHERE id = $1\n UNION ALL\n SELECT w.id, w.parent_workspace_id, chain.depth + 1\n FROM workspace w\n JOIN chain ON w.id = chain.parent_workspace_id\n WHERE chain.depth < 20\n )\n SELECT id AS \"id!\" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bcb36b5d38a07dbdfed71983aec1af05c5a6882f988ba5e0ec1f63828353987b" +} diff --git a/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json new file mode 100644 index 0000000000..35d3ace40e --- /dev/null +++ b/backend/.sqlx/query-bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_diff SET has_changes = NULL\n WHERE path = $2 AND kind = $3\n AND ($1 IN (source_workspace_id, fork_workspace_id))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "bf6e1abbab6bdf0e67eefa5f807a26b06d24a7b3e292ae4be97fbbb197d468db" +} diff --git a/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json new file mode 100644 index 0000000000..a1f7783645 --- /dev/null +++ b/backend/.sqlx/query-bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(\n SELECT 1 FROM workspace\n WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "bfa4fb5998dec9baf89a2d950186e2dc9c0aecf7831ce66082a40f17cc5ac0de" +} diff --git a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json b/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json deleted file mode 100644 index 7361b645b5..0000000000 --- a/backend/.sqlx/query-c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch, consumed_at) VALUES\n ($1, nextval('debounce_batch_seq'), now() - interval '20 minutes'),\n ($2, nextval('debounce_batch_seq'), now() - interval '1 minute'),\n ($3, nextval('debounce_batch_seq'), NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Uuid", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c886e8af0fc8a3999a813371855c0053571e79960280f0714616d13a456d7bed" -} diff --git a/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json b/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json new file mode 100644 index 0000000000..c1e5495204 --- /dev/null +++ b/backend/.sqlx/query-c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1 AND macro_name = ANY($2) AND consumer_path != $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c961131f349c68b91b0aea9805fd592276fa08da3f13f9c74aeedb666a55df05" +} diff --git a/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json b/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json new file mode 100644 index 0000000000..8f0841180b --- /dev/null +++ b/backend/.sqlx/query-cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT m.name AS \"name!\", m.params AS \"params!\", m.body AS \"body!\",\n m.is_table_macro AS \"is_table_macro!\", m.provider_path AS \"provider_path!\"\n FROM macro_definition m\n WHERE m.workspace_id = $1\n AND EXISTS (\n SELECT 1 FROM script s\n WHERE s.workspace_id = m.workspace_id\n AND s.path = m.provider_path\n AND s.archived = false\n AND s.deleted = false\n )\n ORDER BY m.provider_path, m.name", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "params!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "body!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "is_table_macro!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "provider_path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false + ] + }, + "hash": "cb04b72b25b7bd1163316be6a4e25e934a5a7677a9fdc7ea62f9dadbace15ec3" +} diff --git a/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json b/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json new file mode 100644 index 0000000000..1419a1a20c --- /dev/null +++ b/backend/.sqlx/query-cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE macro_definition SET workspace_id = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cb1b8adfe43c193d8b97afc3b04b6ebebd50e943263d6884bcdd322faeaa8345" +} diff --git a/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json new file mode 100644 index 0000000000..abb128f463 --- /dev/null +++ b/backend/.sqlx/query-ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT storage, bytes, computed_at FROM workspace_storage_usage\n WHERE workspace_id = $1 ORDER BY storage", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "storage", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "computed_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "ccff2c556f1171bd10b09b1aa1d2d8ced2d4a75eef905cb40cb5be6eb8bd5e2b" +} diff --git a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json b/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json deleted file mode 100644 index e900b0f9e0..0000000000 --- a/backend/.sqlx/query-d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH ids AS (\n SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )\n ) SELECT args->>$2 FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d0e826043e5a129ae6768c274c67b6254ff6c5fd450ecdab886a3183a894d266" -} diff --git a/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json b/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json new file mode 100644 index 0000000000..fe92251342 --- /dev/null +++ b/backend/.sqlx/query-d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usr (workspace_id, email, username, is_admin, role)\n VALUES ('wm-fork-guard-test', 'test2@windmill.dev', 'test-user-2', true, 'Admin')", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "d3334846d5928e9b260d17e7ca99f6f24e83e019d6cd1590c42b14524c00f4d3" +} diff --git a/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json b/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json new file mode 100644 index 0000000000..deed0f1cdb --- /dev/null +++ b/backend/.sqlx/query-d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM macro_usage WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d350eb5358c62526e70eba991fdd817a998bef51ac27e83942afc548640fb785" +} diff --git a/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json new file mode 100644 index 0000000000..1fd1fbaece --- /dev/null +++ b/backend/.sqlx/query-d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_storage_usage WHERE workspace_id = $1 AND storage != ALL($2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "d38e25120ec7bfcbd210d967ef2406d1773657c62d67e55dac666f9469d50e6a" +} diff --git a/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json new file mode 100644 index 0000000000..9f8b86b4e6 --- /dev/null +++ b/backend/.sqlx/query-d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2\n AND NOT EXISTS (\n SELECT 1 FROM workspace\n WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "d3ce4e7f3dd10548197734a0cf38c644f70be809dcfe7d973147962530c601d5" +} diff --git a/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json new file mode 100644 index 0000000000..5609adaad8 --- /dev/null +++ b/backend/.sqlx/query-d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (SELECT COALESCE(SUM(bytes), 0) FROM workspace_storage_usage WHERE workspace_id = $1)::bigint as \"committed!\",\n COALESCE(\n (SELECT MIN(computed_at) FROM workspace_storage_usage WHERE workspace_id = $1) < now() - interval '10 minutes',\n true) as \"stale!\",\n (SELECT COALESCE(SUM(GREATEST(t.total - t.existing, 0)), 0)\n FROM (SELECT SUM(part_bytes) as total, MAX(target_existing_size) as existing\n FROM workspace_multipart_inflight\n WHERE workspace_id = $1 AND created_at > now() - ($2::text)::interval\n AND ($3::text IS NULL OR upload_id <> $3)\n GROUP BY upload_id) t)::bigint as \"reserved!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "committed!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "stale!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "reserved!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "d4923137cf9b6bf06e0e21ba907e7c80dd0ef92d78224792bec4c3323e14023c" +} diff --git a/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json b/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json new file mode 100644 index 0000000000..3a5531b9a6 --- /dev/null +++ b/backend/.sqlx/query-d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT usage_path FROM asset\n WHERE workspace_id = $1\n AND kind = 'ducklake'\n AND path = $2\n AND usage_kind = 'script'\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d4f047b0a672d5bfe9fd7db60f11563ef05db7cc54a958a72384b08a69eb784e" +} diff --git a/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json new file mode 100644 index 0000000000..4a511c2d6e --- /dev/null +++ b/backend/.sqlx/query-d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "is_admin", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d74dfaa8cb9fca89c2e21f810cd45ff7b595c7c50f5e6ff2733eded6ef544996" +} diff --git a/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json new file mode 100644 index 0000000000..48a5c693ac --- /dev/null +++ b/backend/.sqlx/query-d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d8f234765e2de89f780937a505c236c5be9b36b4df95735fedb44ed944606ac8" +} diff --git a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json deleted file mode 100644 index 2ba317edc6..0000000000 --- a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "kind!: AssetKind", - "type_info": { - "Custom": { - "name": "asset_kind", - "kind": { - "Enum": [ - "s3object", - "resource", - "variable", - "ducklake", - "datatable", - "volume" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "path!", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false - ] - }, - "hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b" -} diff --git a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json b/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json deleted file mode 100644 index 67ce9d8719..0000000000 --- a/backend/.sqlx/query-deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Varchar", - "TextArray" - ] - }, - "nullable": [] - }, - "hash": "deac41298e8b0d0870e314fef0813c24dd55d63bda78a0a5f35ed6f22bea6bef" -} diff --git a/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json b/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json new file mode 100644 index 0000000000..c3db56398e --- /dev/null +++ b/backend/.sqlx/query-e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (path) path AS \"path!\", content AS \"content!\"\n FROM script\n WHERE workspace_id = $1\n AND language = 'duckdb'::script_lang\n AND archived = false\n AND deleted = false\n AND path != $2\n ORDER BY path, created_at DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "content!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "e17d7fb386ec45386401a20d97b50d224da5beb74d289fe17ea960887aa50f1a" +} diff --git a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json b/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json deleted file mode 100644 index f38c023cb3..0000000000 --- a/backend/.sqlx/query-e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH potential AS (\n SELECT email, operator FROM usr\n UNION\n SELECT email, operator FROM workspace_invite\n ),\n per_user AS (\n SELECT email, bool_and(operator) AS only_operator FROM potential GROUP BY email\n )\n SELECT\n COUNT(*) FILTER (WHERE NOT only_operator) AS \"authors!\",\n COUNT(*) FILTER (WHERE only_operator) AS \"operators!\"\n FROM per_user", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "authors!", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "operators!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null, - null - ] - }, - "hash": "e1ada31c1625b453c2ff85edbcd7ad51a4cd5cbdc2fa34038530070d6a579455" -} diff --git a/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json b/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json new file mode 100644 index 0000000000..4609e7adf4 --- /dev/null +++ b/backend/.sqlx/query-e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2) AS \"e!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "e!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e26401ac6ecefe4e11306f2c714165d9a755102d1df51ffe63abf5900bcdb42c" +} diff --git a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json b/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json deleted file mode 100644 index a1b52e81fd..0000000000 --- a/backend/.sqlx/query-e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT status = 'success' AS \"success!\",\n result AS \"result: Json>\",\n started_at AS \"started_at!\"FROM v2_job j JOIN v2_job_completed USING (id)\n WHERE j.workspace_id = $1 AND trigger_kind = 'schedule' AND trigger = $2\n AND parent_job IS NULL\n AND runnable_path = $3\n AND j.id != $4\n ORDER BY created_at DESC\n LIMIT $5", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "success!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "started_at!", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Uuid", - "Int8" - ] - }, - "nullable": [ - null, - true, - true - ] - }, - "hash": "e6a8ddfd74ebab55ede5989fd7d09b246b6f84e49c659bc8e2c7b66cfec6d976" -} diff --git a/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json new file mode 100644 index 0000000000..4f0731f5e4 --- /dev/null +++ b/backend/.sqlx/query-e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_storage_usage (workspace_id, storage, bytes, computed_at)\n VALUES ($1, $2, $3, now())\n ON CONFLICT (workspace_id, storage) DO UPDATE SET bytes = $3, computed_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e82d854f3f8736a9ba7e2604ac505b693965f1bca6fef7418b19f98671c9204c" +} diff --git a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json b/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json deleted file mode 100644 index 146c7f05b3..0000000000 --- a/backend/.sqlx/query-ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT concurrency_settings, debouncing_settings FROM runnable_settings WHERE hash = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "concurrency_settings", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "debouncing_settings", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "ebbbd069e0f33be9609604025d159fe1ecbefc2e9c11f7c4900b7121d4367e01" -} diff --git a/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json new file mode 100644 index 0000000000..1930427b94 --- /dev/null +++ b/backend/.sqlx/query-ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS \"developers!\",\n COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS \"operators!\"\n FROM usr WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "developers!", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "operators!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "ebce1633afe3d3f8a1a68bc8d26adcdd88b137bb892437a3ca67fd716772512b" +} diff --git a/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json new file mode 100644 index 0000000000..844dfb3e47 --- /dev/null +++ b/backend/.sqlx/query-f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, description, resource_type, created_by, edited_at, labels)\n VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Text", + "Varchar", + "Varchar", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f1f79711f131ff4116489153db1b71e528c66fab1508e5c41b63bc5b9077a08c" +} diff --git a/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json b/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json new file mode 100644 index 0000000000..9e32600de9 --- /dev/null +++ b/backend/.sqlx/query-f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "f629c27daf692d7055fa299edaae110a5d6dcdc65aa293b1a15d67dee5308ddc" +} diff --git a/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json new file mode 100644 index 0000000000..7f967b60a5 --- /dev/null +++ b/backend/.sqlx/query-fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "fa04b3660f1f90c39c3d12f39c51df8f25a3bdae60aeaed313612cb858040c8d" +} diff --git a/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json b/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json new file mode 100644 index 0000000000..626c541556 --- /dev/null +++ b/backend/.sqlx/query-fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = true", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "fc1663ecda7167dc12c0fd5a206cbddf85e55f6c7ef226cdd66e025c002bac98" +} diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 9ff2c0be90..ff5d9b2f22 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -6,3 +6,18 @@ - **DB schema**: `backend/summarized_schema.txt` - **API routes entry point**: `windmill-api/src/lib.rs` - **OpenAPI spec**: `windmill-api/openapi.yaml` +- **DuckDB local jobs**: build the dynamic FFI library before running DuckDB scripts locally: + ```bash + cd backend/windmill-duckdb-ffi-internal && ./build_dev.sh + ``` + Re-run after clean builds or when `target/debug/libwindmill_duckdb_ffi_internal.*` is missing. + +## Cloud vs self-hosted gating + +The `cloud` cargo feature is compiled into **all** EE builds, so `#[cfg(feature = "cloud")]` is **not** a "cloud-only" runtime gate — it only means the code is present. The real gate for behavior specific to the managed cloud (app.windmill.dev) is the runtime flag `*CLOUD_HOSTED` (`windmill_common::worker::CLOUD_HOSTED`, from the `CLOUD_HOSTED` env var; note it's loaded from `.env` via `dotenv`, so it won't show in `/proc//environ` — check the running behavior, not the exec env). + +Cloud-only logic must be behind `if *CLOUD_HOSTED { ... }`: feature-gate the helper so it compiles, then **runtime-gate the call**. `#[cfg(feature = "cloud")]` on its own is only sufficient for: +- pure helper/struct definitions (they only run when a gated caller invokes them), +- code already inside an `if *CLOUD_HOSTED { ... }` block, +- handlers that early-return on `!*CLOUD_HOSTED`, +- idempotent no-ops that are harmless off-cloud (e.g. cache invalidation). diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 7072389c21..749dc880eb 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -237,9 +237,9 @@ checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02882884d3e1bc524fb12c79f107f6ad0e1cfd498c536ffb494301740995dfe" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" @@ -517,7 +517,7 @@ dependencies = [ "futures-core", "libc", "portable-atomic", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "tokio", "tokio-stream", "xattr", @@ -775,9 +775,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -785,14 +785,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1530,7 +1531,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "shlex 1.3.0", "syn 2.0.118", ] @@ -1550,7 +1551,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "shlex 1.3.0", "syn 2.0.118", ] @@ -2601,9 +2602,9 @@ dependencies = [ [[package]] name = "curl-sys" -version = "0.4.89+curl-8.20.0" +version = "0.4.90+curl-8.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d680779285438f2d0927485973ab45b212ea990bddb80de8a55a1e3c1d9ba22" +checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" dependencies = [ "cc", "libc", @@ -4211,7 +4212,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf592ae6a864437e98ef9c6ae7936b822077e9d038a3a48ee081ab92313afad4" dependencies = [ "num-bigint", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "swc_atoms", "swc_common", "swc_ecma_ast", @@ -5552,7 +5553,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "triomphe", ] @@ -5660,9 +5661,9 @@ dependencies = [ [[package]] name = "humantime" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" [[package]] name = "hyper" @@ -6085,9 +6086,9 @@ dependencies = [ [[package]] name = "io-uring" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" +checksum = "9080b15e63775b9a2ac7dca720f7050a8b955e092ea0f6020a4a80f69998cdc0" dependencies = [ "bitflags 2.13.0", "cfg-if", @@ -6626,14 +6627,14 @@ dependencies = [ [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" dependencies = [ "bitflags 2.13.0", "libc", "plain", - "redox_syscall 0.8.1", + "redox_syscall 0.9.0", ] [[package]] @@ -7245,7 +7246,7 @@ dependencies = [ "native-tls", "pem 3.0.6", "percent-encoding", - "rand 0.10.1", + "rand 0.10.2", "serde", "socket2 0.6.4", "thiserror 2.0.18", @@ -7562,9 +7563,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6" dependencies = [ "num-integer", "num-traits", @@ -8983,7 +8984,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls 0.23.35", "socket2 0.6.4", "thiserror 2.0.18", @@ -9004,7 +9005,7 @@ dependencies = [ "lru-slab", "rand 0.9.0", "ring 0.17.14", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "rustls 0.23.35", "rustls-pki-types", "slab", @@ -9092,9 +9093,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", @@ -9308,9 +9309,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b44b894f2a6e36457d665d1e08c3866add6ed5e70050c1b4ba8a8ddedb02ce7" +checksum = "c5102a6aaa05aa011a238e178e6bca86d2cb56fc9f586d37cb80f5bca6e07759" dependencies = [ "bitflags 2.13.0", ] @@ -9558,7 +9559,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47" dependencies = [ - "rand 0.10.1", + "rand 0.10.2", ] [[package]] @@ -9842,9 +9843,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustc_version" @@ -9998,9 +9999,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -10888,7 +10889,7 @@ dependencies = [ "data-encoding", "debugid", "if_chain", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "unicode-id-start", @@ -11321,7 +11322,7 @@ dependencies = [ "allocator-api2", "bumpalo", "hashbrown 0.14.5", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", ] [[package]] @@ -11350,7 +11351,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "siphasher 0.3.11", "swc_atoms", @@ -11399,7 +11400,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "string_enum", "swc_atoms", @@ -11420,7 +11421,7 @@ dependencies = [ "num-bigint", "once_cell", "regex", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "ryu-js", "serde", "swc_allocator", @@ -11454,7 +11455,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "seq-macro", "serde", "smallvec", @@ -11474,7 +11475,7 @@ checksum = "c675d14700c92f12585049b22b02356f1e142f4b0c32a4d0eb4b7a968a4c0c1e" dependencies = [ "anyhow", "pathdiff", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11508,7 +11509,7 @@ dependencies = [ "once_cell", "par-core", "phf 0.11.3", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11551,7 +11552,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39b3b34f6a28348416174912009d09994ab71c867682ec78d641a9feb3a96b4e" dependencies = [ "either", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11572,7 +11573,7 @@ dependencies = [ "bytes-str", "indexmap 2.14.0", "once_cell", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "sha1", "string_enum", @@ -11593,7 +11594,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3872c006ccfdcc19f1cf5c01c15915a69964ba7982c9f581cdb7e727e77b9a2c" dependencies = [ "bytes-str", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "swc_atoms", "swc_common", @@ -11614,7 +11615,7 @@ dependencies = [ "num_cpus", "once_cell", "par-core", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "ryu-js", "swc_atoms", "swc_common", @@ -11672,7 +11673,7 @@ dependencies = [ "data-encoding", "debugid", "if_chain", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "unicode-id-start", @@ -11869,7 +11870,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash 2.1.2", + "rustc-hash 2.1.3", "serde", "serde_json", "sketches-ddsketch", @@ -12188,9 +12189,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -12208,9 +12209,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -12922,9 +12923,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd69c5aa8f924c7519d6372789a74eac5b94fb0f8fcf0d4a97eb0bfc3e785f39" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" dependencies = [ "serde", "stable_deref_trait", @@ -13734,7 +13735,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-nats", @@ -13816,7 +13817,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.742.0" +version = "1.746.0" dependencies = [ "async-stream", "async-trait", @@ -13849,7 +13850,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13862,7 +13863,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "argon2", @@ -14000,7 +14001,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14023,7 +14024,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14038,7 +14039,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14064,7 +14065,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.742.0" +version = "1.746.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14074,7 +14075,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14091,7 +14092,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14113,7 +14114,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14136,7 +14137,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14152,7 +14153,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14173,7 +14174,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14194,7 +14195,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14209,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-nats", @@ -14243,7 +14244,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14268,7 +14269,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14286,7 +14287,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14308,7 +14309,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14328,7 +14329,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14365,7 +14366,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14393,7 +14394,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.742.0" +version = "1.746.0" dependencies = [ "lazy_static", "serde", @@ -14405,7 +14406,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.742.0" +version = "1.746.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14430,7 +14431,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14444,7 +14445,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.742.0" +version = "1.746.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14477,7 +14478,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.742.0" +version = "1.746.0" dependencies = [ "chrono", "lazy_static", @@ -14491,7 +14492,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14510,7 +14511,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.746.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14612,7 +14613,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.742.0" +version = "1.746.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14631,7 +14632,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.742.0" +version = "1.746.0" dependencies = [ "regex", "serde", @@ -14646,7 +14647,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14670,7 +14671,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "futures", @@ -14687,7 +14688,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.746.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14703,7 +14704,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -14724,7 +14725,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -14755,7 +14756,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "arc-swap", @@ -14780,7 +14781,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-stream", @@ -14814,7 +14815,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "futures", @@ -14832,7 +14833,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.746.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14841,7 +14842,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -14853,7 +14854,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -14865,7 +14866,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "gosyn", @@ -14877,7 +14878,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -14889,7 +14890,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -14901,7 +14902,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "nu-parser", @@ -14912,7 +14913,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14923,7 +14924,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14935,7 +14936,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14946,7 +14947,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-recursion", @@ -14968,7 +14969,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -14980,7 +14981,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -14994,7 +14995,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15011,7 +15012,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -15024,7 +15025,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde", @@ -15036,7 +15037,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -15054,7 +15055,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15070,7 +15071,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15086,7 +15087,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde", @@ -15097,7 +15098,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-recursion", @@ -15136,7 +15137,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "const_format", @@ -15175,7 +15176,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.742.0" +version = "1.746.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15186,7 +15187,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-recursion", @@ -15220,7 +15221,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15244,7 +15245,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15277,7 +15278,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15310,7 +15311,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15330,7 +15331,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15364,7 +15365,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15400,7 +15401,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15423,7 +15424,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15448,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-nats", @@ -15471,7 +15472,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15506,7 +15507,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15534,7 +15535,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-trait", @@ -15559,7 +15560,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15578,7 +15579,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-once-cell", @@ -15688,7 +15689,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.742.0" +version = "1.746.0" dependencies = [ "bytes", "futures", @@ -16506,9 +16507,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977347db8caa080403f6b6b7c1cda9479a8e869316f7e13a59b19076a40f94e3" +checksum = "5431d5661c32445236631278f27946e444ddafe4684cac70b185272d4f9c52d5" [[package]] name = "zmij" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 45682215cf..276416f450 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.742.0" +version = "1.746.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.742.0" +version = "1.746.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -112,7 +112,7 @@ strip = "none" default = [] private = ["windmill-api/private", "windmill-api-agent-workers?/private", "windmill-autoscaling/private", "windmill-common/private", "windmill-dep-map/private", "windmill-object-store/private", "windmill-git-sync/private", "windmill-indexer/private", "windmill-operator?/private", "windmill-queue/private", "windmill-worker/private", "windmill-test-utils/private"] agent_worker_server = ["windmill-api/agent_worker_server", "dep:windmill-api-agent-workers", "windmill-test-utils/agent_worker_server"] -enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise"] +enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-api-agent-workers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-object-store/enterprise", "license"] local_reports = ["windmill-common/local_reports"] enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"] stripe = ["windmill-api/stripe"] diff --git a/backend/migrations/20260624161218_dev_workspace.down.sql b/backend/migrations/20260624161218_dev_workspace.down.sql new file mode 100644 index 0000000000..45b8e65f67 --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.down.sql @@ -0,0 +1,3 @@ +ALTER TABLE workspace DROP CONSTRAINT IF EXISTS workspace_dev_requires_parent; +DROP INDEX IF EXISTS workspace_canonical_dev_idx; +ALTER TABLE workspace DROP COLUMN is_dev_workspace; diff --git a/backend/migrations/20260624161218_dev_workspace.up.sql b/backend/migrations/20260624161218_dev_workspace.up.sql new file mode 100644 index 0000000000..a0a252eb7c --- /dev/null +++ b/backend/migrations/20260624161218_dev_workspace.up.sql @@ -0,0 +1,14 @@ +-- A dev workspace is a fork (parent_workspace_id set) that is the standing editable +-- environment paired with its parent ("prod"), as opposed to a throwaway fork. +ALTER TABLE workspace ADD COLUMN is_dev_workspace BOOLEAN NOT NULL DEFAULT false; + +-- At most one active canonical dev workspace per parent (one editable source per prod). +-- Excludes soft-deleted (archived) workspaces so a new dev can replace an archived one. +CREATE UNIQUE INDEX workspace_canonical_dev_idx ON workspace (parent_workspace_id) + WHERE is_dev_workspace AND deleted = false; + +-- A dev workspace is a fork, so it must have a parent. Enforce the invariant at the schema level so +-- no path (or manual write) can persist a "root dev workspace". No backfill needed: the column is +-- added above with default false, so no existing row can violate this at creation time. +ALTER TABLE workspace ADD CONSTRAINT workspace_dev_requires_parent + CHECK (NOT is_dev_workspace OR parent_workspace_id IS NOT NULL); diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql new file mode 100644 index 0000000000..2278fbfb2e --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.down.sql @@ -0,0 +1,4 @@ +REVOKE ALL ON dispatch_event FROM windmill_user; +REVOKE ALL ON dispatch_event FROM windmill_admin; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_user; +REVOKE ALL ON SEQUENCE dispatch_event_id_seq FROM windmill_admin; diff --git a/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql new file mode 100644 index 0000000000..5d0b4650ce --- /dev/null +++ b/backend/migrations/20260701080313_grant_dispatch_event_to_windmill_roles.up.sql @@ -0,0 +1,15 @@ +-- The dispatch_event table (migration 20260523055641_dispatch_event) was +-- created relying on ALTER DEFAULT PRIVILEGES to grant access to windmill_user +-- and windmill_admin. Those default privileges only apply to objects created by +-- the role that set them (migration 20250205131523), so deployments whose +-- migration runner is a different role leave dispatch_event ungranted. Direct +-- writes run as the invoking role -- the dispatcher insert (asset_dispatch.rs) +-- and the DELETE in delete_jobs (windmill-common/src/jobs.rs), reached whenever +-- a job's side rows are reaped, e.g. on schedule disable -- and fail with +-- "permission denied for table dispatch_event". Grant explicitly to guarantee +-- access regardless of who ran the migrations (same fix as notify_event in +-- 20260619091631 and script_trigger in 20260619112847). +GRANT ALL ON dispatch_event TO windmill_user; +GRANT ALL ON dispatch_event TO windmill_admin; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_user; +GRANT ALL ON SEQUENCE dispatch_event_id_seq TO windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql new file mode 100644 index 0000000000..b73484e053 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.down.sql @@ -0,0 +1,6 @@ +REVOKE ALL ON workspace_diff FROM windmill_user; +REVOKE ALL ON workspace_diff FROM windmill_admin; +REVOKE ALL ON materialized_partition FROM windmill_user; +REVOKE ALL ON materialized_partition FROM windmill_admin; +REVOKE ALL ON debounce_stale_data FROM windmill_user; +REVOKE ALL ON debounce_stale_data FROM windmill_admin; diff --git a/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql new file mode 100644 index 0000000000..9205934600 --- /dev/null +++ b/backend/migrations/20260701083047_grant_user_db_tables_to_windmill_roles.up.sql @@ -0,0 +1,22 @@ +-- Same grant gap fixed for notify_event (20260619091631), script_trigger +-- (20260619112847), and dispatch_event: tables created after the one-time +-- GRANT ALL in 20250205131523 rely on ALTER DEFAULT PRIVILEGES, which only +-- applies to objects created by the role that set them. On deployments whose +-- migration runner is a different role, these tables end up ungranted, and +-- writes that run under the RLS role (a transaction opened via +-- user_db.begin(&authed) -> SET LOCAL ROLE windmill_user/windmill_admin) fail +-- with "permission denied for table ". +-- +-- Each table below has a confirmed write on a user_db transaction: +-- * workspace_diff -- UPDATE in set_ws_specific (workspaces.rs) +-- * materialized_partition -- INSERT via record_materialization (assets API) +-- * debounce_stale_data -- DELETE in resume_suspended_trigger_jobs +-- (global_handler.rs), the same tx that reaps a +-- job's side rows +-- None has a sequence, so only table grants are needed. +GRANT ALL ON workspace_diff TO windmill_user; +GRANT ALL ON workspace_diff TO windmill_admin; +GRANT ALL ON materialized_partition TO windmill_user; +GRANT ALL ON materialized_partition TO windmill_admin; +GRANT ALL ON debounce_stale_data TO windmill_user; +GRANT ALL ON debounce_stale_data TO windmill_admin; diff --git a/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql new file mode 100644 index 0000000000..2e3e434bcc --- /dev/null +++ b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.down.sql @@ -0,0 +1,3 @@ +-- Irreversible data backfill: once realigned to the derived username, a favorite +-- is indistinguishable from one legitimately created under that username, so there +-- is nothing safe to revert. No-op. diff --git a/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql new file mode 100644 index 0000000000..b06f63d7af --- /dev/null +++ b/backend/migrations/20260701123436_realign_superadmin_favorites_to_derived_username.up.sql @@ -0,0 +1,19 @@ +-- A superadmin acting in a workspace they are not a member of used to be +-- identified by their raw email (so favorites were stored with usr = email). +-- They are now identified by their instance-derived username (password.username), +-- so realign those pre-existing favorites to keep them visible. Only email-keyed +-- rows are ever a superadmin's (members always store a non-email username), and +-- the anti-join skips rows that would collide with an already-derived favorite. +UPDATE favorite f +SET usr = p.username +FROM password p +WHERE f.usr = p.email + AND p.super_admin = true + AND p.username IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM favorite f2 + WHERE f2.workspace_id = f.workspace_id + AND f2.usr = p.username + AND f2.favorite_kind = f.favorite_kind + AND f2.path = f.path + ); diff --git a/backend/migrations/20260702064737_workspace_storage_usage.down.sql b/backend/migrations/20260702064737_workspace_storage_usage.down.sql new file mode 100644 index 0000000000..e32480a602 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_storage_usage; diff --git a/backend/migrations/20260702064737_workspace_storage_usage.up.sql b/backend/migrations/20260702064737_workspace_storage_usage.up.sql new file mode 100644 index 0000000000..d7274e8742 --- /dev/null +++ b/backend/migrations/20260702064737_workspace_storage_usage.up.sql @@ -0,0 +1,17 @@ +-- Cached per-(workspace, storage) byte usage of workspace object storage, +-- refreshed by listing the storage location and adjusted optimistically as +-- uploads complete. Read on every workspace-storage write in CE builds to +-- enforce the storage quota, and by the storage_usage endpoint in all builds. +CREATE TABLE workspace_storage_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + storage VARCHAR(255) NOT NULL, + bytes BIGINT NOT NULL DEFAULT 0, + computed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, storage) +); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role +-- that set them (same gap as workspace_diff, notify_event, script_trigger). +GRANT ALL ON workspace_storage_usage TO windmill_user; +GRANT ALL ON workspace_storage_usage TO windmill_admin; diff --git a/backend/migrations/20260702095830_duckdb_macro_registry.down.sql b/backend/migrations/20260702095830_duckdb_macro_registry.down.sql new file mode 100644 index 0000000000..6f48e71d0f --- /dev/null +++ b/backend/migrations/20260702095830_duckdb_macro_registry.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS macro_usage; +DROP TABLE IF EXISTS macro_definition; diff --git a/backend/migrations/20260702095830_duckdb_macro_registry.up.sql b/backend/migrations/20260702095830_duckdb_macro_registry.up.sql new file mode 100644 index 0000000000..ccc4bcc8cc --- /dev/null +++ b/backend/migrations/20260702095830_duckdb_macro_registry.up.sql @@ -0,0 +1,31 @@ +-- Workspace DuckDB macro registry: one row per macro defined by a deployed +-- `// macros` library script. Names are workspace-unique (macros are injected +-- unqualified into consumer jobs). +CREATE TABLE macro_definition ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + name VARCHAR(255) NOT NULL, + provider_path VARCHAR(510) NOT NULL, + params TEXT NOT NULL, + body TEXT NOT NULL, + is_table_macro BOOLEAN NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, name) +); +CREATE INDEX idx_macro_definition_provider ON macro_definition (workspace_id, provider_path); + +-- Deploy-recorded consumer→macro edges, for the asset graph only (the worker +-- re-detects calls live at job time). +CREATE TABLE macro_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + consumer_path VARCHAR(510) NOT NULL, + macro_name VARCHAR(255) NOT NULL, + PRIMARY KEY (workspace_id, consumer_path, macro_name) +); +CREATE INDEX idx_macro_usage_name ON macro_usage (workspace_id, macro_name); + +-- Both tables are written on user_db transactions (SET LOCAL ROLE); the +-- one-time GRANT ALL migration predates them, so explicit grants are required. +GRANT ALL ON macro_definition TO windmill_user; +GRANT ALL ON macro_definition TO windmill_admin; +GRANT ALL ON macro_usage TO windmill_user; +GRANT ALL ON macro_usage TO windmill_admin; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql new file mode 100644 index 0000000000..e72edd0b12 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.down.sql @@ -0,0 +1 @@ +DROP TABLE workspace_multipart_inflight; diff --git a/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql new file mode 100644 index 0000000000..9768eaf842 --- /dev/null +++ b/backend/migrations/20260702213513_workspace_multipart_inflight.up.sql @@ -0,0 +1,32 @@ +-- Reservation for the parts of in-flight (initiated but not yet completed) +-- multipart uploads to workspace object storage. Uncommitted parts occupy +-- object-store capacity but are invisible to the list-based storage recount +-- until completion, so CE folds this reservation into the remaining quota to +-- bound abandoned uploads. One row per uploaded part so a re-uploaded part +-- (same part_id) replaces rather than double-counts; a part is recorded only +-- after its upstream upload succeeds. Rows are removed on successful complete +-- and lazily expired after a TTL (abort/abandon rely on the TTL, which matches +-- when the object store reaps the uncommitted parts). +-- part_id - S3 part number or Azure block id (string) +-- part_bytes - size of that part +-- target_existing_size - size of the object the upload will overwrite (0 if new), +-- credited so an overwrite only reserves the net growth +CREATE TABLE workspace_multipart_inflight ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace (id) ON DELETE CASCADE, + upload_id VARCHAR(512) NOT NULL, + part_id VARCHAR(256) NOT NULL, + storage VARCHAR(255) NOT NULL, + part_bytes BIGINT NOT NULL DEFAULT 0, + target_existing_size BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, upload_id, part_id) +); + +CREATE INDEX idx_workspace_multipart_inflight_created_at + ON workspace_multipart_inflight (created_at); + +-- Tables created after the one-time GRANT ALL in 20250205131523 need explicit +-- grants: ALTER DEFAULT PRIVILEGES only covers objects created by the role that +-- set them (same gap as workspace_storage_usage, notify_event, script_trigger). +GRANT ALL ON workspace_multipart_inflight TO windmill_user; +GRANT ALL ON workspace_multipart_inflight TO windmill_admin; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index e8a0a1cc40..2282025092 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -262,6 +262,14 @@ impl AssetsFinder { }; match arg_val { + // S3 helpers take an `S3Object` (constructor or dict literal) or an + // `s3://bucket/key` string. Other helpers take a bare resource-path + // string literal. + Some(arg) if matches!(kind, AssetKind::S3Object) => { + let path = s3_object_arg_path(arg).ok_or(())?; + self.assets + .push(ParseAssetsResult { kind, path, access_type, columns: None }); + } Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { let path = parse_asset_syntax(&value, false) .map(|(_, p)| p) @@ -282,6 +290,69 @@ impl AssetsFinder { // Positional arguments in python can also be used by their name struct Arg(usize, &'static str); +/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. +fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { + keywords + .iter() + .find(|kw| kw.arg.as_deref() == Some(name)) + .and_then(|kw| match &kw.value { + Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + _ => None, + }) +} + +/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. +fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { + dict.keys + .iter() + .zip(dict.values.iter()) + .find_map(|(key, value)| match (key.as_ref()?, value) { + ( + Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), + Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), + ) if k.as_str() == name => Some(v.clone()), + _ => None, + }) +} + +/// Resolve the SDK `S3Object` argument of `load_s3_file`/`load_s3_file_reader`/ +/// `write_s3_file` to a canonical asset path, mirroring `windmill-parser-ts-asset`: +/// `S3Object(s3="", storage=""?)` — or the equivalent dict literal — +/// maps to the URI `s3:///` (empty bucket for default storage, i.e. +/// `s3:///`), and a bare `"s3://bucket/key"` string is passed through. +/// The resulting URI is fed through `parse_asset_syntax` so the stored path +/// matches the TS object form and the `# on s3:///…` trigger form exactly. +fn s3_object_arg_path(expr: &Expr) -> Option { + let uri = match expr { + Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => s.clone(), + Expr::Call(call) => { + // `S3Object(...)` imported directly or as `wmill.S3Object(...)` + let func_name = call + .func + .as_name_expr() + .map(|n| n.id.as_str()) + .or_else(|| call.func.as_attribute_expr().map(|a| a.attr.as_str()))?; + if func_name != "S3Object" { + return None; + } + let key = keyword_str_value(&call.keywords, "s3")?; + let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + format!("s3://{storage}/{key}") + } + Expr::Dict(dict) => { + let key = dict_str_value(dict, "s3")?; + let storage = dict_str_value(dict, "storage").unwrap_or_default(); + format!("s3://{storage}/{key}") + } + _ => return None, + }; + Some( + parse_asset_syntax(&uri, false) + .map(|(_, p)| p.to_string()) + .unwrap_or(uri), + ) +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; @@ -307,6 +378,162 @@ def main(): ); } + #[test] + fn test_py_asset_parser_write_s3_object_constructor() { + // The SDK signature is `write_s3_file(s3object: S3Object | str, ...)` and + // its docstring recommends the constructor form with a bare key. It must + // resolve to the same canonical path as the TS object form and a + // `# on s3:///` trigger. + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.write_s3_file(S3Object(s3="analytics/x.csv"), b"content") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/analytics/x.csv".to_string(), + access_type: Some(W), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_with_storage() { + // `S3Object(s3=..., storage=...)` maps to `s3:///`, + // matching the `s3://bucket/key` string form and the TS object form. + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.load_s3_file(S3Object(s3="dir/in.csv", storage="mybucket")) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_reader_and_keyword_arg() { + // load_s3_file_reader + passing the S3Object via the `s3object` keyword + // and via the `wmill.S3Object` attribute form. + let input = r#" +import wmill +def main(): + wmill.load_s3_file_reader(s3object=wmill.S3Object(s3="dir/in.csv")) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + },]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_dict_literal() { + // `S3Object` subclasses dict, so the SDK also accepts a plain dict + // literal — same resolution as the constructor form. + let input = r#" +import wmill +def main(): + wmill.write_s3_file({"s3": "out.json"}, b"{}") + wmill.load_s3_file({"s3": "dir/in.csv", "storage": "mybucket"}) +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/out.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "mybucket/dir/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_multiple_s3_object_writes() { + // Several direct constructor-form writes in main() — all outputs must + // be detected (merge_assets returns a deterministic path-sorted order). + let input = r#" +import wmill +from wmill import S3Object +def main(): + wmill.write_s3_file(S3Object(s3="pipelines/km_real/raw_events.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/enriched.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/summary.json"), b"[]") + wmill.write_s3_file(S3Object(s3="pipelines/km_real/report.json"), b"{}") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/pipelines/km_real/enriched.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/pipelines/km_real/raw_events.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/pipelines/km_real/report.json".to_string(), + access_type: Some(W), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/pipelines/km_real/summary.json".to_string(), + access_type: Some(W), + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_s3_object_dynamic_key_no_false_positive() { + // A computed key can't be resolved statically — must yield nothing + // rather than a bogus path. + let input = r#" +import wmill +from wmill import S3Object +def main(name: str): + wmill.write_s3_file(S3Object(s3=f"pipelines/{name}.json"), b"{}") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!(s.map_err(|e| e.to_string()), Ok(vec![])); + } + #[test] fn test_py_asset_parser_unused_sql() { let input = r#" diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index f7c578a274..c4f93efdb7 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -83,18 +83,56 @@ pub fn parse_assets(input: &str) -> anyhow::Result { // Body-inferred column lineage, with `// column` annotations taking // precedence per output column (explicit declaration overrides inference). pipeline.column_lineage = merge_column_lineage(inferred, pipeline.column_lineage); - Ok(ParseAssetsOutput::new( - merge_assets(collector.assets), - Vec::new(), - pipeline, - )) + // A bare string literal in query position is only weak read evidence: a + // summary `SELECT 's3:///out.csv' AS target` after `COPY … TO + // 's3:///out.csv'` must not turn the write into rw (which draws a + // spurious read edge and an asset⇄script cycle in the pipeline graph). + // Surface weak reads only for assets with no other recorded usage, so a + // path that is *merely* mentioned still shows up linked to the script. + let mut assets = merge_assets(collector.assets); + for weak in merge_assets(collector.weak_string_reads) { + if !assets + .iter() + .any(|a| a.kind == weak.kind && a.path == weak.path) + { + assets.push(weak); + } + } + assets.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(ParseAssetsOutput::new(assets, Vec::new(), pipeline)) +} + +/// Provenance of the innermost access context. The access type alone can't +/// tell a definitive read apart from a mere mention: a bare string literal in +/// generic query position (`QueryRead`) is only *weak* read evidence — e.g. a +/// summary `SELECT 's3:///out.csv' AS target` echoing a path — while the same +/// literal as a read-function argument or a `COPY` target is definitive. +#[derive(Clone, Copy, PartialEq, Eq)] +enum AccessCtx { + QueryRead, + ReadFn, + CopyWrite, +} + +impl AccessCtx { + fn access_type(self) -> AssetUsageAccessType { + match self { + AccessCtx::QueryRead | AccessCtx::ReadFn => R, + AccessCtx::CopyWrite => W, + } + } } /// Visitor that collects S3 asset literals from SQL statements struct AssetCollector { assets: Vec, - // e.g set to Read when we are inside a SELECT ... FROM ... statement - current_access_type_stack: Vec, + // Bare string literals seen in generic query position — weak read + // evidence, surfaced by `parse_assets` only when the script has no other + // recorded usage of the same asset (a real write must not gain a spurious + // read edge from a mention). + weak_string_reads: Vec, + // e.g set to QueryRead when we are inside a SELECT ... FROM ... statement + current_access_type_stack: Vec, // e.g ATTACH 'ducklake://a' AS dl; => { "dl": (Ducklake, "a") } var_identifiers: BTreeMap, // e.g USE dl; @@ -119,6 +157,7 @@ impl AssetCollector { fn new() -> Self { Self { assets: Vec::new(), + weak_string_reads: Vec::new(), current_access_type_stack: Vec::with_capacity(8), var_identifiers: BTreeMap::new(), currently_used_asset: None, @@ -162,7 +201,11 @@ impl AssetCollector { name: &ObjectName, access_type: Option, ) -> Option { - let access_type = access_type.or_else(|| self.current_access_type_stack.last().copied()); + let access_type = access_type.or_else(|| { + self.current_access_type_stack + .last() + .map(|c| c.access_type()) + }); if let Some((kind, path)) = &self.currently_used_asset { // We don't want to infer that any simple identifier refers to an asset if // we are not in a known R/W context @@ -301,12 +344,18 @@ impl AssetCollector { // Check if the string matches our asset syntax patterns if let Some((kind, path)) = parse_asset_syntax(s, false) { if kind == AssetKind::S3Object { - self.assets.push(ParseAssetsResult { + let ctx = self.current_access_type_stack.last().copied(); + let result = ParseAssetsResult { kind, path: path.to_string(), - access_type: self.current_access_type_stack.last().copied(), + access_type: ctx.map(AccessCtx::access_type), columns: None, - }); + }; + if ctx == Some(AccessCtx::QueryRead) { + self.weak_string_reads.push(result); + } else { + self.assets.push(result); + } } } } @@ -314,7 +363,7 @@ impl AssetCollector { fn handle_obj_name_pre(&mut self, name: &ObjectName) { if let Some(fname) = get_trivial_obj_name(name) { if is_read_fn(fname) { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::ReadFn); } } if let Some(str_lit) = get_str_lit_from_obj_name(name) { @@ -691,9 +740,20 @@ impl Visitor for AssetCollector { match table_factor { TableFactor::Table { name, args, .. } => { if args.is_none() { - // Avoid Table Functions - self.handle_obj_name_pre(name); + // FROM 's3:///…' is a definitive read — record it directly + // so it isn't demoted to a weak in-query mention. + if let Some(asset) = self.get_s3_asset_from_str_literal_table(table_factor) { + self.assets.push(asset); + } } + // For a read-function table factor this pushes ReadFn, making + // every literal inside its arguments a definitive read — the + // direct form (read_csv('s3:///…')) but also list and named + // arguments (read_parquet(['s3:///…'])). Must run for BOTH the + // plain-table and table-function branches: post_visit_table_factor + // pops via handle_obj_name_post unconditionally, so skipping the + // push here would unbalance the stack. + self.handle_obj_name_pre(name); } _ => {} } @@ -719,6 +779,13 @@ impl Visitor for AssetCollector { Expr::Value(ValueWithSpan { value: Value::DoubleQuotedString(s), .. }) => { self.handle_string_literal(s); } + // Read-function call in expression position: its argument literals + // are definitive reads. Balances the pop in `post_visit_expr`. + Expr::Function(func) => { + if get_trivial_obj_name(&func.name).is_some_and(is_read_fn) { + self.current_access_type_stack.push(AccessCtx::ReadFn); + } + } _ => {} } std::ops::ControlFlow::Continue(()) @@ -946,7 +1013,7 @@ impl Visitor for AssetCollector { } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { - self.current_access_type_stack.push(W); + self.current_access_type_stack.push(AccessCtx::CopyWrite); self.handle_string_literal(filename); self.current_access_type_stack.pop(); } @@ -1013,7 +1080,7 @@ impl Visitor for AssetCollector { &mut self, query: &sqlparser::ast::Query, ) -> std::ops::ControlFlow { - self.current_access_type_stack.push(R); + self.current_access_type_stack.push(AccessCtx::QueryRead); self.cte_name_stack.push(collect_cte_names(query)); std::ops::ControlFlow::Continue(()) } @@ -1098,6 +1165,127 @@ mod tests { ); } + #[test] + fn test_copy_target_echoed_in_select_stays_write_only() { + // The trailing summary SELECT merely mentions the COPY target — it + // must not add a read (rw would draw an asset⇄script cycle). + let input = r#" + COPY (SELECT 1 AS x) TO 's3:///out.csv'; + SELECT 's3:///out.csv' AS target, 42 AS rows_written; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/out.csv".to_string(), + access_type: Some(W), + columns: None + }]) + ); + } + + #[test] + fn test_bare_string_mention_without_other_usage_is_a_read() { + let input = r#" + SELECT 's3:///referenced.csv' AS path; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/referenced.csv".to_string(), + access_type: Some(R), + columns: None + }]) + ); + } + + #[test] + fn test_self_refresh_read_fn_plus_copy_stays_rw() { + // A definitive read (read_csv) of the same file the script rewrites + // is a real rw — only *bare-literal* mentions are demoted. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_csv('s3:///data.csv'); + COPY (SELECT * FROM tmp) TO 's3:///data.csv'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.csv".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_plus_copy_stays_rw() { + // read_parquet's list form is as definitive as the direct literal — + // it must not be demoted to a weak mention when the file is rewritten. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM read_parquet(['s3:///data.parquet']); + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + + #[test] + fn test_read_fn_list_arg_multiple_files_are_reads() { + let input = r#" + SELECT * FROM read_parquet(['s3:///a.parquet', 's3:///b.parquet']); + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/a.parquet".to_string(), + access_type: Some(R), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/b.parquet".to_string(), + access_type: Some(R), + columns: None + } + ]) + ); + } + + #[test] + fn test_from_string_literal_of_written_file_stays_rw() { + // FROM-position string literal is likewise a definitive read. + let input = r#" + CREATE TABLE tmp AS SELECT * FROM 's3:///data.parquet'; + COPY (SELECT * FROM tmp) TO 's3:///data.parquet'; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/data.parquet".to_string(), + access_type: Some(RW), + columns: None + }]) + ); + } + #[test] fn test_sql_asset_parser_attach_no_usage_is_registered_as_unknown() { let input = r#" diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 931f470d3c..dc094a0a7b 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -826,6 +826,29 @@ fn parse_duckdb_file(code: &str) -> anyhow::Result>> { } args.append(&mut parse_sql_sanitized_interpolation(code)); + + // A `// partitioned` script receives its resolved partition as a job arg + // named `partition` (windmill_common::partition::PARTITION_ARG), and duckdb + // binds named parameters only when they appear in the parsed signature — + // so auto-declare it (as `-- $partition (text)` would) to make `$partition` + // usable without a manual declaration. An explicit declaration wins. + // `has_default` keeps the field optional: the platform resolves the value + // at run start when it is not passed explicitly. + if !args.iter().any(|arg| arg.name == "partition") + && windmill_parser::asset_parser::parse_pipeline_annotations(code) + .partition + .is_some() + { + args.push(Arg { + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + otyp: Some("text".to_string()), + has_default: true, + oidx: None, + otyp_inferred: false, + }); + } Ok(Some(args)) } @@ -1985,4 +2008,63 @@ SELECT x Ok(()) } + + #[test] + fn test_parse_duckdb_partitioned_auto_declares_partition() -> anyhow::Result<()> { + let code = r#"// partitioned daily +// materialize ducklake://main/sales_daily +SELECT $partition AS day, count(*) AS n FROM sales WHERE day = $partition +"#; + let args = parse_duckdb_sig(code)?.args; + assert_eq!( + args, + vec![Arg { + otyp: Some("text".to_string()), + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + has_default: true, + oidx: None, + otyp_inferred: false, + }] + ); + + // `--`-style annotation headers auto-declare too. + let dash_code = "-- partitioned hourly\nSELECT $partition AS h\n"; + assert_eq!(parse_duckdb_sig(dash_code)?.args, args); + + Ok(()) + } + + #[test] + fn test_parse_duckdb_partitioned_explicit_declaration_wins() -> anyhow::Result<()> { + let code = r#"// partitioned daily +-- $partition (text) +-- $limit (int) = 10 +SELECT * FROM sales WHERE day = $partition LIMIT $limit +"#; + let args = parse_duckdb_sig(code)?.args; + // No duplicate: the explicit (required) declaration is kept as-is. + assert_eq!(args.iter().filter(|a| a.name == "partition").count(), 1); + assert_eq!( + args[0], + Arg { + otyp: Some("text".to_string()), + name: "partition".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None, + otyp_inferred: false, + } + ); + Ok(()) + } + + #[test] + fn test_parse_duckdb_unpartitioned_does_not_declare_partition() -> anyhow::Result<()> { + let code = "SELECT 1 AS partition_count\n"; + assert_eq!(parse_duckdb_sig(code)?.args, vec![]); + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 096d93163b..d9f4dac33c 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6191,7 +6191,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.742.0" +version = "1.746.0" dependencies = [ "aho-corasick", "anyhow", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.742.0" +version = "1.746.0" dependencies = [ "proc-macro2", "quote", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.742.0" +version = "1.746.0" dependencies = [ "convert_case", "serde", @@ -6293,7 +6293,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -6305,7 +6305,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -6317,7 +6317,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "gosyn", @@ -6329,7 +6329,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -6353,7 +6353,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "nu-parser", @@ -6364,7 +6364,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6375,7 +6375,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6387,7 +6387,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "async-recursion", @@ -6420,7 +6420,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde_json", @@ -6432,7 +6432,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -6446,7 +6446,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "convert_case", @@ -6463,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde", @@ -6488,7 +6488,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6522,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6538,7 +6538,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6570,7 +6570,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "serde", @@ -6581,7 +6581,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.742.0" +version = "1.746.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index e2afd2483b..b4ea7a2aeb 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.742.0" +version = "1.746.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 5d9945cadd..f2a1a2b824 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -269,6 +269,12 @@ pub struct DelegateToGitRepoDetails { pub commit: Option, pub inventories_location: Option, pub vars_location: Option, + /// Path (relative to the cloned repo root) of an `ansible.cfg` to use as the + /// effective config for the run. When set, Windmill points `ANSIBLE_CONFIG` at + /// it so the repo's own settings (roles paths, inventory plugins, callbacks…) + /// apply, and only injects the settings that depend on runtime state it alone + /// controls (temp/home dirs, vault password) on top. + pub ansible_cfg: Option, #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub install_requirements: bool, } @@ -629,6 +635,10 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option Option, + // Bare `// macros` (must be alone on the line, like `// pipeline`) — + // marks this DuckDB script as a workspace *macro library*: its body is + // CREATE [OR REPLACE] MACRO statements (plus plain setup) registered at + // deploy and injected as TEMP macros into consumer jobs at run time. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub macros: bool, + // `// use ` — force-inject the whole named macro + // library (definitions + its setup statements) into this script's jobs, + // for dynamic SQL that call-detection can't see. Accumulating. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub use_libs: Vec, } #[derive(Serialize, Debug, PartialEq, Clone)] @@ -224,15 +235,21 @@ pub struct RetrySpec { pub delay: Option, } -// `// materialize [manual] [append] [key=]` — declares that this -// script produces a *managed* materialization of `` (a `ducklake://` -// table). By default the runtime generates the write DDL around the script's -// single trailing `SELECT` and owns idempotency, partition-state and snapshot -// capture. `manual` is the escape hatch: the script writes its own DDL and the -// runtime only records state (track-only). The reconciliation strategy options -// (`append`, `key=`) apply to managed mode: none → DELETE-by-partition + -// INSERT (replace); `key=` → MERGE (dedup within slice); `append` → +// `// materialize [manual] [append] [key=] [history] [track=]` +// — declares that this script produces a *managed* materialization of `` +// (a `ducklake://` table). By default the runtime generates the write DDL around +// the script's single trailing `SELECT` and owns idempotency, partition-state +// and snapshot capture. `manual` is the escape hatch: the script writes its own +// DDL and the runtime only records state (track-only). The reconciliation +// strategy options apply to managed mode: none → DELETE-by-partition + INSERT +// (replace); `key=` → MERGE (dedup within slice, SCD type 1); `append` → // INSERT-only. `append` wins if both are given (deploy-time warning). +// `key= history` upgrades the merge to SCD type 2: the SELECT is the current +// snapshot (one row per key), and a change to any tracked column (`track=`, +// default all non-key) closes the prior version and opens a new one, keeping full +// history (`valid_from`/`valid_to`/`is_current`). The leading keyword `scd2` is a +// recognized alias for `history`. `deletes=close` (scd2 only) also closes a key +// that disappears from the snapshot; default leaves absent keys current. #[derive(Serialize, Debug, PartialEq, Clone)] pub struct MaterializeSpec { pub target_kind: AssetKind, @@ -243,6 +260,20 @@ pub struct MaterializeSpec { pub append: bool, #[serde(skip_serializing_if = "Option::is_none", default)] pub unique_key: Option, + // `scd2` managed history mode: the SELECT is the current snapshot (one row + // per `unique_key`), and the runtime maintains a Slowly-Changing-Dimension + // type-2 history (`valid_from`/`valid_to`/`is_current`). `unique_key` (the + // `key=` opt) is the natural key; `track` lists the columns whose change + // opens a new version (empty ⇒ all non-key columns). Managed mode only. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub scd2: bool, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub track: Vec, + // scd2 only: `deletes=close` opts into hard-delete-close — a key that + // disappears from the snapshot has its current version closed (dbt's + // `hard_deletes=close`). Default (false) leaves absent keys current. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub close_deleted: bool, } // `// data_test …` — a data-quality assertion run against the @@ -346,6 +377,8 @@ pub struct PipelineAnnotations { pub materialize: Option, pub data_tests: Vec, pub column_lineage: Vec, + pub macros: bool, + pub use_libs: Vec, } impl ParseAssetsOutput { @@ -372,6 +405,8 @@ impl ParseAssetsOutput { materialize: pipeline.materialize, data_tests: pipeline.data_tests, column_lineage: pipeline.column_lineage, + macros: pipeline.macros, + use_libs: pipeline.use_libs, } } } @@ -662,6 +697,30 @@ pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { continue; } + if let Some(after_kw) = consume_keyword(rest, "macros") { + // Strict like `pipeline`: keyword alone on the line, so prose + // such as `// macros are defined below` never false-positives. + if after_kw.trim().is_empty() { + out.macros = true; + } + continue; + } + + // `// use ` — accumulating. The argument must be a + // single whitespace-free token containing `/` (all script paths do), + // so prose like `// use this script to …` is dropped fail-safe. + if let Some(after_kw) = consume_keyword(rest, "use") { + let path = after_kw.trim(); + if !path.is_empty() + && !path.contains(char::is_whitespace) + && path.contains('/') + && !out.use_libs.iter().any(|p| p == path) + { + out.use_libs.push(path.to_string()); + } + continue; + } + if let Some(after_kw) = consume_keyword(rest, "partitioned") { if out.partition.is_none() { if let Some(spec) = parse_partitioned_spec(after_kw.trim()) { @@ -800,19 +859,31 @@ fn parse_retry_spec(s: &str) -> Option { Some(RetrySpec { count, delay }) } -// Parse a `// materialize [manual] [append] [key=]` right-hand -// side. An optional leading `manual` token (whitespace-delimited) opts out of -// managed mode (track-only). The next whitespace token is the target asset URI +// Parse a `// materialize [manual] [append] [key=] [history] +// [track=]` right-hand side. An optional leading `manual` token opts out +// of managed mode (track-only); a leading `scd2` token is an alias for the +// `history` flag. The next whitespace token is the target asset URI // (default-syntax shorthands enabled, so `ducklake` → `ducklake://main`); the -// remainder are strategy options — bare `append` and `key=` (merge key), -// which apply to managed mode only. A missing/empty target yields `None` (the -// annotation is dropped, fail-safe). +// remainder are strategy options — bare `append`, bare `history` (SCD type-2 on +// a keyed merge), `key=` (merge/scd2 key), `track=` (scd2 tracked +// columns), and `deletes=close` (scd2 hard-delete-close) — which apply to managed +// mode only. A missing/empty target yields `None` (the annotation is dropped, +// fail-safe). fn parse_materialize_spec(s: &str) -> Option { - let (manual, rest) = match s.strip_prefix("manual") { - Some(after) if after.is_empty() || after.starts_with(char::is_whitespace) => { - (true, after.trim_start()) - } - _ => (false, s), + // One optional leading mode keyword: `manual` (escape hatch, track-only) or + // `scd2` (alias for the `history` flag below). A missing keyword is the + // default managed mode. + fn strip_mode<'a>(s: &'a str, kw: &str) -> Option<&'a str> { + s.strip_prefix(kw) + .filter(|after| after.is_empty() || after.starts_with(char::is_whitespace)) + .map(|after| after.trim_start()) + } + let (manual, scd2_kw, rest) = if let Some(after) = strip_mode(s, "manual") { + (true, false, after) + } else if let Some(after) = strip_mode(s, "scd2") { + (false, true, after) + } else { + (false, false, s) }; let mut it = rest.trim().splitn(2, char::is_whitespace); let asset_tok = it.next()?; @@ -822,11 +893,38 @@ fn parse_materialize_spec(s: &str) -> Option { return None; } let append = opts_str.split_whitespace().any(|t| t == "append"); - let unique_key = parse_kv_opts(opts_str) - .get("key") - .filter(|k| !k.is_empty()) - .cloned(); - Some(MaterializeSpec { target_kind, target_path: path.to_string(), manual, append, unique_key }) + // SCD type-2 history mode. The primary spelling is the bare `history` flag on + // a keyed merge (`key= history`) — it reads as "a keyed upsert that keeps + // history"; the leading `scd2` keyword is a recognized alias for the same. + let scd2 = scd2_kw || opts_str.split_whitespace().any(|t| t == "history"); + let opts = parse_kv_opts(opts_str); + let unique_key = opts.get("key").filter(|k| !k.is_empty()).cloned(); + // `track=` (scd2 only): comma-separated columns whose change opens a + // new version. Empty entries dropped; an empty list ⇒ track all non-key cols. + // Like every `=`-option here the value is whitespace-terminated, so the list + // must contain no spaces (`track=a,b`, not `track=a, b` — the rest is dropped). + let track = opts + .get("track") + .map(|v| { + v.split(',') + .map(|c| c.trim().to_string()) + .filter(|c| !c.is_empty()) + .collect::>() + }) + .unwrap_or_default(); + // `deletes=close` (scd2 only) opts into hard-delete-close; any other value + // (or absence) keeps the soft-delete default. + let close_deleted = opts.get("deletes").map(|v| v == "close").unwrap_or(false); + Some(MaterializeSpec { + target_kind, + target_path: path.to_string(), + manual, + append, + unique_key, + scd2, + track, + close_deleted, + }) } // Parse a `// data_test …` right-hand side into one `DataTest`. The @@ -1051,6 +1149,33 @@ mod pipeline_annotation_tests { assert!(!out.in_pipeline); } + #[test] + fn macros_marker_strict_like_pipeline() { + assert!(parse_pipeline_annotations("// macros\nCREATE MACRO m(a) AS a;").macros); + assert!(parse_pipeline_annotations("-- macros \nSELECT 1;").macros); + // Trailing prose / keyword variants disqualify the line. + assert!(!parse_pipeline_annotations("// macros are defined below\n").macros); + assert!(!parse_pipeline_annotations("// macros_v2\n").macros); + } + + #[test] + fn use_accumulates_dedups_and_rejects_prose() { + let out = parse_pipeline_annotations( + "// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\nSELECT 1;", + ); + assert_eq!(out.use_libs, vec!["f/lib/stats", "f/lib/dates"]); + + // Prose, slashless tokens, and multi-token lines are dropped fail-safe. + let out = parse_pipeline_annotations( + "// use this script to compute\n// use standalone\n// use f/lib/ok extra\n", + ); + assert!(out.use_libs.is_empty()); + + // Only the leading comment header is scanned. + let out = parse_pipeline_annotations("SELECT 1;\n-- use f/lib/late\n"); + assert!(out.use_libs.is_empty()); + } + #[test] fn on_schedule_marker() { // `// on schedule` is marker-only — the binding is the schedule row's @@ -1457,6 +1582,53 @@ mod pipeline_annotation_tests { assert_eq!(m.unique_key, None); } + #[test] + fn materialize_scd2_history_flag_with_key_and_track() { + // Primary spelling: `key= history` on a merge. + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history track=name,tier", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(!m.manual); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert_eq!(m.track, vec!["name".to_string(), "tier".to_string()]); + } + + #[test] + fn materialize_scd2_keyword_is_alias_for_history() { + let out = parse_pipeline_annotations("// materialize scd2 ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert_eq!(m.unique_key.as_deref(), Some("id")); + assert!(m.track.is_empty()); + // soft-delete default + assert!(!m.close_deleted); + } + + #[test] + fn materialize_scd2_deletes_close_opt() { + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=close", + ); + let m = out.materialize.expect("materialize"); + assert!(m.scd2); + assert!(m.close_deleted); + // any other value keeps the soft-delete default + let out = parse_pipeline_annotations( + "// materialize ducklake://a/dim key=id history deletes=ignore", + ); + assert!(!out.materialize.expect("materialize").close_deleted); + } + + #[test] + fn materialize_key_without_history_is_plain_merge() { + let out = parse_pipeline_annotations("// materialize ducklake://a/dim key=id"); + let m = out.materialize.expect("materialize"); + assert!(!m.scd2, "no history flag ⇒ SCD1 merge, not scd2"); + assert_eq!(m.unique_key.as_deref(), Some("id")); + } + #[test] fn materialize_default_syntax_shorthand() { let out = parse_pipeline_annotations("// materialize ducklake"); diff --git a/backend/parsers/windmill-parser/src/duckdb_builtins.rs b/backend/parsers/windmill-parser/src/duckdb_builtins.rs new file mode 100644 index 0000000000..d324fa4c30 --- /dev/null +++ b/backend/parsers/windmill-parser/src/duckdb_builtins.rs @@ -0,0 +1,936 @@ +// DuckDB built-in function names (scalar, aggregate, table, macro), used to +// reject workspace macros that would silently shadow a built-in (DuckDB +// allows the shadowing without error — verified on 1.5.4). Only +// identifier-shaped names are listed: workspace macro names are validated +// to `[a-z_][a-z0-9_]*` before this check, so operator names can't collide. +// +// Regenerate on DuckDB upgrades (sort in codepoint order — binary search): +// python3 -c "import duckdb,re; print('\\n'.join(sorted(set(r[0] for r in +// duckdb.connect().execute(\"SELECT DISTINCT lower(function_name) FROM +// duckdb_functions()\").fetchall() if re.fullmatch(r'[a-z_][a-z0-9_]*', r[0])))))" + +pub fn is_duckdb_builtin(name: &str) -> bool { + DUCKDB_BUILTIN_FUNCTIONS + .binary_search(&name.to_ascii_lowercase().as_str()) + .is_ok() +} + +const DUCKDB_BUILTIN_FUNCTIONS: &[&str] = &[ + "__internal_compress_integral_ubigint", + "__internal_compress_integral_uinteger", + "__internal_compress_integral_usmallint", + "__internal_compress_integral_utinyint", + "__internal_compress_string_hugeint", + "__internal_compress_string_ubigint", + "__internal_compress_string_uhugeint", + "__internal_compress_string_uinteger", + "__internal_compress_string_usmallint", + "__internal_compress_string_utinyint", + "__internal_decompress_integral_bigint", + "__internal_decompress_integral_hugeint", + "__internal_decompress_integral_integer", + "__internal_decompress_integral_smallint", + "__internal_decompress_integral_ubigint", + "__internal_decompress_integral_uhugeint", + "__internal_decompress_integral_uinteger", + "__internal_decompress_integral_usmallint", + "__internal_decompress_string", + "abs", + "acos", + "acosh", + "add", + "add_parquet_key", + "age", + "aggregate", + "ago", + "alias", + "all_profiling_output", + "any_value", + "apply", + "approx_count_distinct", + "approx_quantile", + "approx_top_k", + "arbitrary", + "arg_max", + "arg_max_null", + "arg_max_nulls_last", + "arg_min", + "arg_min_null", + "arg_min_nulls_last", + "argmax", + "argmin", + "array_agg", + "array_aggr", + "array_aggregate", + "array_append", + "array_apply", + "array_cat", + "array_concat", + "array_contains", + "array_cosine_distance", + "array_cosine_similarity", + "array_cross_product", + "array_distance", + "array_distinct", + "array_dot_product", + "array_extract", + "array_filter", + "array_grade_up", + "array_has", + "array_has_all", + "array_has_any", + "array_indexof", + "array_inner_product", + "array_intersect", + "array_length", + "array_negative_dot_product", + "array_negative_inner_product", + "array_pop_back", + "array_pop_front", + "array_position", + "array_prepend", + "array_push_back", + "array_push_front", + "array_reduce", + "array_resize", + "array_reverse", + "array_reverse_sort", + "array_select", + "array_slice", + "array_sort", + "array_to_json", + "array_to_string", + "array_to_string_comma_default", + "array_transform", + "array_unique", + "array_value", + "array_where", + "array_zip", + "arrow_scan", + "arrow_scan_dumb", + "ascii", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "avg", + "bar", + "base64", + "bin", + "bit_and", + "bit_count", + "bit_length", + "bit_or", + "bit_position", + "bit_xor", + "bitstring", + "bitstring_agg", + "bool_and", + "bool_or", + "can_cast_implicitly", + "cardinality", + "cast_to_type", + "cbrt", + "ceil", + "ceiling", + "century", + "char_length", + "character_length", + "checkpoint", + "chr", + "col_description", + "collations", + "combine", + "concat", + "concat_ws", + "constant_or_null", + "contains", + "copy_database", + "corr", + "cos", + "cosh", + "cot", + "count", + "count_if", + "count_star", + "countif", + "covar_pop", + "covar_samp", + "create_sort_key", + "cume_dist", + "current_catalog", + "current_connection_id", + "current_database", + "current_date", + "current_localtime", + "current_localtimestamp", + "current_query", + "current_query_id", + "current_role", + "current_schema", + "current_schemas", + "current_setting", + "current_transaction_id", + "current_user", + "currval", + "damerau_levenshtein", + "database_list", + "database_size", + "date_add", + "date_diff", + "date_part", + "date_sub", + "date_trunc", + "datediff", + "datepart", + "datesub", + "datetrunc", + "day", + "dayname", + "dayofmonth", + "dayofweek", + "dayofyear", + "days_in_month", + "decade", + "decode", + "degrees", + "dense_rank", + "disable_checkpoint_on_shutdown", + "disable_logging", + "disable_object_cache", + "disable_optimizer", + "disable_print_progress_bar", + "disable_profile", + "disable_profiling", + "disable_progress_bar", + "disable_verification", + "disable_verify_external", + "disable_verify_fetch_row", + "disable_verify_parallelism", + "disable_verify_serializer", + "divide", + "duckdb_approx_database_count", + "duckdb_columns", + "duckdb_connection_count", + "duckdb_constraints", + "duckdb_coordinate_systems", + "duckdb_databases", + "duckdb_dependencies", + "duckdb_extensions", + "duckdb_external_file_cache", + "duckdb_functions", + "duckdb_indexes", + "duckdb_keywords", + "duckdb_log_contexts", + "duckdb_logs", + "duckdb_logs_parsed", + "duckdb_memory", + "duckdb_optimizers", + "duckdb_prepared_statements", + "duckdb_profiling_settings", + "duckdb_schemas", + "duckdb_secret_types", + "duckdb_secrets", + "duckdb_sequences", + "duckdb_settings", + "duckdb_table_sample", + "duckdb_tables", + "duckdb_temporary_files", + "duckdb_types", + "duckdb_variables", + "duckdb_views", + "editdist3", + "element_at", + "enable_checkpoint_on_shutdown", + "enable_logging", + "enable_object_cache", + "enable_optimizer", + "enable_print_progress_bar", + "enable_profile", + "enable_profiling", + "enable_progress_bar", + "enable_verification", + "encode", + "ends_with", + "entropy", + "enum_code", + "enum_first", + "enum_last", + "enum_range", + "enum_range_boundary", + "epoch", + "epoch_ms", + "epoch_ns", + "epoch_us", + "equi_width_bins", + "era", + "error", + "even", + "exp", + "extension_versions", + "factorial", + "favg", + "fdiv", + "fill", + "filter", + "finalize", + "first", + "first_value", + "flatten", + "floor", + "fmod", + "force_checkpoint", + "format", + "format_bytes", + "format_pg_type", + "format_type", + "formatreadabledecimalsize", + "formatreadablesize", + "from_base64", + "from_binary", + "from_hex", + "from_json", + "from_json_strict", + "fsum", + "functions", + "gamma", + "gcd", + "gen_random_uuid", + "generate_series", + "generate_subscripts", + "geomean", + "geometric_mean", + "get_bit", + "get_block_size", + "get_current_time", + "get_current_timestamp", + "get_type", + "getvariable", + "glob", + "grade_up", + "greatest", + "greatest_common_divisor", + "group_concat", + "hamming", + "has_any_column_privilege", + "has_column_privilege", + "has_database_privilege", + "has_foreign_data_wrapper_privilege", + "has_function_privilege", + "has_language_privilege", + "has_schema_privilege", + "has_sequence_privilege", + "has_server_privilege", + "has_table_privilege", + "has_tablespace_privilege", + "hash", + "hex", + "histogram", + "histogram_exact", + "histogram_values", + "hour", + "icu_calendar_names", + "icu_collate_af", + "icu_collate_am", + "icu_collate_ar", + "icu_collate_ar_sa", + "icu_collate_as", + "icu_collate_az", + "icu_collate_be", + "icu_collate_bg", + "icu_collate_bn", + "icu_collate_bo", + "icu_collate_br", + "icu_collate_bs", + "icu_collate_ca", + "icu_collate_ceb", + "icu_collate_chr", + "icu_collate_cs", + "icu_collate_cy", + "icu_collate_da", + "icu_collate_de", + "icu_collate_de_at", + "icu_collate_dsb", + "icu_collate_dz", + "icu_collate_ee", + "icu_collate_el", + "icu_collate_en", + "icu_collate_en_us", + "icu_collate_eo", + "icu_collate_es", + "icu_collate_et", + "icu_collate_fa", + "icu_collate_fa_af", + "icu_collate_ff", + "icu_collate_fi", + "icu_collate_fil", + "icu_collate_fo", + "icu_collate_fr", + "icu_collate_fr_ca", + "icu_collate_fy", + "icu_collate_ga", + "icu_collate_gl", + "icu_collate_gu", + "icu_collate_ha", + "icu_collate_haw", + "icu_collate_he", + "icu_collate_he_il", + "icu_collate_hi", + "icu_collate_hr", + "icu_collate_hsb", + "icu_collate_hu", + "icu_collate_hy", + "icu_collate_id", + "icu_collate_id_id", + "icu_collate_ig", + "icu_collate_is", + "icu_collate_it", + "icu_collate_ja", + "icu_collate_ka", + "icu_collate_kk", + "icu_collate_kl", + "icu_collate_km", + "icu_collate_kn", + "icu_collate_ko", + "icu_collate_kok", + "icu_collate_ku", + "icu_collate_ky", + "icu_collate_lb", + "icu_collate_lij", + "icu_collate_lkt", + "icu_collate_ln", + "icu_collate_lo", + "icu_collate_lt", + "icu_collate_lv", + "icu_collate_mk", + "icu_collate_ml", + "icu_collate_mn", + "icu_collate_mr", + "icu_collate_ms", + "icu_collate_mt", + "icu_collate_my", + "icu_collate_nb", + "icu_collate_nb_no", + "icu_collate_ne", + "icu_collate_nl", + "icu_collate_nn", + "icu_collate_noaccent", + "icu_collate_nso", + "icu_collate_om", + "icu_collate_or", + "icu_collate_pa", + "icu_collate_pa_in", + "icu_collate_pl", + "icu_collate_ps", + "icu_collate_pt", + "icu_collate_ro", + "icu_collate_ru", + "icu_collate_sa", + "icu_collate_se", + "icu_collate_si", + "icu_collate_sk", + "icu_collate_sl", + "icu_collate_smn", + "icu_collate_sq", + "icu_collate_sr", + "icu_collate_sr_ba", + "icu_collate_sr_me", + "icu_collate_sr_rs", + "icu_collate_st", + "icu_collate_sv", + "icu_collate_sw", + "icu_collate_ta", + "icu_collate_te", + "icu_collate_th", + "icu_collate_tk", + "icu_collate_tn", + "icu_collate_to", + "icu_collate_tr", + "icu_collate_ug", + "icu_collate_uk", + "icu_collate_ur", + "icu_collate_uz", + "icu_collate_vi", + "icu_collate_wae", + "icu_collate_wo", + "icu_collate_xh", + "icu_collate_yi", + "icu_collate_yo", + "icu_collate_yue", + "icu_collate_yue_cn", + "icu_collate_zh", + "icu_collate_zh_cn", + "icu_collate_zh_hk", + "icu_collate_zh_mo", + "icu_collate_zh_sg", + "icu_collate_zh_tw", + "icu_collate_zu", + "icu_sort_key", + "ilike_escape", + "import_database", + "in_search_path", + "inet_client_addr", + "inet_client_port", + "inet_server_addr", + "inet_server_port", + "instr", + "is_histogram_other_bin", + "isfinite", + "isinf", + "isnan", + "isodow", + "isoyear", + "jaccard", + "jaro_similarity", + "jaro_winkler_similarity", + "json", + "json_array", + "json_array_length", + "json_contains", + "json_deserialize_sql", + "json_each", + "json_execute_serialized_sql", + "json_exists", + "json_extract", + "json_extract_path", + "json_extract_path_text", + "json_extract_string", + "json_group_array", + "json_group_object", + "json_group_structure", + "json_keys", + "json_merge_patch", + "json_object", + "json_pretty", + "json_quote", + "json_serialize_plan", + "json_serialize_sql", + "json_structure", + "json_transform", + "json_transform_strict", + "json_tree", + "json_type", + "json_valid", + "json_value", + "julian", + "kahan_sum", + "kurtosis", + "kurtosis_pop", + "lag", + "last", + "last_day", + "last_value", + "lcase", + "lcm", + "lead", + "least", + "least_common_multiple", + "left", + "left_grapheme", + "len", + "length", + "length_grapheme", + "levenshtein", + "lgamma", + "like_escape", + "list", + "list_aggr", + "list_aggregate", + "list_any_value", + "list_append", + "list_apply", + "list_approx_count_distinct", + "list_avg", + "list_bit_and", + "list_bit_or", + "list_bit_xor", + "list_bool_and", + "list_bool_or", + "list_cat", + "list_concat", + "list_contains", + "list_cosine_distance", + "list_cosine_similarity", + "list_count", + "list_distance", + "list_distinct", + "list_dot_product", + "list_element", + "list_entropy", + "list_extract", + "list_filter", + "list_first", + "list_grade_up", + "list_has", + "list_has_all", + "list_has_any", + "list_histogram", + "list_indexof", + "list_inner_product", + "list_intersect", + "list_kurtosis", + "list_kurtosis_pop", + "list_last", + "list_mad", + "list_max", + "list_median", + "list_min", + "list_mode", + "list_negative_dot_product", + "list_negative_inner_product", + "list_pack", + "list_position", + "list_prepend", + "list_product", + "list_reduce", + "list_resize", + "list_reverse", + "list_reverse_sort", + "list_select", + "list_sem", + "list_skewness", + "list_slice", + "list_sort", + "list_stddev_pop", + "list_stddev_samp", + "list_string_agg", + "list_sum", + "list_transform", + "list_unique", + "list_value", + "list_var_pop", + "list_var_samp", + "list_where", + "list_zip", + "listagg", + "ln", + "log", + "log10", + "log2", + "lower", + "lpad", + "ltrim", + "mad", + "make_date", + "make_time", + "make_timestamp", + "make_timestamp_ms", + "make_timestamp_ns", + "make_timestamptz", + "make_type", + "map", + "map_concat", + "map_contains", + "map_contains_entry", + "map_contains_value", + "map_entries", + "map_extract", + "map_extract_value", + "map_from_entries", + "map_keys", + "map_to_pg_oid", + "map_values", + "max", + "max_by", + "md5", + "md5_number", + "md5_number_lower", + "md5_number_upper", + "mean", + "median", + "metadata_info", + "microsecond", + "millennium", + "millisecond", + "min", + "min_by", + "minute", + "mismatches", + "mod", + "mode", + "month", + "monthname", + "multiply", + "nanosecond", + "nextafter", + "nextval", + "nfc_normalize", + "normalized_interval", + "not_ilike_escape", + "not_like_escape", + "now", + "nth_value", + "ntile", + "nullif", + "obj_description", + "octet_length", + "ord", + "pandas_scan", + "parquet_bloom_probe", + "parquet_file_metadata", + "parquet_full_metadata", + "parquet_kv_metadata", + "parquet_metadata", + "parquet_scan", + "parquet_schema", + "parse_dirname", + "parse_dirpath", + "parse_duckdb_log_message", + "parse_filename", + "parse_formatted_bytes", + "parse_path", + "percent_rank", + "pg_collation_is_visible", + "pg_conf_load_time", + "pg_conversion_is_visible", + "pg_function_is_visible", + "pg_get_constraintdef", + "pg_get_expr", + "pg_get_viewdef", + "pg_has_role", + "pg_is_other_temp_schema", + "pg_my_temp_schema", + "pg_opclass_is_visible", + "pg_operator_is_visible", + "pg_opfamily_is_visible", + "pg_postmaster_start_time", + "pg_size_pretty", + "pg_sleep", + "pg_table_is_visible", + "pg_timezone_names", + "pg_ts_config_is_visible", + "pg_ts_dict_is_visible", + "pg_ts_parser_is_visible", + "pg_ts_template_is_visible", + "pg_type_is_visible", + "pg_typeof", + "pi", + "platform", + "position", + "pow", + "power", + "pragma_collations", + "pragma_database_size", + "pragma_metadata_info", + "pragma_platform", + "pragma_show", + "pragma_storage_info", + "pragma_table_info", + "pragma_user_agent", + "pragma_version", + "prefix", + "printf", + "product", + "python_map_function", + "quantile", + "quantile_cont", + "quantile_disc", + "quarter", + "query", + "query_table", + "radians", + "random", + "range", + "rank", + "rank_dense", + "read_blob", + "read_csv", + "read_csv_auto", + "read_duckdb", + "read_json", + "read_json_auto", + "read_json_objects", + "read_json_objects_auto", + "read_ndjson", + "read_ndjson_auto", + "read_ndjson_objects", + "read_parquet", + "read_text", + "reduce", + "regexp_escape", + "regexp_extract", + "regexp_extract_all", + "regexp_full_match", + "regexp_matches", + "regexp_replace", + "regexp_split_to_array", + "regexp_split_to_table", + "regr_avgx", + "regr_avgy", + "regr_count", + "regr_intercept", + "regr_r2", + "regr_slope", + "regr_sxx", + "regr_sxy", + "regr_syy", + "remap_struct", + "repeat", + "repeat_row", + "replace", + "replace_type", + "reservoir_quantile", + "reverse", + "right", + "right_grapheme", + "round", + "round_even", + "roundbankers", + "row", + "row_number", + "row_to_json", + "rpad", + "rtrim", + "second", + "sem", + "seq_scan", + "session_user", + "set_bit", + "setseed", + "sha1", + "sha256", + "shobj_description", + "show", + "show_databases", + "show_tables", + "show_tables_expanded", + "sign", + "signbit", + "sin", + "sinh", + "skewness", + "sleep_ms", + "sniff_csv", + "split", + "split_part", + "sqrt", + "st_asbinary", + "st_astext", + "st_aswkb", + "st_aswkt", + "st_crs", + "st_geomfromwkb", + "st_intersects_extent", + "st_setcrs", + "starts_with", + "stats", + "stddev", + "stddev_pop", + "stddev_samp", + "storage_info", + "str_split", + "str_split_regex", + "strftime", + "string_agg", + "string_split", + "string_split_regex", + "string_to_array", + "strip_accents", + "strlen", + "strpos", + "strptime", + "struct_concat", + "struct_contains", + "struct_extract", + "struct_extract_at", + "struct_has", + "struct_indexof", + "struct_insert", + "struct_keys", + "struct_pack", + "struct_position", + "struct_update", + "struct_values", + "substr", + "substring", + "substring_grapheme", + "subtract", + "suffix", + "sum", + "sum_no_overflow", + "sumkahan", + "summary", + "switch", + "table_info", + "tan", + "tanh", + "test_all_types", + "test_vector_types", + "time_bucket", + "timetz_byte_comparable", + "timezone", + "timezone_hour", + "timezone_minute", + "to_base", + "to_base64", + "to_binary", + "to_centuries", + "to_days", + "to_decades", + "to_hex", + "to_hours", + "to_json", + "to_microseconds", + "to_millennia", + "to_milliseconds", + "to_minutes", + "to_months", + "to_quarters", + "to_seconds", + "to_timestamp", + "to_weeks", + "to_years", + "today", + "transaction_timestamp", + "translate", + "trim", + "trunc", + "truncate_duckdb_logs", + "try_strptime", + "txid_current", + "typeof", + "ucase", + "unbin", + "unhex", + "unicode", + "union_extract", + "union_tag", + "union_value", + "unnest", + "unpivot_list", + "upper", + "url_decode", + "url_encode", + "user", + "user_agent", + "uuid", + "uuid_extract_timestamp", + "uuid_extract_version", + "uuidv4", + "uuidv7", + "var_pop", + "var_samp", + "variance", + "variant_bytes_to_variant", + "variant_extract", + "variant_normalize", + "variant_to_parquet_variant", + "variant_typeof", + "vector_type", + "verify_external", + "verify_fetch_row", + "verify_parallelism", + "verify_serializer", + "version", + "wavg", + "week", + "weekday", + "weekofyear", + "weighted_avg", + "which_secret", + "write_log", + "xor", + "year", + "yearweek", +]; diff --git a/backend/parsers/windmill-parser/src/duckdb_macros.rs b/backend/parsers/windmill-parser/src/duckdb_macros.rs new file mode 100644 index 0000000000..20438d23d1 --- /dev/null +++ b/backend/parsers/windmill-parser/src/duckdb_macros.rs @@ -0,0 +1,591 @@ +//! Workspace DuckDB macro libraries (`// macros` annotation). +//! +//! A macro-library script's body is `CREATE [OR REPLACE] [TEMP] MACRO` +//! statements plus plain setup (ATTACH/INSTALL/LOAD/SET/PRAGMA). At deploy the +//! macros are parsed into the `macro_definition` registry; at job time the +//! worker injects the (transitively) called ones into consumer scripts as +//! `CREATE OR REPLACE TEMP MACRO` blocks. Everything is stored and re-emitted +//! **verbatim** (params, body) — no AST round-trip — so any expression DuckDB +//! accepts survives unchanged. +//! +//! DuckDB bind-checks macro bodies at CREATE time (macro→macro and table +//! references alike), so injected definitions must be emitted in dependency +//! order — hence `topo_order_macros` — and after the consumer's ATTACHes. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use crate::sql_materialize::{classify_block, split_statements, BlockClass}; + +#[derive(Debug, Clone, PartialEq)] +pub struct ParsedMacro { + /// Lowercased bare identifier (DuckDB identifiers are case-insensitive + /// unquoted; qualified / quoted names are rejected at parse). + pub name: String, + /// Verbatim text inside the parameter parens (may be empty). + pub params: String, + /// Verbatim text after `AS [TABLE]`, without the trailing `;`. + pub body: String, + pub is_table: bool, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum LibStatement { + Macro(ParsedMacro), + /// A non-macro statement allowed in a library: setup-class only + /// (ATTACH/INSTALL/LOAD/SET/PRAGMA/USE/CREATE TEMP …). Re-emitted verbatim + /// ahead of the macro definitions when the lib is injected via `// use`. + Setup(String), +} + +/// Statement text for injecting one macro into a consumer job. Always +/// TEMP (session-scoped — no catalog writes) and OR REPLACE (idempotent). +pub fn macro_create_statement(name: &str, params: &str, is_table: bool, body: &str) -> String { + format!( + "CREATE OR REPLACE TEMP MACRO {}({}) AS {}{};", + name, + params, + if is_table { "TABLE " } else { "" }, + body + ) +} + +fn is_ident(s: &str) -> bool { + let mut chars = s.chars(); + matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +// Case-insensitive whole-word prefix strip (whitespace-bounded), returning the +// remainder with leading whitespace trimmed. `get` (not slicing) so a +// multi-byte char straddling the boundary yields None instead of panicking. +fn strip_kw<'a>(s: &'a str, kw: &str) -> Option<&'a str> { + let prefix = s.get(..kw.len())?; + if prefix.eq_ignore_ascii_case(kw) { + let after = &s[kw.len()..]; + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + return Some(after.trim_start()); + } + } + None +} + +/// Parse one (comment-free, `;`-less) statement as a CREATE MACRO. Returns +/// `Ok(None)` when the statement is not macro-shaped at all (caller decides +/// whether it is acceptable setup), `Err` when it is macro-shaped but invalid. +fn parse_create_macro(stmt: &str) -> Result, String> { + let Some(mut rest) = strip_kw(stmt.trim(), "create") else { + return Ok(None); + }; + if let Some(r) = strip_kw(rest, "or") { + rest = strip_kw(r, "replace").ok_or("expected REPLACE after CREATE OR")?; + } + if let Some(r) = strip_kw(rest, "temp").or_else(|| strip_kw(rest, "temporary")) { + rest = r; + } + // `FUNCTION` is DuckDB's alias for `MACRO`. + let Some(rest) = strip_kw(rest, "macro").or_else(|| strip_kw(rest, "function")) else { + return Ok(None); + }; + + let name_end = rest + .find(|c: char| c.is_whitespace() || c == '(') + .unwrap_or(rest.len()); + let raw_name = &rest[..name_end]; + if raw_name.is_empty() { + return Err("CREATE MACRO: missing macro name".to_string()); + } + if raw_name.contains('.') || raw_name.contains('"') || !is_ident(raw_name) { + return Err(format!( + "macro name `{}` must be a plain unqualified identifier ([A-Za-z_][A-Za-z0-9_]*) in v1", + raw_name + )); + } + let name = raw_name.to_ascii_lowercase(); + + let rest = rest[name_end..].trim_start(); + if !rest.starts_with('(') { + return Err(format!( + "macro `{}`: expected a parenthesized parameter list after the name", + name + )); + } + // Balanced-paren scan for the verbatim param list. The statement comes + // from `split_statements` so comments are gone, but strings may contain + // parens (e.g. a default value) — skip quoted spans. + let bytes = rest.as_bytes(); + let mut depth = 0usize; + let mut i = 0usize; + let mut close = None; + while i < bytes.len() { + match bytes[i] { + b'(' => depth += 1, + b')' => { + depth -= 1; + if depth == 0 { + close = Some(i); + break; + } + } + q @ (b'\'' | b'"') => { + i += 1; + while i < bytes.len() && bytes[i] != q { + i += 1; + } + } + _ => {} + } + i += 1; + } + let Some(close) = close else { + return Err(format!( + "macro `{}`: unbalanced parameter parentheses", + name + )); + }; + let params = rest[1..close].trim().to_string(); + + let after_params = rest[close + 1..].trim_start(); + let Some(mut body) = strip_kw(after_params, "as") else { + return Err(format!( + "macro `{}`: expected AS after the parameter list", + name + )); + }; + let is_table = match strip_kw(body, "table") { + Some(r) => { + body = r; + true + } + None => false, + }; + let body = body.trim().trim_end_matches(';').trim_end().to_string(); + if body.is_empty() { + return Err(format!("macro `{}`: empty body", name)); + } + Ok(Some(ParsedMacro { name, params, body, is_table })) +} + +fn stmt_head(stmt: &str) -> String { + stmt.split_whitespace() + .take(4) + .collect::>() + .join(" ") +} + +/// Managed ATTACH forms (`ducklake://…`, `datatable://…`, resource URIs) are +/// rewritten by the worker's transform pass — which `// use`-injected lib +/// setup bypasses — so a library may only use plain, self-contained ATTACHes. +pub fn is_managed_attach(stmt: &str) -> bool { + let s = stmt.trim(); + if strip_kw(s, "attach").is_none() { + return false; + } + let lower = s.to_ascii_lowercase(); + [ + "'ducklake:", + "'datatable:", + "'windmill:", + "'$res:", + "\"ducklake:", + "\"datatable:", + "\"windmill:", + "\"$res:", + ] + .iter() + .any(|p| lower.contains(p)) +} + +/// Parse a `// macros` library body. Statements are either macro definitions +/// or setup; anything else is an error (user-facing message). +pub fn parse_macro_library(sql: &str) -> Result, String> { + let mut out = Vec::new(); + for stmt in split_statements(sql) { + match parse_create_macro(&stmt)? { + Some(m) => out.push(LibStatement::Macro(m)), + None => match classify_block(&stmt) { + BlockClass::Setup => { + if is_managed_attach(&stmt) { + return Err(format!( + "a `// macros` library cannot use managed ATTACH forms \ + (ducklake://, datatable://, resource URIs) in v1 — its setup is \ + injected verbatim into consumers, bypassing the ATTACH rewrite: `{}`", + stmt_head(&stmt) + )); + } + out.push(LibStatement::Setup(stmt)) + } + _ => { + return Err(format!( + "a `// macros` library may only contain CREATE [OR REPLACE] MACRO \ + statements plus setup (ATTACH/INSTALL/LOAD/SET/PRAGMA); found: `{}`", + stmt_head(&stmt) + )) + } + }, + } + } + Ok(out) +} + +/// Whether the statement is macro-definition-shaped (`CREATE [OR REPLACE] +/// [TEMP] MACRO|FUNCTION …`), regardless of whether the rest of the header +/// parses. Used by the worker's injection splice to keep injected blocks +/// *after* a script's own leading definitions (an injected body may only call +/// a local macro once the local CREATE has run — DuckDB binds at CREATE). +pub fn is_macro_definition(stmt: &str) -> bool { + let Some(mut rest) = strip_kw(stmt.trim(), "create") else { + return false; + }; + if let Some(r) = strip_kw(rest, "or") { + let Some(r2) = strip_kw(r, "replace") else { + return false; + }; + rest = r2; + } + if let Some(r) = strip_kw(rest, "temp").or_else(|| strip_kw(rest, "temporary")) { + rest = r; + } + strip_kw(rest, "macro") + .or_else(|| strip_kw(rest, "function")) + .is_some() +} + +/// Parse a single macro-definition statement (`CREATE [OR REPLACE] [TEMP] +/// MACRO …`). `None` when the statement isn't macro-shaped or its header is +/// malformed — callers using definitions as placement anchors skip those +/// (they fail at execution regardless). +pub fn parse_macro_definition(stmt: &str) -> Option { + parse_create_macro(stmt).ok().flatten() +} + +/// Names of macros the given statements define themselves (`CREATE [OR +/// REPLACE] [TEMP] MACRO …`). A consumer's own definition is authoritative +/// over a same-named workspace macro — the worker subtracts these before +/// planning injection, so deploying a library can never silently replace a +/// script's local macro. Malformed macro-shaped statements are skipped (they +/// fail at execution regardless). +pub fn locally_defined_macro_names(statements: &[String]) -> HashSet { + statements + .iter() + .filter_map(|s| parse_create_macro(s).ok().flatten().map(|m| m.name)) + .collect() +} + +/// Names from `names` that `sql` calls: an identifier token immediately +/// followed by `(` (after optional whitespace), not `.`-qualified. Scans the +/// comment-stripped statements; string-literal contents are skipped. Matching +/// is deliberately lexical — over-matching only injects an unused TEMP macro. +pub fn detect_macro_calls(sql: &str, names: &HashSet) -> HashSet { + let mut found = HashSet::new(); + if names.is_empty() { + return found; + } + for stmt in split_statements(sql) { + let bytes = stmt.as_bytes(); + let n = bytes.len(); + let mut i = 0usize; + let mut prev: Option = None; + while i < n { + let c = bytes[i]; + // skip quoted spans ('' escape for single quotes is handled by + // the fact that the reopened quote just starts another skip) + if c == b'\'' || c == b'"' { + let q = c; + i += 1; + while i < n && bytes[i] != q { + i += 1; + } + i += 1; + prev = Some(q); + continue; + } + let is_ident_start = c.is_ascii_alphabetic() || c == b'_'; + let prev_blocks = + matches!(prev, Some(p) if p == b'.' || p.is_ascii_alphanumeric() || p == b'_'); + if is_ident_start && !prev_blocks { + let start = i; + while i < n && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') { + i += 1; + } + let word = stmt[start..i].to_ascii_lowercase(); + let mut j = i; + while j < n && (bytes[j] as char).is_whitespace() { + j += 1; + } + if j < n && bytes[j] == b'(' && names.contains(&word) { + found.insert(word); + } + prev = Some(bytes[i - 1]); + continue; + } + prev = Some(c); + i += 1; + } + } + found +} + +/// Order `selected` macro names so every macro comes after the macros its +/// body calls (Kahn's algorithm, name-sorted ties for determinism). `defs` +/// maps each selected name to its body. Errors on a dependency cycle (only +/// reachable via cross-library deploy interleaving — DuckDB itself could +/// never have bound a cycle). +pub fn topo_order_macros( + selected: &HashSet, + defs: &BTreeMap, +) -> Result, String> { + let all: HashSet = selected.clone(); + // deps[m] = selected macros m's body calls; rev[d] = macros depending on d + let mut deps: BTreeMap> = BTreeMap::new(); + let mut rev: BTreeMap> = BTreeMap::new(); + for name in selected { + let body = defs + .get(name) + .ok_or_else(|| format!("macro `{}` has no definition", name))?; + let mut called = detect_macro_calls(body, &all); + called.remove(name); // ignore self-recursion (DuckDB rejects it at CREATE anyway) + for d in &called { + rev.entry(d.clone()).or_default().insert(name.clone()); + } + deps.insert(name.clone(), called.into_iter().collect()); + } + let mut ready: BTreeSet = deps + .iter() + .filter(|(_, d)| d.is_empty()) + .map(|(n, _)| n.clone()) + .collect(); + let mut out = Vec::with_capacity(selected.len()); + while let Some(name) = ready.iter().next().cloned() { + ready.remove(&name); + out.push(name.clone()); + if let Some(dependents) = rev.get(&name) { + for dep in dependents.clone() { + let d = deps.get_mut(&dep).unwrap(); + d.remove(&name); + if d.is_empty() { + ready.insert(dep); + } + } + } + deps.remove(&name); + } + if !deps.is_empty() { + let cycle: Vec = deps.keys().cloned().collect(); + return Err(format!( + "macro dependency cycle involving: {}", + cycle.join(", ") + )); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn names(list: &[&str]) -> HashSet { + list.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn parses_scalar_macro() { + let lib = + parse_macro_library("CREATE MACRO surrogate_key(a, b) AS md5(concat_ws('||', a, b));") + .unwrap(); + assert_eq!( + lib, + vec![LibStatement::Macro(ParsedMacro { + name: "surrogate_key".into(), + params: "a, b".into(), + body: "md5(concat_ws('||', a, b))".into(), + is_table: false, + })] + ); + } + + #[test] + fn parses_table_macro_or_replace_temp_and_function_alias() { + let lib = parse_macro_library( + "CREATE OR REPLACE TEMP MACRO top_n(t_max) AS TABLE SELECT * FROM t WHERE x <= t_max;\n\ + create function dbl(a) as a * 2;", + ) + .unwrap(); + match &lib[0] { + LibStatement::Macro(m) => { + assert_eq!(m.name, "top_n"); + assert!(m.is_table); + assert_eq!(m.body, "SELECT * FROM t WHERE x <= t_max"); + } + other => panic!("expected macro, got {:?}", other), + } + match &lib[1] { + LibStatement::Macro(m) => { + assert_eq!(m.name, "dbl"); + assert!(!m.is_table); + } + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn parses_default_params_and_nested_parens() { + let lib = parse_macro_library( + "CREATE MACRO safe_div(a, b, fallback := (0)) AS CASE WHEN b = 0 THEN fallback ELSE a / b END;", + ) + .unwrap(); + match &lib[0] { + LibStatement::Macro(m) => { + assert_eq!(m.params, "a, b, fallback := (0)"); + assert!(m.body.starts_with("CASE WHEN")); + } + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn params_with_string_containing_paren() { + let lib = parse_macro_library("CREATE MACRO f(sep := '(') AS concat(sep, 'x');").unwrap(); + match &lib[0] { + LibStatement::Macro(m) => assert_eq!(m.params, "sep := '('"), + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn setup_statements_allowed_and_kept_in_order() { + let lib = + parse_macro_library("-- a comment\nATTACH 'x.duckdb' AS ext;\nCREATE MACRO m() AS 1;") + .unwrap(); + assert_eq!(lib.len(), 2); + assert!(matches!(&lib[0], LibStatement::Setup(s) if s.starts_with("ATTACH"))); + assert!(matches!(&lib[1], LibStatement::Macro(_))); + } + + #[test] + fn rejects_non_setup_statements() { + let err = parse_macro_library("CREATE MACRO m() AS 1; SELECT 1;").unwrap_err(); + assert!(err.contains("may only contain"), "{err}"); + let err = parse_macro_library("CREATE TABLE t(x int);").unwrap_err(); + assert!(err.contains("may only contain"), "{err}"); + } + + #[test] + fn rejects_managed_attach_setup() { + let err = + parse_macro_library("ATTACH 'ducklake://analytics' AS lake;\nCREATE MACRO m() AS 1;") + .unwrap_err(); + assert!(err.contains("managed ATTACH"), "{err}"); + } + + #[test] + fn rejects_qualified_and_quoted_names() { + assert!(parse_macro_library("CREATE MACRO lake.m(a) AS a;") + .unwrap_err() + .contains("unqualified")); + assert!(parse_macro_library("CREATE MACRO \"weird name\"(a) AS a;").is_err()); + } + + #[test] + fn rejects_missing_params_or_body() { + assert!(parse_macro_library("CREATE MACRO m AS 1;").is_err()); + assert!(parse_macro_library("CREATE MACRO m(a);").is_err()); + assert!(parse_macro_library("CREATE MACRO m(a) AS ;").is_err()); + } + + #[test] + fn detect_basic_and_word_boundaries() { + let ns = names(&["dbl", "avg_x"]); + let found = detect_macro_calls("SELECT dbl(1), my_dbl(2), avg_x (3) FROM t", &ns); + assert!(found.contains("dbl")); + assert!(found.contains("avg_x")); // whitespace before paren ok + assert_eq!(found.len(), 2); + } + + #[test] + fn detect_skips_qualified_strings_and_comments() { + let ns = names(&["dbl"]); + assert!(detect_macro_calls("SELECT lake.dbl(1)", &ns).is_empty()); + assert!(detect_macro_calls("SELECT 'dbl(1)'", &ns).is_empty()); + assert!(detect_macro_calls("-- dbl(1)\nSELECT 1", &ns).is_empty()); + assert!(detect_macro_calls("SELECT dbl FROM t", &ns).is_empty()); // no call parens + } + + #[test] + fn detect_case_insensitive_and_table_macro_position() { + let ns = names(&["top_n"]); + assert!(!detect_macro_calls("SELECT * FROM TOP_N(3)", &ns).is_empty()); + } + + #[test] + fn topo_chain_and_diamond() { + let mut defs = BTreeMap::new(); + defs.insert("a".to_string(), "b(1) + c(2)".to_string()); + defs.insert("b".to_string(), "d(1)".to_string()); + defs.insert("c".to_string(), "d(2)".to_string()); + defs.insert("d".to_string(), "1".to_string()); + let sel = names(&["a", "b", "c", "d"]); + let order = topo_order_macros(&sel, &defs).unwrap(); + let pos = |n: &str| order.iter().position(|x| x == n).unwrap(); + assert!(pos("d") < pos("b")); + assert!(pos("d") < pos("c")); + assert!(pos("b") < pos("a")); + assert!(pos("c") < pos("a")); + } + + #[test] + fn topo_cycle_errors() { + let mut defs = BTreeMap::new(); + defs.insert("a".to_string(), "b(1)".to_string()); + defs.insert("b".to_string(), "a(1)".to_string()); + let err = topo_order_macros(&names(&["a", "b"]), &defs).unwrap_err(); + assert!(err.contains("cycle"), "{err}"); + } + + #[test] + fn non_ascii_body_survives_verbatim() { + // Regression: the statement splitter must not Latin-1-mojibake + // multi-byte text — macro bodies are persisted and re-executed. + let lib = parse_macro_library("CREATE MACRO greet(a) AS a || ' café ☕';").unwrap(); + match &lib[0] { + LibStatement::Macro(m) => assert_eq!(m.body, "a || ' café ☕'"), + other => panic!("expected macro, got {:?}", other), + } + } + + #[test] + fn non_ascii_garbage_errors_instead_of_panicking() { + // Regression: keyword matching must not byte-slice across a char + // boundary (panicked on inputs like this before). + assert!(parse_macro_library("abcé foo;").is_err()); + assert!(parse_macro_library("créate macro m(a) AS a;").is_err()); + } + + #[test] + fn locally_defined_names_extracted() { + let blocks = vec![ + "ATTACH 'x' AS a;".to_string(), + "CREATE TEMP MACRO dbl(a) AS a * 2;".to_string(), + "SELECT dbl(2);".to_string(), + ]; + let local = locally_defined_macro_names(&blocks); + assert!(local.contains("dbl")); + assert_eq!(local.len(), 1); + } + + #[test] + fn create_statement_roundtrip() { + assert_eq!( + macro_create_statement("m", "a, b := 1", true, "SELECT a + b"), + "CREATE OR REPLACE TEMP MACRO m(a, b := 1) AS TABLE SELECT a + b;" + ); + } + + #[test] + fn builtin_lookup() { + use crate::duckdb_builtins::is_duckdb_builtin; + assert!(is_duckdb_builtin("concat")); + assert!(is_duckdb_builtin("CONCAT")); + assert!(is_duckdb_builtin("read_csv")); + assert!(!is_duckdb_builtin("surrogate_key")); + } +} diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index 19bc5602cd..4270edcc79 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -13,6 +13,8 @@ use serde::Serialize; use serde_json::Value; pub mod asset_parser; +pub mod duckdb_builtins; +pub mod duckdb_macros; pub mod sql_materialize; /// S3 output format for SQL queries (moved here to avoid pulling sqlx into WASM via windmill-types) diff --git a/backend/parsers/windmill-parser/src/sql_materialize.rs b/backend/parsers/windmill-parser/src/sql_materialize.rs index 7a3c72bdce..e40cd0f057 100644 --- a/backend/parsers/windmill-parser/src/sql_materialize.rs +++ b/backend/parsers/windmill-parser/src/sql_materialize.rs @@ -87,28 +87,32 @@ impl WrapError { pub fn split_statements(sql: &str) -> Vec { let mut out = Vec::new(); let mut cur = String::new(); - let bytes = sql.as_bytes(); + // Char-wise, not byte-wise: the emitted strings are re-executed + // (materialize codegen) and persisted (macro registry), so a Latin-1 + // `bytes[i] as char` decode would corrupt any multi-byte text inside + // statements. All delimiters are ASCII — split positions are unaffected. + let chars: Vec = sql.chars().collect(); let mut i = 0; - let n = bytes.len(); + let n = chars.len(); while i < n { - let c = bytes[i] as char; + let c = chars[i]; // line comment — `--` (SQL) or `//`. The `//` form is not SQL, but it // is how Windmill pipeline annotations (`// materialize`, `// pipeline`, // …) are written, and they sit above the SQL in the same script; strip // them so they don't pollute the first statement block's classification // or the generated setup SQL. - if (c == '-' && i + 1 < n && bytes[i + 1] == b'-') - || (c == '/' && i + 1 < n && bytes[i + 1] == b'/') + if (c == '-' && i + 1 < n && chars[i + 1] == '-') + || (c == '/' && i + 1 < n && chars[i + 1] == '/') { - while i < n && bytes[i] != b'\n' { + while i < n && chars[i] != '\n' { i += 1; } continue; } // block comment - if c == '/' && i + 1 < n && bytes[i + 1] == b'*' { + if c == '/' && i + 1 < n && chars[i + 1] == '*' { i += 2; - while i + 1 < n && !(bytes[i] == b'*' && bytes[i + 1] == b'/') { + while i + 1 < n && !(chars[i] == '*' && chars[i + 1] == '/') { i += 1; } i += 2; @@ -119,10 +123,10 @@ pub fn split_statements(sql: &str) -> Vec { cur.push(c); i += 1; while i < n { - cur.push(bytes[i] as char); - if bytes[i] == b'\'' { + cur.push(chars[i]); + if chars[i] == '\'' { // doubled '' is an escaped quote, stay in string - if i + 1 < n && bytes[i + 1] == b'\'' { + if i + 1 < n && chars[i + 1] == '\'' { cur.push('\''); i += 2; continue; @@ -139,8 +143,8 @@ pub fn split_statements(sql: &str) -> Vec { cur.push(c); i += 1; while i < n { - cur.push(bytes[i] as char); - if bytes[i] == b'"' { + cur.push(chars[i]); + if chars[i] == '"' { i += 1; break; } @@ -327,20 +331,45 @@ fn snippet(stmt: &str) -> String { // --------------------------------------------------------------------------- /// How a (partition of a) materialized table is reconciled on each run. -/// Derived at deploy from `unique_key`/`append`: `append` → `Append`, else -/// `unique_key` → `Merge`, else `Replace`. +/// Derived at deploy from the annotation: `key= history` (or the `scd2` +/// alias) → `Scd2`, else `append` → `Append`, else `unique_key` → `Merge`, else +/// `Replace`. #[derive(Debug, Clone, PartialEq, Eq)] pub enum MaterializeStrategy { /// DELETE the current partition, then INSERT — partition becomes exactly /// what the SELECT returned. Full-refresh of the slice. Replace, /// Upsert within the slice on `unique_key` (delete-by-key + insert); rows - /// absent from the SELECT are left in place. + /// absent from the SELECT are left in place. This is SCD type 1: a changed + /// row overwrites the prior value, keeping no history. Merge { unique_key: String }, /// INSERT only — immutable event-log semantics. Append, + /// Slowly Changing Dimension type 2: the SELECT is the *current* snapshot + /// (one row per `key`); a change to any tracked column closes the prior + /// version (`valid_to`/`is_current=false`) and opens a new one, so the full + /// history is preserved. `track` empty ⇒ every non-key column is tracked. + /// `close_deleted` (opt-in `deletes=close`) also closes the current version + /// of a key that disappears from the snapshot (dbt's `hard_deletes=close`); + /// default (false) leaves absent keys current (soft delete). + /// Unpartitioned only (the worker rejects `// partitioned` + scd2). + Scd2 { key: String, track: Vec, close_deleted: bool }, } +/// SCD2 metadata columns appended to the managed history table. Fixed names so +/// the generated diff/close/open SQL and any `// data_test` on them agree. +const SCD2_VALID_FROM: &str = "valid_from"; +const SCD2_VALID_TO: &str = "valid_to"; +const SCD2_IS_CURRENT: &str = "is_current"; +/// Connection-local temp table holding the keys whose version must be rotated +/// this run (changed + new). Captured before the write so the close and the +/// open see the same set. `_wm_` prefix so it never collides with user tables. +const SCD2_CHANGED_KEYS: &str = "_wm_scd2_changed"; +/// Connection-local temp table holding the keys that disappeared from the +/// snapshot this run (present-and-current in the table, absent from the SELECT). +/// Only used when `close_deleted` (`deletes=close`) is set. +const SCD2_DELETED_KEYS: &str = "_wm_scd2_deleted"; + /// Inputs to materialization codegen, all resolved at run time by the worker. /// Pure: produces SQL text; executes nothing. #[derive(Debug, Clone)] @@ -377,6 +406,13 @@ impl<'a> MaterializeCodegen<'a> { let pval = self.partition_value_sql; let mut out = Vec::new(); + // SCD2 has a shape unlike the DELETE/INSERT strategies (diff → close old + // → open new) and does not support partitioning (rejected at the worker), + // so it is generated up front by its own helper. + if let MaterializeStrategy::Scd2 { key, track, close_deleted } = &self.strategy { + return self.scd2_statements(key, track, *close_deleted); + } + // Whole-table replace: rebuild the table to match the SELECT's *current* // schema each run with one atomic `CREATE OR REPLACE` (which DuckLake // still snapshots). This is the only path that survives a changed SELECT @@ -441,10 +477,160 @@ impl<'a> MaterializeCodegen<'a> { )); out.push(format!("INSERT INTO {t} {source};")); } + // Handled by the early return above (scd2 has no partitioned form). + MaterializeStrategy::Scd2 { .. } => unreachable!("scd2 handled before this match"), } out.push("COMMIT;".to_string()); out } + + /// SCD2 codegen: the incoming SELECT is the *current desired snapshot* (one + /// row per `key`); we diff it against the live current rows, close the prior + /// version of every changed/new key, and open a fresh one — so history is + /// kept. `track` empty ⇒ every non-key column is tracked for change + /// detection. + /// + /// Shape (all one transaction for the mutation, mirroring the other + /// strategies so a partial failure leaves the prior snapshot intact): + /// 1. bootstrap the table (business columns + `valid_from/valid_to/ + /// is_current`), idempotent; + /// 2. capture changed+new keys into a connection-local temp table *before* + /// the write — the close below flips `is_current`, so recomputing the + /// diff after it would see a different set; + /// 3. close the prior open version of those keys (`UPDATE` — not `MERGE`: + /// DuckLake's MERGE is the unreliable path, plain UPDATE works); + /// 3b. when `close_deleted`, also capture the keys that vanished from the + /// snapshot and close their current version (no reopen) — dbt's + /// `hard_deletes=close`; + /// 4. open a new current version from the snapshot; + /// 5. create the `_current` convenience view once (`IF NOT EXISTS`), + /// inside the same transaction so it doesn't advance the DuckLake snapshot + /// past the data write the summary records (and so an unchanged rerun, + /// whose UPDATE/INSERT touch no rows, stays a true no-op). + /// + /// Close/open match keys with `IS NOT DISTINCT FROM` (via a correlated + /// `EXISTS`), not `key IN (…)`: SQL `IN` never matches `NULL`, so a `NULL` + /// natural key would be flagged as changed yet silently skipped by both the + /// close and the open, dropping the row. Null-safe matching materializes it + /// instead (a `NULL` key is still ill-formed for a dimension — guard it with + /// `// data_test not_null ` — but it must not vanish). + /// + /// Without `close_deleted`, keys present in the table but absent from the + /// SELECT are left current (soft delete — dbt's `hard_deletes=ignore` default; + /// with `close_deleted` they are closed instead — see step 3b). The effective + /// timestamp is `now()`, which DuckDB fixes to + /// the transaction start, so `valid_from`/`valid_to` are consistent within a + /// run without a nondeterministic per-statement clock. + /// + /// Reserved columns: `valid_from`/`valid_to`/`is_current` are appended to the + /// user's SELECT with these fixed names (kept clean so consumers write + /// `WHERE is_current` / `ASOF JOIN … >= valid_from`). A SELECT that already + /// projects one of them is a v1 constraint violation — the bootstrap then + /// produces a duplicate output column and the run fails at execution + /// (documented; not statically checkable here since the SELECT's columns + /// aren't known at codegen time). + fn scd2_statements(&self, key: &str, track: &[String], close_deleted: bool) -> Vec { + let t = self.target_qualified; + let sel = self.select_sql; + let k = quote_ident(key); + let vf = SCD2_VALID_FROM; + let vt = SCD2_VALID_TO; + let ic = SCD2_IS_CURRENT; + let changed = SCD2_CHANGED_KEYS; + let deleted = SCD2_DELETED_KEYS; + // Transaction-stable effective timestamp (see doc above). Cast to plain + // TIMESTAMP so it matches the bootstrapped column type (now() is TZ-aware). + let ts = "CAST(now() AS TIMESTAMP)"; + + // Projection compared to detect change. Empty `track` ⇒ all business + // columns via `* EXCLUDE ()` on the table side (which carries + // the extra metadata columns) and `*` on the snapshot side. An explicit + // `track` ⇒ key + those columns on both sides. `EXCEPT` treats NULLs as + // equal, so an unchanged NULL is not read as a change. + let (src_proj, tgt_proj) = if track.is_empty() { + ( + format!("SELECT * FROM ({sel})"), + format!("SELECT * EXCLUDE ({vf}, {vt}, {ic}) FROM {t} WHERE {ic}"), + ) + } else { + let cols = std::iter::once(key) + .chain(track.iter().map(String::as_str)) + .map(quote_ident) + .collect::>() + .join(", "); + ( + format!("SELECT {cols} FROM ({sel})"), + format!("SELECT {cols} FROM {t} WHERE {ic}"), + ) + }; + + let mut out = vec![ + format!( + "CREATE TABLE IF NOT EXISTS {t} AS SELECT *, \ + CAST(NULL AS TIMESTAMP) AS {vf}, \ + CAST(NULL AS TIMESTAMP) AS {vt}, \ + CAST(NULL AS BOOLEAN) AS {ic} FROM ({sel}) WHERE false;" + ), + format!( + "CREATE OR REPLACE TEMP TABLE {changed} AS \ + SELECT {k} FROM ({src_proj} EXCEPT {tgt_proj});" + ), + ]; + // Hard-delete-close (`deletes=close`): the keys that vanished from the + // snapshot — present-and-current in the table, absent from the SELECT. + // Captured before the transaction (like `changed`) and disjoint from it (a + // key is either in the snapshot or not), so the two closes never overlap. + if close_deleted { + out.push(format!( + "CREATE OR REPLACE TEMP TABLE {deleted} AS \ + SELECT {k} FROM (SELECT {k} FROM {t} WHERE {ic} EXCEPT SELECT {k} FROM ({sel}));" + )); + } + out.push("BEGIN TRANSACTION;".to_string()); + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {changed} \ + WHERE {changed}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + // Close vanished keys — no matching INSERT below, so they close without + // reopening. A key that later reappears isn't in `WHERE is_current`, so the + // `changed` diff treats it as new and opens a fresh version (a validity gap + // between the delete and the reactivation — correct SCD2). + if close_deleted { + out.push(format!( + "UPDATE {t} SET {vt} = {ts}, {ic} = false \ + WHERE {ic} AND EXISTS (SELECT 1 FROM {deleted} \ + WHERE {deleted}.{k} IS NOT DISTINCT FROM {t}.{k});" + )); + } + out.push(format!( + "INSERT INTO {t} SELECT s.*, {ts} AS {vf}, CAST(NULL AS TIMESTAMP) AS {vt}, \ + true AS {ic} FROM ({sel}) s WHERE EXISTS (SELECT 1 FROM {changed} c \ + WHERE c.{k} IS NOT DISTINCT FROM s.{k});" + )); + out.push( + // Consumer convenience: a `_current` view (the live slice) so the + // common "just the latest version" read needs no `WHERE is_current`, + // and downstream scripts can `// on` / read it directly. For the + // effective-dated payoff, consumers `ASOF JOIN ON fact.key = + // dim. AND fact.ts >= dim.valid_from`. + // + // `CREATE VIEW IF NOT EXISTS` (not `OR REPLACE`), created inside the + // write transaction, on purpose: the view definition never changes + // (`SELECT * WHERE is_current` always reflects live data), and a + // catalog write advances the DuckLake snapshot — so `OR REPLACE` on + // every run would (a) advance the snapshot on an otherwise no-op + // unchanged run and (b) make the summary's `max(snapshot_id)` record + // the view DDL instead of the data write. `IF NOT EXISTS` creates it + // once (folded into the first data-write snapshot) and is a true no-op + // afterwards. The `_current` name is reserved: if a real table by + // that name already exists, `IF NOT EXISTS` skips silently (no view, + // no error) — documented as a reserved suffix. + format!("CREATE VIEW IF NOT EXISTS {t}_current AS SELECT * FROM {t} WHERE {ic};"), + ); + out.push("COMMIT;".to_string()); + out + } } /// The read that captures the DuckLake snapshot id produced by the write, for @@ -485,6 +671,7 @@ pub fn build_wrap_blocks( tests: &[DataTestResolved], ) -> Result, String> { let target_qualified = format!("{TARGET_ALIAS}.{target_table}"); + let scd2 = matches!(strategy, MaterializeStrategy::Scd2 { .. }); let cg = MaterializeCodegen { target_qualified: &target_qualified, select_sql: &plan.output, @@ -499,6 +686,7 @@ pub fn build_wrap_blocks( partition_col, partition_value_sql, partitioned, + scd2, }; let test_sql = build_data_test_checks(tests, &ctx)?; let mut blocks: Vec = Vec::new(); @@ -663,6 +851,11 @@ pub struct DataTestCtx<'a> { pub partition_value_sql: &'a str, /// Whether the target is partitioned (scopes probes to the slice). pub partitioned: bool, + /// Whether the target is an SCD2 history table. Built-in probes then assert + /// the *current snapshot* (`is_current` rows): the history legitimately + /// repeats the natural key across closed versions, so an unscoped + /// `unique()` would fail on the second change of any key. + pub scd2: bool, } /// A data test resolved enough to generate SQL. Built-ins carry only their @@ -723,17 +916,31 @@ fn quote_lit(s: &str) -> String { format!("'{}'", s.replace('\'', "''")) } -// The `WHERE`/`AND` fragment scoping a probe to the current partition, or -// empty when unpartitioned. `prefix` is `WHERE ` or `AND ` per call site. +// The `WHERE`/`AND` fragment scoping a probe to the current partition and/or +// the SCD2 current snapshot, or empty when neither applies. `prefix` is +// `WHERE ` or `AND ` per call site; further conditions chain with `AND`. +// (`partitioned` and `scd2` are mutually exclusive today — the combo is +// rejected at codegen — but the chaining keeps this correct if that changes.) fn partition_scope(ctx: &DataTestCtx, prefix: &str, table_alias: Option<&str>) -> String { - if !ctx.partitioned { + let qualify = |col: String| match table_alias { + Some(a) => format!("{a}.{col}"), + None => col, + }; + let mut conds: Vec = Vec::new(); + if ctx.partitioned { + conds.push(format!( + "{} = {}", + qualify(quote_ident(ctx.partition_col)), + ctx.partition_value_sql + )); + } + if ctx.scd2 { + conds.push(qualify(SCD2_IS_CURRENT.to_string())); + } + if conds.is_empty() { return String::new(); } - let col = match table_alias { - Some(a) => format!("{a}.{}", quote_ident(ctx.partition_col)), - None => quote_ident(ctx.partition_col), - }; - format!("{prefix}{col} = {}", ctx.partition_value_sql) + format!("{prefix}{}", conds.join(" AND ")) } // Record one check: its display `name` plus `count_query` (which yields a @@ -1111,6 +1318,131 @@ mod tests { ); } + #[test] + fn codegen_scd2_default_track_closes_old_opens_new() { + // Empty `track` ⇒ diff on all business columns via `* EXCLUDE (scd cols)`. + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim_scd2", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: false, + }, + }; + let st = cg.statements(); + // bootstrap adds the three SCD metadata columns + assert!(st[0].starts_with("CREATE TABLE IF NOT EXISTS _wm_target.dim_scd2 AS SELECT *,")); + assert!(st[0].contains("AS valid_from")); + assert!(st[0].contains("AS valid_to")); + assert!(st[0].contains("AS is_current")); + // changed-key set captured before the transaction, all cols compared + assert!( + st[1].contains("CREATE OR REPLACE TEMP TABLE _wm_scd2_changed AS SELECT \"id\" FROM") + ); + assert!(st[1].contains("SELECT * FROM (SELECT id, name FROM dl.src) EXCEPT")); + assert!(st[1].contains("SELECT * EXCLUDE (valid_from, valid_to, is_current) FROM _wm_target.dim_scd2 WHERE is_current")); + assert_eq!(st[2], "BEGIN TRANSACTION;"); + // close: UPDATE (not MERGE) the prior open version of changed keys, with + // null-safe key matching (IS NOT DISTINCT FROM, not IN — IN drops NULLs) + assert!(st[3].starts_with("UPDATE _wm_target.dim_scd2 SET valid_to = CAST(now() AS TIMESTAMP), is_current = false")); + assert!(st[3].contains( + "WHERE is_current AND EXISTS (SELECT 1 FROM _wm_scd2_changed \ + WHERE _wm_scd2_changed.\"id\" IS NOT DISTINCT FROM _wm_target.dim_scd2.\"id\");" + )); + // open: INSERT the new current version, null-safe key matching + assert!(st[4].starts_with( + "INSERT INTO _wm_target.dim_scd2 SELECT s.*, CAST(now() AS TIMESTAMP) AS valid_from" + )); + assert!(st[4].contains( + "true AS is_current FROM (SELECT id, name FROM dl.src) s WHERE EXISTS \ + (SELECT 1 FROM _wm_scd2_changed c WHERE c.\"id\" IS NOT DISTINCT FROM s.\"id\");" + )); + // consumer-convenience `_current` view: `IF NOT EXISTS` (created once, + // no-op on unchanged reruns) and INSIDE the txn (folded into the write snapshot) + assert_eq!( + st[5], + "CREATE VIEW IF NOT EXISTS _wm_target.dim_scd2_current AS SELECT * FROM _wm_target.dim_scd2 WHERE is_current;" + ); + assert_eq!(st[6], "COMMIT;"); + // no fragile constructs: no MERGE INTO, and no NULL-dropping `IN (SELECT` + assert!(!st.iter().any(|s| s.contains("MERGE INTO"))); + assert!(!st.iter().any(|s| s.contains("IN (SELECT"))); + // soft-delete default: no deleted-key set, no second close + assert!(!st.iter().any(|s| s.contains("_wm_scd2_deleted"))); + } + + #[test] + fn codegen_scd2_explicit_track_projects_key_and_tracked_cols() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name, addr FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec!["name".to_string()], + close_deleted: false, + }, + }; + let st = cg.statements(); + // only key + tracked cols are compared (addr changes don't rotate a version) + assert!(st[1].contains("SELECT \"id\", \"name\" FROM (SELECT id, name, addr FROM dl.src) EXCEPT SELECT \"id\", \"name\" FROM _wm_target.dim WHERE is_current")); + } + + #[test] + fn codegen_scd2_close_deleted_adds_deleted_set_and_second_close() { + let cg = MaterializeCodegen { + target_qualified: "_wm_target.dim", + select_sql: "SELECT id, name FROM dl.src", + partition_col: "_wm_partition", + partition_value_sql: "''", + partitioned: false, + strategy: MaterializeStrategy::Scd2 { + key: "id".to_string(), + track: vec![], + close_deleted: true, + }, + }; + let st = cg.statements(); + // the deleted-key set: current keys absent from the snapshot, captured + // before the transaction (like `changed`) + assert!(st.iter().any(|s| s.contains( + "CREATE OR REPLACE TEMP TABLE _wm_scd2_deleted AS SELECT \"id\" FROM \ + (SELECT \"id\" FROM _wm_target.dim WHERE is_current EXCEPT SELECT \"id\" FROM (SELECT id, name FROM dl.src));" + ))); + // a second close UPDATE against the deleted set (null-safe), and NO INSERT + // that reopens deleted keys (the only INSERT filters on `_wm_scd2_changed`) + assert!(st + .iter() + .any(|s| s.starts_with("UPDATE _wm_target.dim SET valid_to") + && s.contains( + "EXISTS (SELECT 1 FROM _wm_scd2_deleted \ + WHERE _wm_scd2_deleted.\"id\" IS NOT DISTINCT FROM _wm_target.dim.\"id\");" + ))); + assert_eq!( + st.iter().filter(|s| s.starts_with("INSERT INTO")).count(), + 1 + ); + assert!(st + .iter() + .find(|s| s.starts_with("INSERT INTO")) + .unwrap() + .contains("_wm_scd2_changed")); + // the deleted close is inside the transaction (between BEGIN and COMMIT) + let begin = st.iter().position(|s| s == "BEGIN TRANSACTION;").unwrap(); + let commit = st.iter().position(|s| s == "COMMIT;").unwrap(); + let del_close = st + .iter() + .position(|s| s.starts_with("UPDATE") && s.contains("_wm_scd2_deleted")) + .unwrap(); + assert!(begin < del_close && del_close < commit); + } + #[test] fn snapshot_capture_targets_alias() { assert_eq!( @@ -1165,11 +1497,15 @@ mod tests { partition_col: "_wm_partition", partition_value_sql: "'2026-06-19'", partitioned: true, + scd2: false, } } fn ctx_unpartitioned() -> DataTestCtx<'static> { DataTestCtx { partitioned: false, ..ctx_partitioned() } } + fn ctx_scd2() -> DataTestCtx<'static> { + DataTestCtx { partitioned: false, scd2: true, ..ctx_partitioned() } + } #[test] fn data_test_unique_and_not_null_partition_scoped() { @@ -1210,6 +1546,30 @@ mod tests { assert!(!sql.checks[0].violating.contains("_wm_partition")); } + #[test] + fn data_test_scd2_scopes_builtins_to_current_rows() { + // On an SCD2 history table the natural key repeats across closed + // versions, so built-in probes must assert the current snapshot only. + let tests = vec![ + DataTestResolved::BuiltIn(DataTest::Unique { column: "customer_id".into() }), + DataTestResolved::BuiltIn(DataTest::NotNull { column: "tier".into() }), + DataTestResolved::BuiltIn(DataTest::AcceptedValues { + column: "region".into(), + values: vec!["emea".into()], + }), + ]; + let sql = build_data_test_checks(&tests, &ctx_scd2()).unwrap(); + assert!(sql.checks[0] + .violating + .contains("WHERE \"customer_id\" IS NOT NULL AND is_current")); + assert!(sql.checks[1] + .violating + .contains("WHERE \"tier\" IS NULL AND is_current")); + assert!(sql.checks[2].violating.contains("AND is_current")); + // no partition scope leaks in (scd2 is unpartitioned in v1) + assert!(!sql.checks[0].violating.contains("_wm_partition")); + } + #[test] fn data_test_accepted_values_escapes_literals() { let tests = vec![DataTestResolved::BuiltIn(DataTest::AcceptedValues { diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json index 4ecbc992bc..f7977239c1 100644 --- a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -48,7 +48,11 @@ "expected": { "in_pipeline": true, "asset_triggers": [], - "native_triggers": ["kafka", "schedule", "data_upload"], + "native_triggers": [ + "kafka", + "schedule", + "data_upload" + ], "partition": null, "freshness": null, "tag": null, @@ -154,7 +158,10 @@ "partition": null, "freshness": null, "tag": null, - "retry": { "count": 3, "delay": "5s" } + "retry": { + "count": 3, + "delay": "5s" + } } }, { @@ -167,7 +174,10 @@ "partition": null, "freshness": null, "tag": null, - "retry": { "count": 2, "delay": null } + "retry": { + "count": 2, + "delay": null + } } }, { @@ -201,7 +211,9 @@ "code": "// pipeline\n// on s3://bucket/daily/{partition}/data.parquet\nexport function main() {}", "expected": { "in_pipeline": true, - "asset_triggers": ["s3object:bucket/daily/{partition}/data.parquet"], + "asset_triggers": [ + "s3object:bucket/daily/{partition}/data.parquet" + ], "native_triggers": [], "partition": null, "freshness": null, @@ -214,7 +226,9 @@ "code": " -- pipeline\n\t-- on datatable://main/x\nSELECT 1;", "expected": { "in_pipeline": true, - "asset_triggers": ["datatable:main/x"], + "asset_triggers": [ + "datatable:main/x" + ], "native_triggers": [], "partition": null, "freshness": null, @@ -253,6 +267,67 @@ } } }, + { + "name": "materialize scd2 via history flag with key, track and deletes=close", + "code": "// pipeline\n// materialize ducklake://analytics/dim_customer key=id history track=name,tier deletes=close\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim_customer", + "unique_key": "id", + "scd2": true, + "track": [ + "name", + "tier" + ], + "close_deleted": true + } + } + }, + { + "name": "materialize scd2 keyword alias with key only (track all non-key cols)", + "code": "// materialize scd2 ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id", + "scd2": true + } + } + }, + { + "name": "materialize key without history is plain merge (SCD1, not scd2)", + "code": "// materialize ducklake://analytics/dim key=id\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "materialize": { + "target_kind": "ducklake", + "target_path": "analytics/dim", + "unique_key": "id" + } + } + }, { "name": "materialize manual escape hatch, first value wins", "code": "// materialize manual ducklake://analytics/orders_daily\n// materialize ducklake://other/x\nexport function main() {}", @@ -319,12 +394,22 @@ "unique_key": "order_id" }, "data_tests": [ - { "type": "unique", "column": "order_id" }, - { "type": "not_null", "column": "user_id" }, + { + "type": "unique", + "column": "order_id" + }, + { + "type": "not_null", + "column": "user_id" + }, { "type": "accepted_values", "column": "status", - "values": ["paid", "pending", "refunded"] + "values": [ + "paid", + "pending", + "refunded" + ] }, { "type": "relationships", @@ -348,7 +433,15 @@ "tag": null, "retry": null, "data_tests": [ - { "type": "accepted_values", "column": "kind", "values": ["a b", "c", "d"] } + { + "type": "accepted_values", + "column": "kind", + "values": [ + "a b", + "c", + "d" + ] + } ] } }, @@ -363,7 +456,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "custom", "path": "f/tests/orders_amount_sane" }] + "data_tests": [ + { + "type": "custom", + "path": "f/tests/orders_amount_sane" + } + ] } }, { @@ -399,7 +497,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "unique", "column": "id" }] + "data_tests": [ + { + "type": "unique", + "column": "id" + } + ] } }, { @@ -413,7 +516,12 @@ "freshness": null, "tag": null, "retry": null, - "data_tests": [{ "type": "unique", "column": "id" }] + "data_tests": [ + { + "type": "unique", + "column": "id" + } + ] } }, { @@ -446,7 +554,11 @@ { "column": "user_name", "inputs": [ - { "from_kind": "datatable", "from_path": "prod/users", "from_column": "name" } + { + "from_kind": "datatable", + "from_path": "prod/users", + "from_column": "name" + } ] } ] @@ -506,5 +618,89 @@ } ] } + }, + { + "name": "bare macros marker", + "code": "// macros\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "macros": true + } + }, + { + "name": "macros with trailing prose is not a marker", + "code": "// macros are defined below\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "macros marker with sql comment prefix", + "code": "-- macros\n-- pipeline\nCREATE MACRO dbl(a) AS a * 2;", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "macros": true + } + }, + { + "name": "use accumulates in order and dedups", + "code": "// use f/lib/stats\n// use f/lib/dates\n// use f/lib/stats\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null, + "use_libs": [ + "f/lib/stats", + "f/lib/dates" + ] + } + }, + { + "name": "use requires a slashed single token", + "code": "// use this script to compute stuff\n// use standalone\n// use f/lib/ok extra\nSELECT 1;", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "use stops at first code line", + "code": "// pipeline\nSELECT 1;\n-- use f/lib/late\n", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } } ] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs index 883ddebcbc..2df4acd2e4 100644 --- a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -47,6 +47,12 @@ struct Expected { // against `to_value(got.column_lineage)`. Absent === []. #[serde(default)] column_lineage: Vec, + // `// macros` marker (strict, alone on the line). Absent === false. + #[serde(default)] + macros: bool, + // `// use ` accumulation, declaration order, deduped. Absent === []. + #[serde(default)] + use_libs: Vec, } #[derive(Deserialize)] @@ -59,6 +65,12 @@ struct ExpectedMaterialize { append: bool, #[serde(default)] unique_key: Option, + #[serde(default)] + scd2: bool, + #[serde(default)] + track: Vec, + #[serde(default)] + close_deleted: bool, } #[derive(Deserialize)] @@ -191,6 +203,12 @@ fn pipeline_annotation_fixtures_match() { assert_eq!(m.manual, e.manual, "{ctx}: materialize manual"); assert_eq!(m.append, e.append, "{ctx}: materialize append"); assert_eq!(m.unique_key, e.unique_key, "{ctx}: materialize key"); + assert_eq!(m.scd2, e.scd2, "{ctx}: materialize scd2"); + assert_eq!(m.track, e.track, "{ctx}: materialize track"); + assert_eq!( + m.close_deleted, e.close_deleted, + "{ctx}: materialize close_deleted" + ); } (got, want) => panic!( "{ctx}: materialize mismatch — got {:?}, want present={}", @@ -213,5 +231,8 @@ fn pipeline_annotation_fixtures_match() { serde_json::Value::Array(f.expected.column_lineage.clone()), "{ctx}: column lineage" ); + + assert_eq!(got.macros, f.expected.macros, "{ctx}: macros"); + assert_eq!(got.use_libs, f.expected.use_libs, "{ctx}: use_libs"); } } diff --git a/backend/src/main.rs b/backend/src/main.rs index 94b8e2f4f6..0186f271e6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1679,6 +1679,13 @@ async fn process_notify_event( ); windmill_queue::asset_dispatch::ASSET_PRODUCER_WRITES_CACHE.remove(payload); } + "notify_macro_registry_change" => { + tracing::debug!( + "Macro registry change for workspace {}, invalidating macro registry cache", + payload + ); + windmill_common::assets::MACRO_REGISTRY_CACHE.remove(payload); + } "notify_workspace_key_change" => { tracing::info!( "Workspace key change detected, invalidating workspace key cache: {}", @@ -1708,6 +1715,10 @@ async fn process_notify_event( match *source_type { "script" => { windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + // Bundle-cache key resolution for imported scripts; evicted + // together with the content-side caches below so key and + // inlined content flip to the new version in the same window. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&key); // Evict the relative-import latest-hash cache so a redeployed // imported script flips the content cache to its new version // across all replicas within a poll interval (see #6769). Keyed diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 6553abb5cc..380adf679e 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -115,6 +115,10 @@ kafka_pending_commits: id(bigint), workspace_id(char), kafka_trigger_path(char), FK: (workspace_id, kafka_trigger_path) -> kafka_trigger(workspace_id, path) kafka_trigger: path(char), kafka_resource_path(char), topics(char), group_id(char), script_path(char), is_flow(bool), workspace_id(char), edited_by(char), email(char), edited_at(ts), extra_perms(jsonb), server_id(char), last_server_ping(ts), error(text), error_handler_path(char), error_handler_args(jsonb), retry(jsonb), mode(trigger_mode), filters(jsonb[]), auto_commit(bool), labels(text[]) log_file: hostname(char), log_ts(ts), ok_lines(bigint), err_lines(bigint), mode(log_mode), worker_group(char), file_path(char), json_fmt(bool) +macro_definition: workspace_id(char), name(char), provider_path(char), params(text), body(text), is_table_macro(bool), created_at(ts) + FK: (workspace_id) -> workspace(id) +macro_usage: workspace_id(char), consumer_path(char), macro_name(char) + FK: (workspace_id) -> workspace(id) magic_link: email(char), token(char), expiration(ts) mcp_oauth_client: mcp_server_url(text), client_id(text), client_secret(text), client_secret_expires_at(ts), token_endpoint(text), created_at(ts) mcp_oauth_refresh_token: id(bigint), refresh_token(char), access_token_hash(char), client_id(char), user_email(char), workspace_id(char), scopes(text[]), token_family(uuid), created_at(ts), expires_at(ts), used_at(ts), revoked(bool) @@ -182,7 +186,7 @@ windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) -workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char) +workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) workspace_diff: source_workspace_id(char), fork_workspace_id(char), path(char), kind(char), ahead(int), behind(int), has_changes(bool), exists_in_source(bool), exists_in_fork(bool) diff --git a/backend/tests/bun_jobs.rs b/backend/tests/bun_jobs.rs index d51e51f58b..fc766253bc 100644 --- a/backend/tests/bun_jobs.rs +++ b/backend/tests/bun_jobs.rs @@ -1,3 +1,4 @@ +use futures::StreamExt; use sqlx::postgres::Postgres; use sqlx::Pool; use uuid::Uuid; @@ -818,6 +819,126 @@ export function main() { Ok(()) } +// ============================================================================ +// Bundle cache invalidation on transitive relative-import change +// ============================================================================ + +async fn insert_deployed_bun_script(db: &Pool, path: &str, hash: i64, content: &str) { + // What gen_bun_lockfile stores for a script with no npm dependencies; a + // bare '' lock fails split_lockfile when the script is run directly. + const EMPTY_BUN_LOCK: &str = "{\n \"dependencies\": {}\n}\n//bun.lock\n"; + // Runtime query to avoid touching the sqlx offline cache. + sqlx::query( + "INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) + VALUES ('test-workspace', 'test-user', $1, '{\"$schema\":\"https://json-schema.org/draft/2020-12/schema\",\"properties\":{},\"required\":[],\"type\":\"object\"}', '', '', $2, $3, 'bun', $4)", + ) + .bind(content) + .bind(path) + .bind(hash) + .bind(EMPTY_BUN_LOCK) + .execute(db) + .await + .unwrap(); +} + +fn run_main_script_job(hash: i64) -> RunJob { + RunJob::from(JobPayload::ScriptHash { + path: "f/stale_bundle/main_script".to_string(), + hash: windmill_common::scripts::ScriptHash(hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bun, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }) +} + +/// Editing a script that a runnable imports only TRANSITIVELY (main -> mid -> +/// leaf) must invalidate the runnable's cached bundle: the leaf's code is +/// inlined in the bundle, so the cache key has to cover the whole closure, not +/// just direct imports. +#[sqlx::test(fixtures("base"))] +async fn test_bun_transitive_import_change_rebundles(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Hashes/paths unique across this test binary: the script/hash caches are + // process-global while parallel tests each run in their own DB. + const LEAF_V1: i64 = 41230001; + const MID: i64 = 41230002; + const MAIN: i64 = 41230003; + const LEAF_V2: i64 = 41230004; + + insert_deployed_bun_script( + &db, + "f/stale_bundle/leaf", + LEAF_V1, + r#"export function leafValue() { return "V1_FROM_LEAF"; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/mid", + MID, + r#"import { leafValue } from "./leaf"; +export function midValue() { return `M(${leafValue()})`; }"#, + ) + .await; + insert_deployed_bun_script( + &db, + "f/stale_bundle/main_script", + MAIN, + r#"import { midValue } from "./mid"; +export function main() { return midValue(); }"#, + ) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + in_test_worker( + &db, + async move { + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V1_FROM_LEAF)")); + + // Deploy a new version of ONLY the leaf; main_script and mid keep + // their hash, content, and lock byte-identical. + insert_deployed_bun_script( + &db2, + "f/stale_bundle/leaf", + LEAF_V2, + r#"export function leafValue() { return "V2_FROM_LEAF"; }"#, + ) + .await; + + // Tests don't run the notify_event poll loop, so replay what its + // `notify_runnable_version_change` handler (main.rs) does on deploy: + // evict the leaf's latest-hash cache entries. + windmill_common::IMPORTED_SCRIPT_HASH_CACHE.remove(&( + "test-workspace".to_string(), + "f/stale_bundle/leaf".to_string(), + )); + windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE + .remove(&format!("test-workspace:f/stale_bundle/leaf")); + + let job = run_main_script_job(MAIN).push(&db2).await; + completed.next().await; + let result = completed_job(job, &db2).await.json_result().unwrap(); + assert_eq!(result, serde_json::json!("M(V2_FROM_LEAF)")); + }, + port, + ) + .await; + Ok(()) +} + #[sqlx::test(fixtures("base", "bun_edge_cases"))] async fn test_bun_shared_imports_both_styles(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; diff --git a/backend/tests/preserve_on_behalf_of.rs b/backend/tests/preserve_on_behalf_of.rs index 7ed5fc7697..d808348937 100644 --- a/backend/tests/preserve_on_behalf_of.rs +++ b/backend/tests/preserve_on_behalf_of.rs @@ -2704,7 +2704,9 @@ async fn test_schedule_permissions_workspace_admin(db: Pool) -> anyhow Ok(()) } -/// Superadmin NOT in workspace creates a schedule — uses email as permissioned_as +/// Superadmin NOT in workspace creates a schedule — uses their instance-derived +/// username (`password.username`) as permissioned_as, not the raw email. The +/// email is still stored directly on the schedule for downstream resolution. #[sqlx::test(fixtures("preserve_on_behalf_of"))] async fn test_schedule_permissions_superadmin_not_in_workspace( db: Pool, @@ -2758,16 +2760,16 @@ async fn test_schedule_permissions_superadmin_not_in_workspace( .fetch_one(&db) .await?; - // Superadmin not in workspace: username_to_permissioned_as uses the email directly - // since the authed username for a superadmin not in workspace IS the email + // Superadmin not in workspace: the authed username is now their instance-derived + // username (`password.username` = 'superadmin-external'), so permissioned_as is + // `u/` rather than the raw email. The email is still stored directly. assert_eq!( schedule.email, "superadmin-external@windmill.dev", "schedule email should be superadmin email" ); assert_eq!( - schedule.permissioned_as, - schedule.email.clone(), - "permissioned_as should match email for superadmin not in workspace" + schedule.permissioned_as, "u/superadmin-external", + "permissioned_as should use the instance-derived username, not the email" ); // Update by the same superadmin diff --git a/backend/windmill-ai/src/ai_providers.rs b/backend/windmill-ai/src/ai_providers.rs index 5d8e763b69..776228cea7 100644 --- a/backend/windmill-ai/src/ai_providers.rs +++ b/backend/windmill-ai/src/ai_providers.rs @@ -49,6 +49,8 @@ pub enum AIProvider { OpenAI, #[serde(rename = "azure_openai")] AzureOpenAI, + #[serde(rename = "azure_foundry")] + AzureFoundry, Anthropic, Mistral, DeepSeek, @@ -114,9 +116,12 @@ impl AIProvider { AIProvider::TogetherAI => Ok("https://api.together.xyz/v1".to_string()), AIProvider::Anthropic => Ok("https://api.anthropic.com/v1".to_string()), AIProvider::Mistral => Ok("https://api.mistral.ai/v1".to_string()), - p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI) => Err(Error::BadRequest( - format!("{:?} provider requires a base URL in the resource", p), - )), + p @ (AIProvider::CustomAI | AIProvider::AzureOpenAI | AIProvider::AzureFoundry) => { + Err(Error::BadRequest(format!( + "{:?} provider requires a base URL in the resource", + p + ))) + } AIProvider::AWSBedrock => { // AWS Bedrock uses the SDK directly, not HTTP base URL Err(Error::internal_err( @@ -131,13 +136,16 @@ impl AIProvider { matches!(self, AIProvider::Anthropic) } - /// Check if this provider/URL combination represents Azure OpenAI - pub fn is_azure_openai(&self, base_url: &str) -> bool { + /// Check whether this provider/URL combination uses Azure conventions + /// (the `api-key` auth header and Azure URL building). This covers Azure + /// OpenAI, Azure AI Foundry, and the `OpenAI` provider pointed at an Azure + /// base path override. + pub fn is_azure(&self, base_url: &str) -> bool { (matches!(self, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL) - || matches!(self, AIProvider::AzureOpenAI) + || matches!(self, AIProvider::AzureOpenAI | AIProvider::AzureFoundry) } - /// Build Azure OpenAI URL with deployment model path + /// Build an Azure-style URL (Azure OpenAI / Azure AI Foundry) for the given path pub fn build_azure_openai_url(base_url: &str, path: &str) -> String { let base_url = base_url.trim_end_matches('/'); if base_url.ends_with("/openai") { diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index 650ce1c19e..7b4dab5ab5 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -248,7 +248,7 @@ impl QueryBuilder for OtherQueryBuilder { } fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { - if self.provider_kind.is_azure_openai(base_url) { + if self.provider_kind.is_azure(base_url) { AIProvider::build_azure_openai_url(base_url, "chat/completions") } else { format!("{}/chat/completions", base_url) @@ -261,7 +261,7 @@ impl QueryBuilder for OtherQueryBuilder { base_url: &str, _output_type: &OutputType, ) -> Vec<(&'static str, String)> { - if self.provider_kind.is_azure_openai(base_url) { + if self.provider_kind.is_azure(base_url) { vec![("api-key", api_key.to_string())] } else { vec![("Authorization", format!("Bearer {}", api_key))] diff --git a/backend/windmill-ai/src/proxy.rs b/backend/windmill-ai/src/proxy.rs index 891fa89486..5de4d231c4 100644 --- a/backend/windmill-ai/src/proxy.rs +++ b/backend/windmill-ai/src/proxy.rs @@ -51,6 +51,7 @@ pub fn supports_openai_compatible_proxy(provider: &AIProvider) -> bool { provider, AIProvider::OpenAI | AIProvider::AzureOpenAI + | AIProvider::AzureFoundry | AIProvider::Mistral | AIProvider::DeepSeek | AIProvider::Groq @@ -64,6 +65,7 @@ pub fn proxy_execution_mode(provider: &AIProvider) -> ProxyExecutionMode { match provider { AIProvider::OpenAI | AIProvider::AzureOpenAI + | AIProvider::AzureFoundry | AIProvider::Anthropic | AIProvider::Mistral | AIProvider::DeepSeek @@ -89,7 +91,7 @@ pub fn build_openai_compatible_proxy_request(args: &ProxyBuildArgs<'_>) -> Resul }; let base_url = credentials.base_url.trim_end_matches('/'); - let is_azure = credentials.provider.is_azure_openai(base_url); + let is_azure = credentials.provider.is_azure(base_url); let url = if is_azure { AIProvider::build_azure_openai_url(base_url, args.path) } else { @@ -201,6 +203,7 @@ mod tests { let cases = [ (AIProvider::OpenAI, ProxyExecutionMode::HttpForward), (AIProvider::AzureOpenAI, ProxyExecutionMode::HttpForward), + (AIProvider::AzureFoundry, ProxyExecutionMode::HttpForward), (AIProvider::Anthropic, ProxyExecutionMode::HttpForward), (AIProvider::Mistral, ProxyExecutionMode::HttpForward), (AIProvider::DeepSeek, ProxyExecutionMode::HttpForward), @@ -253,6 +256,35 @@ mod tests { .contains(&("api-key".to_string(), "api-key".to_string()))); } + #[test] + fn builds_azure_foundry_proxy_request() { + // Foundry's OpenAI-compatible endpoint uses the same Azure conventions + // (api-key header, /openai -> /openai/v1 path) as Azure OpenAI. + let credentials = credentials( + AIProvider::AzureFoundry, + "https://example.services.ai.azure.com/openai", + ); + let method = Method::POST; + let headers = HeaderMap::new(); + + let request = build_openai_compatible_proxy_request(&ProxyBuildArgs { + method: &method, + path: "chat/completions", + headers: &headers, + body: br#"{"model":"gpt-4o","messages":[]}"#, + credentials: &credentials, + }) + .unwrap(); + + assert_eq!( + request.url, + "https://example.services.ai.azure.com/openai/v1/chat/completions" + ); + assert!(request + .headers + .contains(&("api-key".to_string(), "api-key".to_string()))); + } + #[test] fn injects_user_into_proxy_body() { let mut credentials = credentials(AIProvider::OpenAI, "https://api.openai.com/v1"); diff --git a/backend/windmill-api-assets/Cargo.toml b/backend/windmill-api-assets/Cargo.toml index c6e7676b6b..c75715a3ac 100644 --- a/backend/windmill-api-assets/Cargo.toml +++ b/backend/windmill-api-assets/Cargo.toml @@ -8,6 +8,9 @@ edition.workspace = true name = "windmill_api_assets" path = "src/lib.rs" +[features] +private = ["windmill-common/private"] + [dependencies] windmill-api-auth.workspace = true windmill-common = { workspace = true, default-features = false } diff --git a/backend/windmill-api-assets/src/backfill_oss.rs b/backend/windmill-api-assets/src/backfill_oss.rs new file mode 100644 index 0000000000..b5f0da1d58 --- /dev/null +++ b/backend/windmill-api-assets/src/backfill_oss.rs @@ -0,0 +1,18 @@ +//! OSS fallback: backfilling a range of partitions is an enterprise +//! feature (the resolution/enumeration logic lives in `windmill-ee-private`, +//! `windmill-api-assets/src/backfill_ee.rs`). Single-partition runs with an +//! explicit `partition` arg remain available in OSS. + +use windmill_common::error::{Error, Result}; + +use crate::{PartitionsInRangeQuery, PartitionsInRangeResponse}; + +pub(crate) async fn partitions_in_range( + _tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + _w_id: &str, + _q: &PartitionsInRangeQuery, +) -> Result { + Err(Error::BadRequest( + "Backfilling a range of partitions is an enterprise feature".to_string(), + )) +} diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 29122bd793..b4b0583863 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -13,7 +13,19 @@ use windmill_common::{ utils::escape_ilike_pattern, }; -use windmill_api_auth::ApiAuthed; +use windmill_api_auth::{build_scope_path_predicate, ApiAuthed}; + +// Partition-range backfill preview. The logic (producer resolution, range +// enumeration, status join) is enterprise: the `private` build compiles the +// EE module, the public build a stub that errors. +#[cfg(feature = "private")] +mod backfill_ee; +#[cfg(feature = "private")] +use backfill_ee as backfill; +#[cfg(not(feature = "private"))] +mod backfill_oss; +#[cfg(not(feature = "private"))] +use backfill_oss as backfill; pub fn workspaced_service() -> Router { Router::new() @@ -23,8 +35,66 @@ pub fn workspaced_service() -> Router { .route("/graph", get(asset_graph)) .route("/pipelines", get(list_pipeline_folders)) .route("/partitions", get(list_partitions)) + .route("/partitions_in_range", get(list_partitions_in_range)) .route("/asset_schemas", get(list_asset_schemas)) .route("/record_materialization", post(record_materialization)) + .route("/macros", get(list_macros)) +} + +// One registry macro, with its full definition — drives the macro-explorer +// drawer (body preview) and the DuckDB editor autocomplete (signatures). +#[derive(Serialize)] +struct MacroListItem { + name: String, + params: String, + body: String, + is_table: bool, + provider_path: String, +} + +// Every workspace macro (`// macros` libraries), grouped client-side by +// provider. Small by construction — one row per macro definition. The rows +// copy script body text, so visibility must match reading the provider +// script itself: the EXISTS join runs under the user_db transaction (script +// RLS filters libraries the caller can't read) and the scope predicate +// covers path-scoped tokens, mirroring `list_scripts`. +async fn list_macros( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let scope_allowed = build_scope_path_predicate(&authed, "scripts", "read"); + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#"SELECT m.name AS "name!", m.params AS "params!", m.body AS "body!", + m.is_table_macro AS "is_table_macro!", m.provider_path AS "provider_path!" + FROM macro_definition m + WHERE m.workspace_id = $1 + AND EXISTS ( + SELECT 1 FROM script s + WHERE s.workspace_id = m.workspace_id + AND s.path = m.provider_path + AND s.archived = false + AND s.deleted = false + ) + ORDER BY m.provider_path, m.name"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + Ok(Json( + rows.into_iter() + .filter(|r| scope_allowed(&r.provider_path)) + .map(|r| MacroListItem { + name: r.name, + params: r.params, + body: r.body, + is_table: r.is_table_macro, + provider_path: r.provider_path, + }) + .collect(), + )) } #[derive(Deserialize)] @@ -54,6 +124,55 @@ async fn list_partitions( Ok(Json(rows)) } +// Only the EE `backfill` module reads the fields; the OSS stub errors without +// touching them. +#[cfg_attr(not(feature = "private"), allow(dead_code))] +#[derive(Deserialize)] +struct PartitionsInRangeQuery { + // The materialized ducklake asset path (`/`). + path: String, + // Inclusive calendar-day range (YYYY-MM-DD), local to the producer's + // partition tz. + from: chrono::NaiveDate, + to: chrono::NaiveDate, +} + +#[derive(Serialize)] +struct PartitionInRange { + partition: String, + // `missing` | `running` | `materialized` | `failed` — `missing` means no + // materialization was ever recorded for the slice. + status: &'static str, +} + +#[derive(Serialize)] +struct PartitionsInRangeResponse { + // The pipeline script that materializes the asset (managed `// materialize` + // target, or a partitioned writer using the SDK helpers) — the runnable a + // backfill launches (with an explicit `partition` arg per slice). + producer_path: String, + partition_kind: String, + partitions: Vec, +} + +// Backfill range preview: every partition the producer's `// partitioned` spec +// expects in `[from, to]`, joined with what `materialized_partition` records — +// the missing/failed subset is the backfill worklist. The logic is in the +// `backfill` module pair: EE resolves and enumerates, the OSS stub errors +// (single-partition runs stay available everywhere; fanning out over a range +// is enterprise). +async fn list_partitions_in_range( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let res = backfill::partitions_in_range(&mut tx, &w_id, &q).await?; + tx.commit().await?; + Ok(Json(res)) +} + // Per-asset captured output schema versions for a ducklake asset (gap #2a) — // the schema-evolution history persisted after each managed `// materialize`. // Newest version first; materialization targets are ducklake-only in v1, so the @@ -537,6 +656,32 @@ struct GraphRunnableNode { // `merge` / any partitioned write INSERTs into a fixed-schema table. #[serde(skip_serializing_if = "Option::is_none", default)] materialize_strategy: Option, + // Macros this script provides to the workspace registry (deployed + // `// macros` library). Drives the library node state + details-pane + // signature list. Lockstep with TS `AssetGraphRunnableNode.macros`. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + macros: Vec, +} + +// One macro of a `// macros` library, as surfaced on its graph node. +#[derive(Serialize, Debug, Clone)] +struct MacroInfo { + name: String, + // Verbatim parameter list, for the `name(params)` signature display. + params: String, + is_table: bool, +} + +// A macro-library → consumer edge: the consumer calls `macro_names` of +// `lib_path`'s macros (deploy-recorded detection), or pulls in the whole +// library via `// use` (`via_use`, in which case `macro_names` lists all of +// the library's macros). +#[derive(Serialize, Debug)] +struct MacroEdge { + lib_path: String, + consumer_path: String, + macro_names: Vec, + via_use: bool, } // The output asset a producer's column lineage belongs to (the `// materialize` @@ -637,6 +782,8 @@ struct AssetGraphResponse { runnables: Vec, edges: Vec, triggers: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + macro_edges: Vec, } async fn asset_graph( @@ -794,6 +941,31 @@ async fn asset_graph( .fetch_all(&mut *tx) .await?; + // Workspace macro registry + deploy-recorded call edges. Definitions are + // fetched unfiltered so an out-of-folder library still appears as the + // provider endpoint of in-scope consumers' edges; consumers honor the + // folder filter like every other runnable query. + let macro_def_rows = sqlx::query!( + r#"SELECT name AS "name!", provider_path AS "provider_path!", + params AS "params!", is_table_macro AS "is_table_macro!" + FROM macro_definition + WHERE workspace_id = $1 + ORDER BY provider_path, name"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + let macro_usage_rows = sqlx::query!( + r#"SELECT consumer_path AS "consumer_path!", macro_name AS "macro_name!" + FROM macro_usage + WHERE workspace_id = $1 + AND ($2::text IS NULL OR consumer_path LIKE $2)"#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; // Parse each pipeline member's body once into its badge annotations, keyed @@ -950,6 +1122,74 @@ async fn asset_graph( triggers.push(edge); } + // Macro libraries + lib→consumer edges. Group per-provider macro lists, + // resolve each usage row's name to its provider (names are + // workspace-unique), and merge `// use` whole-lib edges from the parsed + // member annotations. Both endpoints are forced into the runnable set so + // an out-of-folder library still renders as the edge's provider node. + let mut macros_by_provider: std::collections::HashMap> = + Default::default(); + let mut provider_by_name: std::collections::HashMap = Default::default(); + for r in macro_def_rows { + provider_by_name.insert(r.name.clone(), r.provider_path.clone()); + macros_by_provider + .entry(r.provider_path) + .or_default() + .push(MacroInfo { name: r.name, params: r.params, is_table: r.is_table_macro }); + } + let mut macro_edge_map: std::collections::BTreeMap< + (String, String), + (std::collections::BTreeSet, bool), + > = Default::default(); + for u in macro_usage_rows { + // Same orphan filter as the other edge loops. + if !runnable_exists(AssetUsageKind::Script, &u.consumer_path) { + continue; + } + let Some(lib) = provider_by_name.get(&u.macro_name) else { + continue; + }; + let e = macro_edge_map + .entry((lib.clone(), u.consumer_path)) + .or_default(); + e.0.insert(u.macro_name); + } + for (path, ann) in &annotations_by_path { + for lib in &ann.use_libs { + // An undeployed `// use` target has no registry rows — the live + // draft overlay is the only surface that can render it. + let Some(lib_macros) = macros_by_provider.get(lib) else { + continue; + }; + let e = macro_edge_map + .entry((lib.clone(), path.clone())) + .or_default(); + e.1 = true; + e.0.extend(lib_macros.iter().map(|m| m.name.clone())); + } + } + let macro_edges: Vec = macro_edge_map + .into_iter() + // Same orphan filter as the other edge families: a registry row whose + // provider script no longer exists must not synthesize a phantom + // library node (consumers were filtered above, but the `// use` pass + // re-adds them, so re-check both endpoints). + .filter(|((lib_path, consumer_path), _)| { + runnable_exists(AssetUsageKind::Script, lib_path) + && runnable_exists(AssetUsageKind::Script, consumer_path) + }) + .map(|((lib_path, consumer_path), (names, via_use))| MacroEdge { + lib_path, + consumer_path, + macro_names: names.into_iter().collect(), + via_use, + }) + .collect(); + for e in ¯o_edges { + runnable_set.insert((AssetUsageKind::Script, e.lib_path.clone())); + runnable_set.insert((AssetUsageKind::Script, e.consumer_path.clone())); + } + let mut assets: Vec = asset_set .into_iter() .map(|(kind, path)| GraphAssetNode { kind, path }) @@ -1002,6 +1242,11 @@ async fn asset_graph( Some("replace".to_string()) } }), + macros: (usage_kind == AssetUsageKind::Script) + .then(|| macros_by_provider.get(&path)) + .flatten() + .cloned() + .unwrap_or_default(), path, usage_kind, } @@ -1014,6 +1259,7 @@ async fn asset_graph( runnables, edges, triggers, + macro_edges, })) } diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs index a6a0fc4a8e..dea200e417 100644 --- a/backend/windmill-api-auth/src/auth.rs +++ b/backend/windmill-api-auth/src/auth.rs @@ -30,6 +30,7 @@ use windmill_common::{ }, error::{Error, JsonResult}, jwt, + usernames::get_instance_username_or_fallback_to_email, users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, }; @@ -419,18 +420,34 @@ impl AuthCache { read_only, }) } - None if super_admin => Some(ApiAuthed { - email: email.clone(), - username: email, - is_admin: super_admin, - is_operator: false, - groups: vec![], - folders: vec![], - scopes, - username_override, - token_prefix: Some(safe_token_prefix(token)), - read_only, - }), + None if super_admin => { + // Fail closed on a DB error rather than + // letting the email leak in as the username. + match get_instance_username_or_fallback_to_email( + &self.db, &email, + ) + .await + { + Ok(username) => Some(ApiAuthed { + email, + username, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + token_prefix: Some(safe_token_prefix(token)), + read_only, + }), + Err(e) => { + tracing::error!( + "Failed to resolve instance username for superadmin {email}: {e:#}" + ); + None + } + } + } None => None, } } else { diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index b5f8e2f285..a9376d02ed 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -155,10 +155,11 @@ async fn list_flows( "o.labels", "draft.email IS NOT NULL as is_draft", // Per-path draft owners as a JSON array; see scripts.rs for the rationale - // (admins-workspace identity fallback, legacy NULL-email row). - "(SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END)) ORDER BY COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) NULLS LAST) \ + // (non-member superadmin identity fallback via `password`, legacy NULL-email row). + "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'flow') as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) diff --git a/backend/windmill-api-groups/src/folders.rs b/backend/windmill-api-groups/src/folders.rs index eedb43bc1a..67997bc965 100644 --- a/backend/windmill-api-groups/src/folders.rs +++ b/backend/windmill-api-groups/src/folders.rs @@ -146,12 +146,11 @@ async fn list_foldernames( } fn validate_owner(owner: &str) -> Result<()> { - if !owner - .chars() - .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-') - { + if !owner.chars().all(|c| { + c.is_ascii_alphanumeric() || c == '_' || c == '/' || c == '-' || c == '.' || c == '@' + }) { return Err(error::Error::BadRequest( - "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, or slashes".to_string(), + "Invalid owner: must contain only alphanumeric characters, underscores, hyphens, slashes, dots, or at-signs".to_string(), )); } Ok(()) diff --git a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs index 88ffe6bb26..e98ee07a61 100644 --- a/backend/windmill-api-integration-tests/tests/workspace_comparison.rs +++ b/backend/windmill-api-integration-tests/tests/workspace_comparison.rs @@ -1399,7 +1399,9 @@ async fn test_delete_fork_purges_workspace_diff(db: Pool) -> anyhow::R // Delete the fork through the real handler. let delete_response = client .client() - .delete(&format!("{base_url}/workspaces/delete/wm-fork-test-workspace")) + .delete(&format!( + "{base_url}/workspaces/delete/wm-fork-test-workspace" + )) .send() .await?; assert!( @@ -1521,3 +1523,249 @@ async fn test_create_fork_purges_stale_diff_state(db: Pool) -> anyhow: Ok(()) } + +/// Regression: a stale/phantom trigger diff row must never block a privileged +/// user's deploy. Triggers (unlike scripts/flows) are not re-validated by +/// `compare_workspaces`, so a cached `has_changes=true` row for a trigger that +/// no longer exists in the table is trusted, then dropped by the visibility +/// filter (the row is gone) — flipping `all_ahead_items_visible` to false and +/// hiding the deploy button. This used to happen even for a superadmin, because +/// the item's absence is indistinguishable from a permission-hidden item. +/// +/// The blast-radius guard forces the flag true for anyone who sees the relevant +/// side in full: a target/fork admin (or superadmin) for ahead items. A regular +/// user with no such visibility still gets the (conservative) warning, since we +/// cannot tell a phantom from a genuine permission gap on their behalf. +/// +/// The diff row and visibility query are OSS, so this runs on any build (no +/// tally needed — the row is inserted directly, mimicking a delete that left the +/// diff behind). +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_compare_workspaces_phantom_trigger_shortfuse( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base_url = format!("http://localhost:{port}/api"); + let superadmin = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + let non_admin = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN_2".to_string(), + ); + + // Fork of test-workspace (INSERT directly; we only need the pair to exist). + sqlx::query!( + "INSERT INTO workspace (id, name, owner, parent_workspace_id) + VALUES ('wm-fork-test-workspace', 'Fork', 'test-user', 'test-workspace')" + ) + .execute(&db) + .await?; + sqlx::query!("INSERT INTO workspace_settings (workspace_id) VALUES ('wm-fork-test-workspace')") + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO workspace_key(workspace_id, kind, key) + VALUES ('wm-fork-test-workspace', 'cloud', 'test-key')" + ) + .execute(&db) + .await?; + + // Phantom rows: cached diffs for http_triggers with no backing row (the exact + // state a trigger delete used to leave behind before it reset the tally). One + // ahead (fork side), one behind (source side) so both guard branches are hit. + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork) + VALUES + ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost', 'http_trigger', 1, 0, true, false, true), + ('test-workspace', 'wm-fork-test-workspace', 'f/rt/ghost_behind', 'http_trigger', 0, 1, true, true, false)" + ) + .execute(&db) + .await?; + + // Superadmin: the guard forces `all_ahead_items_visible = true`, and the + // non-existent trigger is not surfaced as a diff. + let comparison: serde_json::Value = superadmin + .client() + .get(&format!( + "{base_url}/w/test-workspace/workspaces/compare/wm-fork-test-workspace" + )) + .send() + .await? + .json() + .await?; + assert_eq!( + comparison["all_ahead_items_visible"].as_bool(), + Some(true), + "phantom trigger row must not trip the 'not visible' warning for a superadmin: {comparison}" + ); + assert_eq!( + comparison["all_behind_items_visible"].as_bool(), + Some(true), + "phantom behind trigger row must not trip the warning for a superadmin: {comparison}" + ); + assert!( + !comparison["diffs"] + .as_array() + .unwrap() + .iter() + .any(|d| d["path"] == "f/rt/ghost" || d["path"] == "f/rt/ghost_behind"), + "non-existent triggers must not be surfaced as diffs: {comparison}" + ); + + // Non-superadmin, non-fork-admin member of the source: no full-visibility + // guarantee, so the warning still (conservatively) fires. + let comparison: serde_json::Value = non_admin + .client() + .get(&format!( + "{base_url}/w/test-workspace/workspaces/compare/wm-fork-test-workspace" + )) + .send() + .await? + .json() + .await?; + assert_eq!( + comparison["all_ahead_items_visible"].as_bool(), + Some(false), + "a user without full visibility must not be short-circuited by the guard: {comparison}" + ); + assert_eq!( + comparison["all_behind_items_visible"].as_bool(), + Some(false), + "the behind-side guard must not fire for a non-admin either: {comparison}" + ); + + Ok(()) +} + +/// Regression: the "sees everything" guard must require admin of BOTH sides, not +/// just the fork. `filter_visible_diffs` keeps a modified/conflict row (one that +/// exists in the source AND the fork) only when the caller can see it on both +/// sides, so an ahead diff can be dropped for a *source-side* visibility gap even +/// when the caller is a fork admin. If the guard cleared the ahead flag on +/// fork-admin alone, the UI would report "all ahead visible" and let the user +/// deploy from an incomplete comparison. Here test-user-2 is admin of the fork +/// but only a plain member of the parent with no access to folder `restricted`, +/// so the parent copy of the modified script is hidden from them and the ahead +/// diff is (correctly) dropped — the flag must stay false. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_compare_workspaces_fork_admin_source_hidden_ahead( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base_url = format!("http://localhost:{port}/api"); + let admin = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + let fork_admin_user = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN_2".to_string(), + ); + + // Parent folder `restricted` owned by test-user (the admin), NOT test-user-2, + // and a script inside it (access flows through the folder). test-user-2 is a + // plain member of the parent, so RLS hides this script from them. + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, summary, created_by) + VALUES ('test-workspace', 'restricted', 'restricted', ARRAY['u/test-user']::varchar[], '{\"u/test-user\": true}'::jsonb, '', 'test-user')" + ) + .execute(&db) + .await?; + sqlx::query!( + "INSERT INTO script (workspace_id, path, hash, content, summary, description, language, created_by, created_at, archived, schema_validation, ws_error_handler_muted, deleted, extra_perms) + VALUES ('test-workspace', 'f/restricted/item', 314159, 'def main(): return 1', '', '', 'python3', 'test-user', NOW(), false, false, false, false, '{}'::jsonb)" + ) + .execute(&db) + .await?; + + // Fork (clones the folder + script into the fork). + let resp = admin + .client() + .post(&format!( + "{base_url}/w/test-workspace/workspaces/create_fork" + )) + .json(&json!({"id": "wm-fork-guard-test", "name": "Guard Fork", "color": "#0000ff"})) + .send() + .await?; + assert!( + resp.status().is_success(), + "fork creation failed: {}", + resp.status() + ); + + // test-user-2 is ADMIN of the fork (so they see the fork copy via RLS bypass) + // but only a plain member of the parent. + sqlx::query!( + "INSERT INTO usr (workspace_id, email, username, is_admin, role) + VALUES ('wm-fork-guard-test', 'test2@windmill.dev', 'test-user-2', true, 'Admin')" + ) + .execute(&db) + .await?; + + // A confirmed modified/ahead diff on the script that exists on both sides. + sqlx::query!( + "INSERT INTO workspace_diff + (source_workspace_id, fork_workspace_id, path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork) + VALUES ('test-workspace', 'wm-fork-guard-test', 'f/restricted/item', 'script', 1, 0, true, true, true)" + ) + .execute(&db) + .await?; + sqlx::query!( + "DELETE FROM skip_workspace_diff_tally WHERE workspace_id IN ('test-workspace', 'wm-fork-guard-test')" + ) + .execute(&db) + .await?; + + let comparison: serde_json::Value = fork_admin_user + .client() + .get(&format!( + "{base_url}/w/test-workspace/workspaces/compare/wm-fork-guard-test" + )) + .send() + .await? + .json() + .await?; + + // The parent copy is hidden from test-user-2, so the ahead diff is dropped. + // Fork-admin alone must NOT clear the flag — the comparison is incomplete. + assert_eq!( + comparison["all_ahead_items_visible"].as_bool(), + Some(false), + "fork admin without source-side visibility must not be reported as seeing all ahead items: {comparison}" + ); + assert!( + !comparison["diffs"] + .as_array() + .unwrap() + .iter() + .any(|d| d["path"] == "f/restricted/item"), + "source-hidden item must not appear in the fork admin's diff list: {comparison}" + ); + + // Sanity: a superadmin (admin of both sides) still sees everything. + let comparison: serde_json::Value = admin + .client() + .get(&format!( + "{base_url}/w/test-workspace/workspaces/compare/wm-fork-guard-test" + )) + .send() + .await? + .json() + .await?; + assert_eq!( + comparison["all_ahead_items_visible"].as_bool(), + Some(true), + "superadmin must see all ahead items: {comparison}" + ); + + Ok(()) +} diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 9aecd66947..4f9de4f416 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -211,12 +211,15 @@ async fn list_scripts( // Canonical reference for the draft-feature comments; flows/apps point here. // Per-path draft owners as a JSON array (`Json>`); NULL -> None, // never an empty array. LEFT JOIN `usr` keeps orphaned drafts (user left workspace) - // visible with `username = None`. In the `admins` workspace username IS the email, - // so fall back to `d.email` there or the authed user's own draft resolves to a phantom - // "Legacy workspace draft"; the genuine NULL-email legacy row stays None. - "(SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END)) ORDER BY COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) NULLS LAST) \ + // visible with `username = None`. A superadmin authoring in a workspace they are not + // a member of has no `usr` row, so fall back to their instance-derived username + // (`password.username`), or their email when derivation is disabled — this keeps the + // raw email out of the payload whenever a derived username exists. The genuine + // NULL-email legacy row stays None (no `usr`/`password` match, `d.email` is NULL). + "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ WHERE d.workspace_id = o.workspace_id AND d.path = o.path AND d.typ = 'script') as draft_users", "folder_labels(o.workspace_id, o.path) as inherited_labels" ]) @@ -408,6 +411,14 @@ async fn list_scripts( .and_then(|s| s.as_str()) .filter(|s| !s.is_empty() && *s != row.path.as_str()) .map(|s| s.to_string()); + // A draft-only pipeline node (`// pipeline`) has no deployed row to carry + // auto_kind, so compute it from the draft content — mirroring the create + // path — so the home page folds it into its pipeline like a deployed member. + let auto_kind = v + .get("content") + .and_then(|s| s.as_str()) + .filter(|c| parse_pipeline_annotations(c).in_pipeline) + .map(|_| "pipeline".to_string()); rows.push(ListableScript { hash: ScriptHash(0), path: row.path, @@ -429,7 +440,7 @@ async fn list_scripts( draft_only: Some(true), has_deploy_errors: false, ws_error_handler_muted: None, - auto_kind: None, + auto_kind, use_codebase: false, deployment_msg: None, kind, @@ -1289,16 +1300,94 @@ async fn create_script_internal<'c>( // but the executor parses the signature from the un-wrapped script, so // `$name` references in the SELECT stay bound at run time. } - // `key=` (merge) and `append` are mutually exclusive reconciliation - // strategies; append (INSERT-only) wins. Surface the conflict rather - // than silently dropping the dedup the author may have intended. - if m.unique_key.is_some() && m.append { + // Reconciliation strategies are mutually exclusive; surface a conflict + // rather than silently dropping behavior the author may have intended. + // Precedence must mirror the runtime (`duckdb_executor` strategy + // derivation): scd2 (`history`) > append > merge (`key=`) > replace. + if m.scd2 && m.append { + tracing::warn!( + "script {}: both `history`/`scd2` and `append` set on // materialize; history wins (SCD2), append ignored", + ns.path + ); + } else if m.unique_key.is_some() && m.append { tracing::warn!( "script {}: both `key=` and `append` set on // materialize; append wins (INSERT-only, no dedup)", ns.path ); } } + // `// macros` — this script is a workspace macro library: its body is + // CREATE [OR REPLACE] MACRO statements plus plain setup, registered into + // `macro_definition` and injected as TEMP macros into consumer jobs. + // Pure shape checks happen here; the registry writes and the cross-lib + // name-collision check run inside the deploy transaction below. + let macro_lib_defs: Option> = + if pipeline_annotations.macros { + if ns.language != ScriptLang::DuckDb { + return Err(Error::BadRequest(format!( + "`// macros` is only supported for DuckDB scripts, not {}", + ns.language.as_str() + ))); + } + if pipeline_annotations.materialize.is_some() { + return Err(Error::BadRequest( + "`// macros` and `// materialize` are mutually exclusive — a macro \ + library defines reusable macros, it does not produce an asset" + .to_string(), + )); + } + let lib = windmill_parser::duckdb_macros::parse_macro_library(&ns.content) + .map_err(Error::BadRequest)?; + let macros: Vec<_> = lib + .into_iter() + .filter_map(|s| match s { + windmill_parser::duckdb_macros::LibStatement::Macro(m) => Some(m), + windmill_parser::duckdb_macros::LibStatement::Setup(_) => None, + }) + .collect(); + if macros.is_empty() { + return Err(Error::BadRequest( + "`// macros` library defines no macros".to_string(), + )); + } + let all_names: std::collections::HashSet = + macros.iter().map(|m| m.name.clone()).collect(); + // DuckDB bind-checks a macro body at CREATE time, so a body may + // only call same-file macros defined *earlier* — this also makes + // file order a valid creation order for whole-lib (`// use`) + // injection and rules out within-lib cycles. + let mut defined: std::collections::HashSet = Default::default(); + for m in ¯os { + if windmill_parser::duckdb_builtins::is_duckdb_builtin(&m.name) { + return Err(Error::BadRequest(format!( + "macro `{}` shadows a DuckDB built-in function; pick another name", + m.name + ))); + } + if defined.contains(&m.name) { + return Err(Error::BadRequest(format!( + "macro `{}` is defined twice in this library", + m.name + ))); + } + let called = + windmill_parser::duckdb_macros::detect_macro_calls(&m.body, &all_names); + if let Some(fwd) = called + .iter() + .find(|c| !defined.contains(*c) && *c != &m.name) + { + return Err(Error::BadRequest(format!( + "macro `{}` calls `{}`, which is defined later in the file; define \ + `{}` first (DuckDB bind-checks macro bodies at creation)", + m.name, fwd, fwd + ))); + } + defined.insert(m.name.clone()); + } + Some(macros) + } else { + None + }; let in_pipeline = pipeline_annotations.in_pipeline; // `// trigger all` → AND join barrier (else OR, the default). let pipeline_join_all = !pipeline_annotations.join_mode.is_any(); @@ -1335,20 +1424,36 @@ async fn create_script_internal<'c>( let effective_assets = if let Some(m) = pipeline_annotations.materialize.as_ref() { let kind = windmill_common::assets::asset_kind_from_parser(m.target_kind); let mut a = effective_assets.unwrap_or_default(); - if !a.iter().any(|x| x.kind == kind && x.path == m.target_path) { - a.push(windmill_common::assets::AssetWithAltAccessType { - path: m.target_path.clone(), - kind, - access_type: Some(windmill_common::assets::AssetUsageAccessType::W), - alt_access_type: None, - columns: None, - }); + // Produced assets: the managed table, plus — for managed scd2 — the + // `_current` companion view the runtime (re)creates each run. + // Registering the view as a write asset lets `// on + // ducklake://…/_current` subscribers be dispatched by the cascade + // (which fans out from these deploy-time asset rows); without it a + // subscriber on the view would silently never fire. Gated on `!manual`: + // manual mode owns its own DDL and short-circuits before the scd2 codegen + // (no view is created), so registering it there would be a false edge. + let mut targets = vec![m.target_path.clone()]; + if m.scd2 && !m.manual { + targets.push(format!("{}_current", m.target_path)); + } + for path in targets { + if !a.iter().any(|x| x.kind == kind && x.path == path) { + a.push(windmill_common::assets::AssetWithAltAccessType { + path, + kind, + access_type: Some(windmill_common::assets::AssetUsageAccessType::W), + alt_access_type: None, + columns: None, + }); + } } Some(a) } else { effective_assets }; - let auto_kind = if in_pipeline { + let auto_kind = if in_pipeline || macro_lib_defs.is_some() { + // A macro library is a pipeline member (graph node) even without the + // bare `// pipeline` marker. Some("pipeline".to_string()) } else if ci_test_refs.is_some() { Some("test".to_string()) @@ -1469,6 +1574,173 @@ async fn create_script_internal<'c>( } } + // DuckDB workspace macros: wipe-and-reinsert this script's registry rows + // (as provider) and call-site edges (as consumer), by new+old path so + // renames are handled — mirrors the ci_test_reference block above. + let macro_defs_deleted = sqlx::query!( + "DELETE FROM macro_definition WHERE workspace_id = $1 AND (provider_path = $2 OR provider_path = $3)", + &w_id, + &ns.path, + old_path.unwrap_or(&ns.path) + ) + .execute(&mut *tx) + .await? + .rows_affected(); + sqlx::query!( + "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 OR consumer_path = $3)", + &w_id, + &ns.path, + old_path.unwrap_or(&ns.path) + ) + .execute(&mut *tx) + .await?; + // Workers cache the registry per workspace; a mutation must evict it on + // every replica. Transactional emit — pollers see it only on commit. + if macro_lib_defs.is_some() || macro_defs_deleted > 0 { + sqlx::query!( + "INSERT INTO notify_event (channel, payload) VALUES ('notify_macro_registry_change', $1)", + &w_id + ) + .execute(&mut *tx) + .await?; + } + + if let Some(ref macros) = macro_lib_defs { + // Macro names are workspace-unique (they're injected unqualified into + // consumers). This lib's own rows — old and new path — were deleted + // just above, so any remaining match is a genuine cross-lib clash. + let names: Vec = macros.iter().map(|m| m.name.clone()).collect(); + if let Some(clash) = sqlx::query!( + "SELECT name, provider_path FROM macro_definition WHERE workspace_id = $1 AND name = ANY($2) LIMIT 1", + &w_id, + &names[..] + ) + .fetch_optional(&mut *tx) + .await? + { + return Err(Error::BadRequest(format!( + "macro `{}` is already defined by `{}`; macro names are workspace-unique", + clash.name, clash.provider_path + ))); + } + for m in macros { + sqlx::query!( + "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro) \ + VALUES ($1, $2, $3, $4, $5, $6)", + &w_id, + &m.name, + &ns.path, + &m.params, + &m.body, + m.is_table + ) + .execute(&mut *tx) + .await?; + } + } + + if ns.language == ScriptLang::DuckDb { + // Record this script's macro-call edges for the asset graph (the + // worker re-detects calls live at job time, so these are display + // metadata only). A library's edges point at *other* libs' macros. + let registry_names: Vec = sqlx::query_scalar!( + "SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path != $2", + &w_id, + &ns.path + ) + .fetch_all(&mut *tx) + .await?; + let names_set: std::collections::HashSet = registry_names.into_iter().collect(); + let mut called: Vec = + windmill_parser::duckdb_macros::detect_macro_calls(&ns.content, &names_set) + .into_iter() + .collect(); + called.sort(); + for name in called { + sqlx::query!( + "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + &ns.path, + &name + ) + .execute(&mut *tx) + .await?; + } + // `// use` targets should be deployed macro libraries — warn only: + // git-sync deploys scripts in arbitrary order, so the consumer landing + // first must not hard-fail. The runtime is the enforcement point. + for lib in &pipeline_annotations.use_libs { + let exists = sqlx::query_scalar!( + r#"SELECT EXISTS(SELECT 1 FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2) AS "e!""#, + &w_id, + lib + ) + .fetch_one(&mut *tx) + .await?; + if !exists { + tracing::warn!( + "script {}: `// use {}` does not (yet) match a deployed macro library", + ns.path, + lib + ); + } + } + } else if !pipeline_annotations.use_libs.is_empty() { + tracing::warn!( + "script {}: `// use` is only honored on DuckDB scripts; annotation ignored", + ns.path + ); + } + + if let Some(ref macros) = macro_lib_defs { + // Refresh every other DuckDB script's edges for this lib's names (a + // redeploy may add/remove macros; late-bound runtime picks changes up + // regardless — this only keeps the deployed graph current). The rescan + // covers ALL live DuckDB scripts, not just pipeline members: their + // edges were recorded at their own deploy and must survive the wipe. + let names: Vec = macros.iter().map(|m| m.name.clone()).collect(); + sqlx::query!( + "DELETE FROM macro_usage WHERE workspace_id = $1 AND macro_name = ANY($2) AND consumer_path != $3", + &w_id, + &names[..], + &ns.path + ) + .execute(&mut *tx) + .await?; + let names_set: std::collections::HashSet = names.into_iter().collect(); + let members = sqlx::query!( + r#"SELECT DISTINCT ON (path) path AS "path!", content AS "content!" + FROM script + WHERE workspace_id = $1 + AND language = 'duckdb'::script_lang + AND archived = false + AND deleted = false + AND path != $2 + ORDER BY path, created_at DESC"#, + &w_id, + &ns.path + ) + .fetch_all(&mut *tx) + .await?; + for member in members { + let mut called: Vec = + windmill_parser::duckdb_macros::detect_macro_calls(&member.content, &names_set) + .into_iter() + .collect(); + called.sort(); + for name in called { + sqlx::query!( + "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING", + &w_id, + &member.path, + &name + ) + .execute(&mut *tx) + .await?; + } + } + } + let p_path_opt = parent_hashes_and_perms.as_ref().map(|x| x.p_path.clone()); if let Some(ref p_path) = p_path_opt { if !skip_draft_deletion { @@ -2745,6 +3017,7 @@ async fn archive_script_by_path( // anything. Wipe declared `// on ...` edges (asset-event subscribers // look these up). clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + clear_macro_registry(&mut *tx, &w_id, path).await?; audit_log( &mut *tx, @@ -2825,6 +3098,7 @@ async fn archive_script_by_hash( // Pipeline event hygiene: archived scripts must not be triggered by // anything. Wipe declared `// on ...` edges. clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; + clear_macro_registry(&mut *tx, &w_id, &script.path).await?; audit_log( &mut *tx, @@ -2889,6 +3163,7 @@ async fn delete_script_by_hash( // anything. Wipe declared `// on ...` edges. Idempotent — safe even if // the script was never a pipeline member. clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; + clear_macro_registry(&mut *tx, &w_id, &script.path).await?; audit_log( &mut *tx, @@ -3005,6 +3280,7 @@ async fn delete_script_by_path( // anything. Wipe declared `// on ...` edges. Idempotent — safe even if // the script was never a pipeline member. clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + clear_macro_registry(&mut *tx, &w_id, path).await?; if !query.keep_captures.unwrap_or(false) { sqlx::query!( @@ -3167,6 +3443,10 @@ async fn delete_scripts_bulk( .execute(&mut *tx) .await?; + for path in &deleted_paths { + clear_macro_registry(&mut *tx, &w_id, path).await?; + } + audit_log( &mut *tx, &authed, @@ -3217,6 +3497,39 @@ async fn delete_scripts_bulk( Ok(Json(deleted_paths)) } +/// Deployed-macro hygiene for an archived/deleted script: it must neither +/// provide macros to the workspace registry nor keep stale call-site edges. +/// Usage edges pointing at the script's own macros go too (names are +/// workspace-unique, so those edges can only reference this provider). +async fn clear_macro_registry(db: &mut sqlx::PgConnection, w_id: &str, path: &str) -> Result<()> { + sqlx::query!( + "DELETE FROM macro_usage WHERE workspace_id = $1 AND (consumer_path = $2 \ + OR macro_name IN (SELECT name FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2))", + w_id, + path + ) + .execute(&mut *db) + .await?; + let deleted = sqlx::query!( + "DELETE FROM macro_definition WHERE workspace_id = $1 AND provider_path = $2", + w_id, + path + ) + .execute(&mut *db) + .await? + .rows_affected(); + // Evict workers' per-workspace registry cache (transactional emit). + if deleted > 0 { + sqlx::query!( + "INSERT INTO notify_event (channel, payload) VALUES ('notify_macro_registry_change', $1)", + w_id + ) + .execute(&mut *db) + .await?; + } + Ok(()) +} + /// Validates that script debouncing configuration is supported by all workers /// Returns an error if debouncing is configured but workers are behind required version async fn guard_script_from_debounce_data(ns: &NewScript) -> Result<()> { diff --git a/backend/windmill-api-settings/Cargo.toml b/backend/windmill-api-settings/Cargo.toml index 5c4f429d9e..bc5078bce4 100644 --- a/backend/windmill-api-settings/Cargo.toml +++ b/backend/windmill-api-settings/Cargo.toml @@ -10,7 +10,7 @@ path = "src/lib.rs" [features] default = [] -enterprise = [] +enterprise = ["license"] private = ["windmill-common/private"] parquet = ["windmill-common/parquet", "windmill-object-store/parquet"] license = ["dep:rsa"] diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a7e1322ad9..0a58f1f4b3 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -406,7 +406,11 @@ async fn validate_object_storage_test(settings: &ObjectSettings) -> error::Resul ) } ObjectSettings::Gcs(gcs) => { - if gcs.service_account_key.is_empty() { + // Mirror `build_gcs_client`'s blank-key check (shared predicate): a blank/`{}` key falls + // back to the instance's ambient credentials there, so it must be rejected here too — + // otherwise an untrusted caller could probe with the server's identity (the very + // SSRF/credential-exfil this function guards against). + if windmill_object_store::gcs_service_account_key_is_blank(&gcs.service_account_key) { return Err(error::Error::NotAuthorized( "Testing GCS storage without a service account key requires a super admin" .to_string(), @@ -865,6 +869,37 @@ async fn run_setting_pre_write_hook( err )) })?; + } else { + // Disabling is only allowed before any instance-wide username has been + // assigned. Once usernames exist they are globally unique and are baked + // into stored `u/` identities (schedules, triggers, drafts, + // and non-member superadmin ownership). Disabling would drop back to + // workspace-local username uniqueness, letting a member reuse an + // existing instance username and silently take over those identities — + // so the setting is effectively one-way once derivation has taken + // effect. Re-saving `false` on an already-disabled instance is a no-op + // and stays allowed (guarded by the current-value check). + let currently_enabled = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + AUTOMATE_USERNAME_CREATION_SETTING + ) + .fetch_optional(db) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(true); + if currently_enabled { + let usernames_exist = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM password WHERE username IS NOT NULL)" + ) + .fetch_one(db) + .await? + .unwrap_or(false); + if usernames_exist { + return Err(error::Error::BadRequest( + "automate_username_creation cannot be disabled once instance-wide usernames have been assigned: existing u/ identities (schedules, triggers, drafts, superadmin ownership) rely on those usernames staying stable and globally unique.".to_string(), + )); + } + } } } CRITICAL_ALERT_MUTE_UI_SETTING => { @@ -2187,6 +2222,26 @@ mod object_storage_test_hardening { ); } + #[tokio::test] + async fn rejects_gcs_blank_service_account_key() { + // A blank key makes build_gcs_client fall back to the instance's ambient credentials, so an + // untrusted caller must not be allowed to test with it. The `serviceAccountKey` field is + // serialized via serde's `as_string` (`to_string` of the JSON value), so the settings UI's + // "no key" empty object arrives as `"{}"` and a null as `"null"` — both must be rejected. + for key in [serde_json::json!({}), serde_json::json!(null)] { + let settings: ObjectSettings = serde_json::from_value(serde_json::json!({ + "type": "Gcs", + "bucket": "b", + "serviceAccountKey": key + })) + .unwrap(); + assert!( + validate_object_storage_test(&settings).await.is_err(), + "blank key {key:?} should be rejected" + ); + } + } + fn ip(s: &str) -> IpAddr { s.parse().unwrap() } diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index 2963b91a50..3fcebdc083 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -240,6 +240,11 @@ pub struct UserInfo { pub folders_owners: Vec, pub name: Option, pub is_service_account: bool, + // True when this row is a superadmin viewing a workspace they are not a + // member of (so `is_admin`/`role` reflect the superadmin fallback, not an + // actual membership). Always false for real member rows. + #[serde(default)] + pub non_member: bool, } #[derive(FromRow, Serialize)] @@ -699,13 +704,17 @@ async fn whoami( ) -> JsonResult { let ApiAuthed { username, email, is_admin, groups, folders, .. } = authed; let user = get_user(&w_id, &username, &db).await?; - if let Some(user) = user { + // Only treat the row as "this user is a member" when its email matches; the + // derived username is instance-unique so a match on a different email should + // never happen, but guard against it so a non-member superadmin is never + // shown another member's identity/role. + if let Some(user) = user.filter(|u| u.email == email) { Ok(Json(user)) } else { Ok(Json(UserInfo { workspace_id: w_id, - email: email.clone(), - username: email, + email, + username, name: None, is_admin, is_super_admin: is_admin, @@ -725,6 +734,7 @@ async fn whoami( .filter_map(|x| if x.2 { Some(x.0) } else { None }) .collect(), is_service_account: false, + non_member: true, })) } } @@ -889,6 +899,7 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result, Extension(db): Extension, ) -> Result { + // Members only: this workspace-scoped endpoint has no superadmin/target gate, + // so it must NOT use the `password` superadmin fallback — otherwise any + // workspace-authenticated caller could turn a guessed derived username into a + // non-member superadmin's email. Internal callers that legitimately need the + // fallback (schedule/trigger/draft resolution) use `resolve_username_to_email` + // directly and never return the email to an arbitrary caller. let email = sqlx::query_scalar!( "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", &username, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 8544513d01..17dd1d88c5 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6,7 +6,10 @@ * LICENSE-AGPL for a copy of the license. */ -use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; +use windmill_api_auth::{ + build_scope_path_predicate, check_scopes, require_devops_role, require_is_writer, + require_super_admin, ApiAuthed, +}; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; use windmill_common::usernames::{get_instance_username_or_create_pending, VALID_USERNAME}; @@ -40,9 +43,10 @@ use windmill_common::workspaces::GitRepositorySettings; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; use windmill_common::workspaces::{ - check_user_against_rule, get_datatable_resource_from_db_unchecked, validate_fork_workspace_id, - DataTable, DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, - ProtectionRules, ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, + check_deploy_rules, check_user_against_rule, get_datatable_resource_from_db_unchecked, + validate_dev_workspace_id, validate_fork_workspace_id, validate_workspace_name, DataTable, + DataTableCatalogResourceType, DataTableForkBehavior, ProtectionRuleKind, ProtectionRules, + ProtectionRuleset, RuleCheckResult, WorkspaceGitSyncSettings, DEV_WORKSPACE_LOCK_RULE_NAME, }; use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; @@ -151,6 +155,9 @@ pub fn workspaced_service() -> Router { .route("/leave", post(leave_workspace)) .route("/get_workspace_name", get(get_workspace_name)) .route("/create_fork", post(create_workspace_fork)) + .route("/attach_dev_workspace", post(attach_dev_workspace)) + .route("/detach_dev_workspace", post(detach_dev_workspace)) + .route("/get_dev_workspace", get(get_dev_workspace)) .route("/change_workspace_name", post(change_workspace_name)) .route("/change_workspace_color", post(change_workspace_color)) .route( @@ -192,6 +199,7 @@ pub fn workspaced_service() -> Router { .route("/prune_versions", post(prune_versions)) .route("/list_ws_specific", get(list_ws_specific)) .route("/list_ws_specific_versions", get(list_ws_specific_versions)) + .route("/set_ws_specific", post(set_ws_specific)) } pub fn global_service() -> Router { Router::new() @@ -448,6 +456,20 @@ struct CreateWorkspaceFork { /// forked workspace's datatable config to point to the new database. #[serde(default)] forked_datatables: Vec, + /// Create the fork as a persistent dev workspace: the id is not required to carry the + /// `wm-fork-` prefix, and at most one dev workspace may exist per parent. + #[serde(default)] + is_dev_workspace: bool, + /// When creating a dev workspace, lock the parent ("prod") against direct deployment and/or + /// ad-hoc forking, so edits are funneled through the dev workspace. + #[serde(default)] + lock_prod_deploy: bool, + #[serde(default)] + lock_prod_forking: bool, + /// Copy the parent's members (usr rows + group memberships) into the fork so + /// the team can work in it. Defaults off; the dev-workspace UI defaults it on. + #[serde(default)] + copy_members: bool, } #[derive(Deserialize)] @@ -476,6 +498,7 @@ struct UserWorkspace { pub color: Option, pub operator_settings: Option>, pub parent_workspace_id: Option, + pub is_dev_workspace: bool, pub disabled: bool, } @@ -613,11 +636,13 @@ async fn list_pending_invites( } async fn is_premium( - authed: ApiAuthed, + _authed: ApiAuthed, Extension(_db): Extension, Path(_w_id): Path, ) -> JsonResult { - require_admin(authed.is_admin, &authed.username)?; + // Any workspace member (not just admins) may read whether the workspace is on a paid plan: it's a + // single boolean, and the frontend needs it to decide whether to surface premium-gated affordances + // (e.g. forking) to non-admin developers too. The `_authed` extractor still enforces membership. #[cfg(feature = "cloud")] let premium = windmill_common::workspaces::get_team_plan_status(&_db, &_w_id) .await? @@ -644,6 +669,34 @@ async fn exists_workspace( Ok(Json(exists)) } +/// Whether this workspace already has an active canonical dev workspace. The create-fork UI can't +/// rely on the caller's workspace list to decide this — a dev paired to this prod may exist that the +/// caller isn't a member of — so it asks the server, which sees all children. +#[derive(Serialize)] +struct DevWorkspaceInfo { + id: String, + name: String, +} + +/// This workspace's active canonical dev workspace, if any. The create-fork UI and the dev-workspace +/// settings tab can't rely on the caller's workspace list — a dev paired to this prod may exist that +/// the caller isn't a member of — so they ask the server, which sees all children. Returns its id/name +/// so a prod admin who isn't a dev member can still see the pairing and detach it. +async fn get_dev_workspace( + _authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, +) -> JsonResult> { + let dev = sqlx::query_as!( + DevWorkspaceInfo, + "SELECT id, name FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + &w_id + ) + .fetch_optional(&db) + .await?; + Ok(Json(dev)) +} + async fn list_workspaces( authed: ApiAuthed, Extension(user_db): Extension, @@ -3659,6 +3712,7 @@ async fn user_workspaces( let workspaces = sqlx::query_as!( UserWorkspace, "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, workspace.parent_workspace_id, + workspace.is_dev_workspace, CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings, usr.disabled FROM workspace @@ -3794,6 +3848,24 @@ lazy_static::lazy_static! { } }; + // Cloud only: how many forks a premium workspace may have per paid (developer) seat. + pub static ref MAX_FORKS_PER_SEAT: i64 = std::env::var("MAX_FORKS_PER_SEAT") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0) + .unwrap_or(5); + + // How deep a fork chain may nest (root = depth 0, a direct fork = depth 1). A general guardrail + // for all builds, independent of the cloud per-seat cap: deep fork chains are a footgun and no + // real use case needs them. Clamped to [1, 20] so it's always a real limit and can never exceed + // the fork-walk recursion backstop (20) that billing/count resolution uses (a chain deeper than + // the backstop would truncate and mis-resolve its root). + pub static ref MAX_FORK_DEPTH: i64 = std::env::var("MAX_FORK_DEPTH") + .ok() + .and_then(|v| v.parse::().ok()) + .map(|v| v.clamp(1, 20)) + .unwrap_or(5); + } async fn create_workspace_require_superadmin() -> String { @@ -3815,6 +3887,21 @@ async fn _check_nb_of_workspaces(db: &DB) -> Result<()> { return Ok(()); } +async fn _check_nb_of_archived_workspaces(db: &DB) -> Result<()> { + let nb_archived = sqlx::query_scalar!( + "SELECT COUNT(*) FROM workspace WHERE id != 'admins' AND deleted = true", + ) + .fetch_one(db) + .await?; + if nb_archived.unwrap_or(0) >= 1 { + return Err(Error::BadRequest( + "You have reached the maximum number of archived workspaces (1) without an enterprise license. Permanently delete or unarchive the existing archived workspace first" + .to_string(), + )); + } + return Ok(()); +} + async fn create_workspace( authed: ApiAuthed, Extension(db): Extension, @@ -3842,6 +3929,8 @@ async fn create_workspace( } } + validate_workspace_name(&nw.name)?; + let mut tx: Transaction<'_, Postgres> = db.begin().await?; check_w_id_conflict(&mut tx, &nw.id).await?; @@ -4006,6 +4095,7 @@ async fn clone_workspace_data( // Clone CI test references clone_ci_test_references(tx, source_workspace_id, target_workspace_id).await?; + clone_macro_registry(tx, source_workspace_id, target_workspace_id).await?; // Clone flows with new versions clone_flows(tx, source_workspace_id, target_workspace_id).await?; @@ -4436,6 +4526,28 @@ async fn clone_groups( Ok(()) } +/// Copy the source workspace's members (the `usr` rows, carrying each member's role) into the +/// target so a fork/dev can be a shared environment. Idempotent — skips members the target already +/// has. Group memberships are not handled here: the sole caller is the create-fork path, where +/// `clone_groups` already copies the source's full group structure (including `all` membership). +async fn copy_workspace_members( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO usr (workspace_id, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via) + SELECT $1, username, email, is_admin, created_at, operator, disabled, role, is_service_account, added_via + FROM usr WHERE workspace_id = $2 + ON CONFLICT DO NOTHING", + target_workspace_id, + source_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_resource_types( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -4547,6 +4659,35 @@ async fn clone_ci_test_references( Ok(()) } +// DuckDB macro registry + call-site edges are deploy-derived like +// ci_test_reference: without cloning them, a forked consumer script fails at +// run time until every macro library is manually redeployed in the fork. +async fn clone_macro_registry( + tx: &mut Transaction<'_, Postgres>, + source_workspace_id: &str, + target_workspace_id: &str, +) -> Result<()> { + sqlx::query!( + "INSERT INTO macro_definition (workspace_id, name, provider_path, params, body, is_table_macro) + SELECT $2, name, provider_path, params, body, is_table_macro + FROM macro_definition WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( + "INSERT INTO macro_usage (workspace_id, consumer_path, macro_name) + SELECT $2, consumer_path, macro_name + FROM macro_usage WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + async fn clone_flows( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, @@ -4651,9 +4792,18 @@ async fn clone_apps( .await?; let mut app_id_mapping: HashMap = HashMap::new(); + // Only a raw app's current (last) version has a bundle worth carrying into the fork: bundles exist + // only for raw apps, and older versions aren't viewable/runnable (the bundle secret is only ever + // minted for `versions.last()`). Copying a bundle for every version of every app is what makes + // forking a workspace with many app versions hang — a serial S3 round-trip per version, held inside + // the fork transaction. Collect each app's current version here; intersect with raw versions below. + let mut latest_version_ids: HashSet = HashSet::new(); // Clone apps with new IDs for app in apps { + if let Some(¤t_version) = app.versions.last() { + latest_version_ids.insert(current_version); + } let new_app_id = sqlx::query_scalar!( "INSERT INTO app (workspace_id, path, summary, policy, versions, extra_perms, custom_path) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -4673,6 +4823,7 @@ async fn clone_apps( } let mut version_id_mapping: HashMap = HashMap::new(); + let mut raw_version_ids: HashSet = HashSet::new(); { // Clone app versions @@ -4688,6 +4839,9 @@ async fn clone_apps( for version in app_versions { if let Some(&new_app_id) = app_id_mapping.get(&version.app_id) { + if version.raw_app { + raw_version_ids.insert(version.id); + } let new_version_id = sqlx::query_scalar!( "INSERT INTO app_version (app_id, value, created_by, created_at, raw_app) VALUES ($1, $2, $3, $4, $5) RETURNING id", @@ -4705,9 +4859,16 @@ async fn clone_apps( } } - // Clone app bundles for raw apps - if !version_id_mapping.is_empty() { - let old_ids: Vec = version_id_mapping.keys().copied().collect(); + // The bundles worth cloning: each raw app's current version (latest ∩ raw). Everything else either + // has no bundle (low-code apps) or an unreachable one (older versions), so we don't touch S3 for it. + let bundle_version_ids: HashSet = latest_version_ids + .intersection(&raw_version_ids) + .copied() + .collect(); + + // Clone app bundles — only the current version of each raw app (see bundle_version_ids). + if !bundle_version_ids.is_empty() { + let old_ids: Vec = bundle_version_ids.iter().copied().collect(); let bundles = sqlx::query!( "SELECT app_version_id, file_type, data FROM app_bundles WHERE app_version_id = ANY($1) AND w_id = $2", @@ -4743,40 +4904,31 @@ async fn clone_apps( let object_store = windmill_object_store::get_object_store().await; if let Some(os) = object_store { for (&old_version_id, &new_version_id) in &version_id_mapping { + if !bundle_version_ids.contains(&old_version_id) { + continue; + } for file_type in &["js", "css"] { if cloned_from_db.contains(&(old_version_id, file_type.to_string())) { continue; } - let src_path = format!( - "/app_bundles/{}/{}.{}", - source_workspace_id, old_version_id, file_type - ); - let get_result = os - .get(&windmill_object_store::object_store_reexports::Path::from( - src_path, - )) - .await; - match get_result { - Ok(result) => { - let data = result.bytes().await.map_err( - windmill_object_store::object_store_error_to_error, - )?; - let dst_path = format!( - "/app_bundles/{}/{}.{}", - target_workspace_id, new_version_id, file_type - ); - os.put( - &windmill_object_store::object_store_reexports::Path::from( - dst_path.clone(), - ), - data.into(), - ) - .await - .map_err( - windmill_object_store::object_store_error_to_error, - )?; + // Prefer a server-side copy (no bytes through the backend). A missing source — + // e.g. a raw app with a js bundle but no css — surfaces as NotFound and is + // skipped. Not every object-store provider supports server-side copy, so fall + // back to download+upload on any other error. + let src_path = + windmill_object_store::object_store_reexports::Path::from(format!( + "/app_bundles/{}/{}.{}", + source_workspace_id, old_version_id, file_type + )); + let dst_path = + windmill_object_store::object_store_reexports::Path::from(format!( + "/app_bundles/{}/{}.{}", + target_workspace_id, new_version_id, file_type + )); + match os.copy(&src_path, &dst_path).await { + Ok(()) => { tracing::info!( - "Cloned app bundle from S3: {}.{} -> {}.{}", + "Cloned app bundle in object store: {}.{} -> {}.{}", old_version_id, file_type, new_version_id, @@ -4784,12 +4936,39 @@ async fn clone_apps( ); } Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) => { - // No bundle in S3 for this version/type, skip + // No bundle in the object store for this version/type, skip } - Err(e) => { - return Err( - windmill_object_store::object_store_error_to_error(e), + Err(copy_err) => { + // Provider may not support server-side copy — fall back to get+put. + tracing::warn!( + "object store copy failed ({copy_err:#}), falling back to get+put for app bundle {}.{}", + old_version_id, file_type ); + match os.get(&src_path).await { + Ok(result) => { + let data = result.bytes().await.map_err( + windmill_object_store::object_store_error_to_error, + )?; + os.put(&dst_path, data.into()).await.map_err( + windmill_object_store::object_store_error_to_error, + )?; + tracing::info!( + "Cloned app bundle via get+put fallback: {}.{} -> {}.{}", + old_version_id, + file_type, + new_version_id, + file_type + ); + } + Err(windmill_object_store::object_store_reexports::ObjectStoreError::NotFound { .. }) => { + // No bundle for this version/type, skip + } + Err(e) => { + return Err( + windmill_object_store::object_store_error_to_error(e), + ); + } + } } } } @@ -4963,10 +5142,12 @@ async fn create_workspace_fork_branch( Path(w_id): Path, Json(nw): Json, ) -> JsonResult> { + // Pre-check the fork guards before creating any git branch, so we don't leave orphaned branches + // behind when the follow-up create_workspace_fork would be rejected anyway. + enforce_fork_depth(&db, &w_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &w_id).await?; } if *DISABLE_WORKSPACE_FORK { @@ -4985,7 +5166,27 @@ async fn create_workspace_fork_branch( return Err(Error::PermissionDenied(msg)); } - validate_fork_workspace_id(&nw.id)?; + // Two-phase create for git-synced workspaces: this endpoint only creates the git branch(es) and + // validates up front; it does NOT create the workspace row. The caller follows up with + // `create_workspace_fork`, which inserts the row and applies the dev designation + prod lock + + // member copy. So the dev/lock/copy_members fields here are validated only — they are acted on by + // that second call. Validating early lets a bad request fail before any branch is created. + if nw.is_dev_workspace { + validate_dev_workspace_id(&nw.id)?; + ensure_dev_parent_is_root(&db, &w_id).await?; + // Reject before creating any git branch if the parent already has a dev workspace, + // otherwise the deferred branch-creation job leaves a dangling branch on the synced repos. + ensure_no_existing_dev_workspace(&db, &w_id).await?; + // Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking the + // parent mutates its protection rules), so require admin of the parent regardless of the lock + // flags — mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker + // could claim the dev slot. Enforced in this first phase too so the request fails before any + // git branch is created rather than leaving dangling branches. + require_admin(authed.is_admin, &authed.username)?; + } else { + validate_fork_workspace_id(&nw.id)?; + } + validate_workspace_name(&nw.name)?; // Fail before creating any git branch so a name conflict doesn't leave a // dangling branch on the synced repos. @@ -5118,19 +5319,91 @@ async fn apply_forked_datatable( Ok(()) } +/// Cloud: require the fork/dev's root (billing) workspace be premium; returns the resolved root id. +#[cfg(feature = "cloud")] +async fn require_cloud_fork_premium(db: &DB, parent_workspace_id: &str) -> Result { + let root = + windmill_common::workspaces::get_billing_workspace_id(db, parent_workspace_id).await?; + if !windmill_common::workspaces::get_team_plan_status(db, &root) + .await? + .premium + { + return Err(Error::BadRequest( + "Creating a fork or dev workspace on the cloud requires a paid team plan. Upgrade the workspace first.".to_string(), + )); + } + Ok(root) +} + +/// Cloud: reject if adding `incoming` fork/dev workspaces would push `root`'s family over its per-seat +/// allotment. `incoming` is the number of workspaces the operation adds to the family — 1 for a plain +/// create, but `1 + candidate_subtree` for an attach whose candidate already has child forks. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_count(db: &DB, root: &str, incoming: i64) -> Result<()> { + let seats = windmill_common::workspaces::count_paid_seats(db, root).await?; + let per_seat = *MAX_FORKS_PER_SEAT; + // Any premium workspace has at least one paid seat, so floor the seat count at 1. + let allowed = seats.max(1) * per_seat; + + let existing = windmill_common::workspaces::count_workspace_forks(db, root).await?; + let projected = existing + incoming; + if projected > allowed { + return Err(Error::BadRequest(format!( + "Fork limit reached: this would bring the workspace family to {projected} fork(s), over the cap of {allowed} ({seats} paid seat(s) × {per_seat} per seat). Delete a fork or add seats." + ))); + } + + Ok(()) +} + +/// Cloud-only guard for creating a fork/dev workspace. Forks piggyback on the parent's plan (a fork +/// inherits the root's premium and meters its usage into the root's bill), so forking is limited to +/// premium workspaces and capped at `MAX_FORKS_PER_SEAT` per paid (developer) seat of the root. +#[cfg(feature = "cloud")] +async fn enforce_cloud_fork_cap(db: &DB, parent_workspace_id: &str) -> Result<()> { + let root = require_cloud_fork_premium(db, parent_workspace_id).await?; + enforce_cloud_fork_count(db, &root, 1).await +} + +/// General guardrail (all builds): reject creating a fork/dev under `parent` when it would nest deeper +/// than `MAX_FORK_DEPTH`. `added_subtree_height` is the height of the subtree grafted below the new +/// node — 0 for a plain fork, or the candidate's own subtree height for an attach. +async fn enforce_fork_depth( + db: &DB, + parent_workspace_id: &str, + added_subtree_height: i64, +) -> Result<()> { + let parent_depth = + windmill_common::workspaces::fork_chain_depth(db, parent_workspace_id).await?; + // The new node sits one level below the parent; its deepest descendant adds the grafted height. + let resulting_depth = parent_depth + 1 + added_subtree_height; + if resulting_depth > *MAX_FORK_DEPTH { + return Err(Error::BadRequest(format!( + "Fork depth limit reached: forks can be nested at most {} level(s) deep, but this would create a fork at depth {}. Fork from a workspace closer to the root instead.", + *MAX_FORK_DEPTH, resulting_depth + ))); + } + Ok(()) +} + async fn create_workspace_fork( authed: ApiAuthed, Extension(db): Extension, Path(parent_workspace_id): Path, Json(nw): Json, ) -> Result { + enforce_fork_depth(&db, &parent_workspace_id, 0).await?; + #[cfg(feature = "cloud")] if *CLOUD_HOSTED { - return Err(Error::BadRequest(format!( - "Forking workspaces is not available on app.windmill.dev" - ))); + enforce_cloud_fork_cap(&db, &parent_workspace_id).await?; } - validate_fork_workspace_id(&nw.id)?; + if nw.is_dev_workspace { + validate_dev_workspace_id(&nw.id)?; + } else { + validate_fork_workspace_id(&nw.id)?; + } + validate_workspace_name(&nw.name)?; // Check the id conflict before the CE workspace-count limit so that // re-using a taken (possibly archived) fork id reports the actual // conflict instead of a misleading "maximum number of workspaces" error. @@ -5156,18 +5429,29 @@ async fn create_workspace_fork( return Err(Error::PermissionDenied(msg)); } + if nw.is_dev_workspace { + ensure_dev_parent_is_root(&db, &parent_workspace_id).await?; + // Creating the canonical dev consumes the parent's one-dev-per-prod slot (and locking prod + // mutates its protection rules), so require admin of the parent regardless of the lock flags — + // mirrors attach/detach, which are prod-admin gated. Without this a non-admin forker could + // claim the dev slot (and, without member copy, prod admins might not even see it to detach). + require_admin(authed.is_admin, &authed.username)?; + ensure_no_existing_dev_workspace(&db, &parent_workspace_id).await?; + } + let mut tx: Transaction<'_, Postgres> = db.begin().await?; let forked_id = nw.id; sqlx::query!( "INSERT INTO workspace - (id, name, owner, parent_workspace_id) - VALUES ($1, $2, $3, $4)", + (id, name, owner, parent_workspace_id, is_dev_workspace) + VALUES ($1, $2, $3, $4, $5)", forked_id, nw.name, authed.email, parent_workspace_id, + nw.is_dev_workspace, ) .execute(&mut *tx) .await?; @@ -5182,11 +5466,24 @@ async fn create_workspace_fork( .execute(&mut *tx) .await?; + // Optionally bring the parent's members into the fork (a shared dev env). Dev-only: it's part of the + // dev-workspace feature (and the frontend only offers it there), so the backend enforces it rather + // than trusting the client — copying the parent's whole team into an ordinary throwaway fork isn't + // intended. Dev creation is already admin-gated, so this is transitively admin-only too. Done before + // the explicit creator insert below so the creator (a parent member) is copied with full metadata + // (operator/role/is_service_account/added_via), not the bare row the insert alone would leave. + if nw.copy_members && nw.is_dev_workspace { + copy_workspace_members(&mut tx, &parent_workspace_id, &forked_id).await?; + } + + // Ensure the creator is a member of the fork even without copy_members (or if they aren't a parent + // member). No-op when copy_members already brought their full row. sqlx::query!( "INSERT INTO usr (workspace_id, email, username, is_admin) SELECT $1, email, username, is_admin FROM usr WHERE workspace_id = $3 AND email = $2 + ON CONFLICT DO NOTHING ", forked_id, authed.email, @@ -5209,6 +5506,18 @@ async fn create_workspace_fork( apply_forked_datatable(&db, &mut tx, &parent_workspace_id, &forked_id, fdt).await?; } + // Lock the parent ("prod") so edits are funneled through this dev workspace. + let locked_prod = nw.is_dev_workspace && (nw.lock_prod_deploy || nw.lock_prod_forking); + if locked_prod { + lock_prod_workspace( + &mut tx, + &parent_workspace_id, + nw.lock_prod_deploy, + nw.lock_prod_forking, + ) + .await?; + } + audit_log( &mut *tx, &authed, @@ -5221,9 +5530,266 @@ async fn create_workspace_fork( .await?; tx.commit().await?; + if locked_prod { + windmill_common::workspaces::invalidate_protection_rules_cache(&parent_workspace_id); + } + Ok(format!("Created forked workspace {}", &forked_id)) } +#[derive(Deserialize)] +struct AttachDevWorkspace { + dev_workspace_id: String, + #[serde(default)] + lock_prod_deploy: bool, + #[serde(default)] + lock_prod_forking: bool, +} + +#[derive(Deserialize)] +struct DetachDevWorkspace { + dev_workspace_id: String, +} + +/// Pair an existing standalone workspace to this workspace ("prod") as its dev workspace, without +/// cloning any data (both already exist). Sets the dev's parent + deploy_to to prod and, optionally, +/// locks prod against direct deployment. +async fn attach_dev_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Path(prod_w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + // Attaching grafts the candidate (and its own fork subtree) under prod, so enforce the general + // depth limit on the deepest resulting node. + let candidate_height = + windmill_common::workspaces::fork_subtree_height(&db, &req.dev_workspace_id).await?; + enforce_fork_depth(&db, &prod_w_id, candidate_height).await?; + + // Attaching reparents a workspace under prod (one dev per prod, admin-gated) and it then draws + // prod's plan, so hold it to the same premium requirement as creating a fork. Only enforce the + // per-seat count when the attach actually adds a new workspace to the family: re-designating a + // workspace already under this root as its dev doesn't increase the descendant count. + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + let root = require_cloud_fork_premium(&db, &prod_w_id).await?; + // Only count against the cap when this attach adds workspaces to the family (candidate not + // already under this root). The candidate may itself have child forks, so reserve slots for its + // whole incoming subtree (the candidate + its descendants), not just one. + if windmill_common::workspaces::get_billing_workspace_id(&db, &req.dev_workspace_id).await? + != root + { + let incoming = + 1 + windmill_common::workspaces::count_workspace_forks(&db, &req.dev_workspace_id) + .await?; + enforce_cloud_fork_count(&db, &root, incoming).await?; + } + } + + let dev_w_id = req.dev_workspace_id; + if dev_w_id == prod_w_id { + return Err(Error::BadRequest( + "A workspace cannot be its own dev workspace".to_string(), + )); + } + + // The id is interpolated into a `wm-fork//` branch name like any fork. + validate_dev_workspace_id(&dev_w_id)?; + + let dev = sqlx::query!( + r#"SELECT parent_workspace_id, deleted FROM workspace WHERE id = $1"#, + &dev_w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Workspace {} not found", dev_w_id)))?; + + if dev.deleted { + return Err(Error::BadRequest(format!( + "Workspace {} is archived", + dev_w_id + ))); + } + // A candidate that already belongs to a DIFFERENT parent can't be attached. A candidate already + // parented to this prod is allowed: it's the recovery path after renaming a dev workspace (the + // rename keeps the parent but drops the dev flag), and re-designating an existing fork of this + // prod as its dev. + if dev + .parent_workspace_id + .as_deref() + .is_some_and(|p| p != prod_w_id) + { + return Err(Error::BadRequest(format!( + "Workspace {} is already a fork or dev workspace of another workspace", + dev_w_id + ))); + } + // The candidate can't itself be a prod with its own dev workspace (no nested dev chains). + ensure_no_existing_dev_workspace(&db, &dev_w_id).await?; + + // Prod must be a root workspace, otherwise attaching could form a parent<->child cycle (e.g. + // attaching A as the dev of B when B is already the dev of A), which breaks hierarchy traversal. + let prod_has_parent = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#, + &prod_w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| Error::NotFound(format!("Workspace {} not found", prod_w_id)))?; + if prod_has_parent { + return Err(Error::BadRequest(format!( + "Workspace {} is itself a fork or dev workspace and cannot be a prod workspace", + prod_w_id + ))); + } + + // The caller must be admin of the dev workspace too (or a superadmin). + let is_admin_of_dev = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &dev_w_id, + &authed.email + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if !is_admin_of_dev && !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Attaching workspace '{dev_w_id}' as a dev requires being an admin of it (or a superadmin)" + ))); + } + + ensure_no_existing_dev_workspace(&db, &prod_w_id).await?; + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace SET parent_workspace_id = $1, is_dev_workspace = true WHERE id = $2", + &prod_w_id, + &dev_w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "UPDATE workspace_settings SET deploy_to = $1 WHERE workspace_id = $2", + &prod_w_id, + &dev_w_id + ) + .execute(&mut *tx) + .await?; + + if req.lock_prod_deploy || req.lock_prod_forking { + lock_prod_workspace( + &mut tx, + &prod_w_id, + req.lock_prod_deploy, + req.lock_prod_forking, + ) + .await?; + } + + audit_log( + &mut *tx, + &authed, + "workspaces.attach_dev_workspace", + ActionKind::Update, + &prod_w_id, + Some(&dev_w_id), + None, + ) + .await?; + tx.commit().await?; + + // The dev workspace's parent just changed (none -> prod); drop its cached fork->parent mapping + // so per-workspace job tags route to the prod family immediately rather than after the TTL. + windmill_queue::tags::invalidate_fork_parent_cache(&dev_w_id); + // Same reparent invalidates the billing-workspace mapping so its usage meters to prod at once. The + // candidate can bring its own fork subtree, whose descendants had resolved their (now-stale) root + // to the candidate's old family; invalidate them too so they meter to prod without waiting out the + // 60s TTL. Their immediate fork->parent links don't move, so the tag-routing cache needs no change. + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(&dev_w_id); + for id in windmill_common::workspaces::list_fork_descendants(&db, &dev_w_id).await? { + windmill_common::workspaces::invalidate_billing_workspace_cache(&id); + } + } + + if req.lock_prod_deploy || req.lock_prod_forking { + windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); + } + + Ok(format!( + "Attached {} as dev workspace of {}", + dev_w_id, prod_w_id + )) +} + +/// Reverse [`attach_dev_workspace`] / clear the dev designation: unset the dev flag and remove the +/// prod lock. The workspace keeps its `parent_workspace_id` (it remains an ordinary fork). +async fn detach_dev_workspace( + authed: ApiAuthed, + Extension(db): Extension, + Path(prod_w_id): Path, + Json(req): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let dev_w_id = req.dev_workspace_id; + let is_dev_of_prod = sqlx::query_scalar!( + r#"SELECT EXISTS( + SELECT 1 FROM workspace + WHERE id = $1 AND parent_workspace_id = $2 AND is_dev_workspace + )"#, + &dev_w_id, + &prod_w_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + if !is_dev_of_prod { + return Err(Error::BadRequest(format!( + "{} is not the dev workspace of {}", + dev_w_id, prod_w_id + ))); + } + + let mut tx = db.begin().await?; + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + &dev_w_id + ) + .execute(&mut *tx) + .await?; + // Only one dev per prod, so detaching it means prod no longer has a dev: drop the lock rule. + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2", + &prod_w_id, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "workspaces.detach_dev_workspace", + ActionKind::Update, + &prod_w_id, + Some(&dev_w_id), + None, + ) + .await?; + tx.commit().await?; + + windmill_common::workspaces::invalidate_protection_rules_cache(&prod_w_id); + + Ok(format!( + "Detached dev workspace {} from {}", + dev_w_id, prod_w_id + )) +} + async fn edit_workspace( authed: ApiAuthed, Extension(db): Extension, @@ -5263,6 +5829,10 @@ pub(crate) async fn archive_workspace_impl( db: &DB, w_id: &str, username: &str, + // When archiving a dev workspace, its parent prod. The pairing teardown (clear is_dev + drop the + // prod's lock) is folded into the same transaction as `deleted = true` so it's atomic with the + // archive — a later failure can't strand a half-archived dev that's still flagged/locked. + dev_lock_parent: Option<&str>, ) -> Result<(usize, usize, usize)> { // Step 1: Disable all schedules and clear their queued jobs let mut tx = db.begin().await?; @@ -5304,6 +5874,30 @@ pub(crate) async fn archive_workspace_impl( .execute(&mut *tx) .await?; + if let Some(prod) = dev_lock_parent { + // Dissolve the dev pairing atomically with the archive: clear the canonical-dev flag (so the + // archived row no longer occupies the parent's one-dev slot), and drop the prod's lock unless a + // replacement dev already holds it (NOT EXISTS sees the just-cleared flag within this tx, so the + // row being archived doesn't count). + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2 + AND NOT EXISTS ( + SELECT 1 FROM workspace + WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false + )", + prod, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; // Step 2: Get all remaining queued jobs for this workspace (non-schedule jobs) @@ -5347,8 +5941,44 @@ async fn archive_workspace( ) -> Result { require_admin(authed.is_admin, &authed.username)?; + // CE caps the number of archived (soft-deleted) workspaces so archiving can't be used to + // stockpile hidden workspaces. Enforced here so a second archive is refused up front. + #[cfg(not(feature = "enterprise"))] + _check_nb_of_archived_workspaces(&db).await?; + + // If this is an attached dev workspace, archiving it leaves the prod with no active dev (the + // unique index and user_workspaces both ignore deleted=true), so clear the prod's + // dev_workspace_lock too. Gate it on prod-admin since it removes prod's protection rule (mirrors + // detach/delete) — a dev-admin who isn't a prod-admin must not be able to unlock prod this way. + let dev_lock_parent: Option = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&db) + .await? + .flatten(); + if let Some(ref prod) = dev_lock_parent { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + prod, + &authed.email + ) + .fetch_optional(&db) + .await? + .unwrap_or(false); + if !is_prod_admin + && !windmill_common::auth::is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied(format!( + "Archiving dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } + } + + // The dev pairing teardown (clear is_dev + drop the prod lock) runs inside archive_workspace_impl's + // transaction, atomically with `deleted = true`. let (schedules_count, canceled_count, deleted_tokens_count) = - archive_workspace_impl(&db, &w_id, &authed.username).await?; + archive_workspace_impl(&db, &w_id, &authed.username, dev_lock_parent.as_deref()).await?; // Audit log let mut tx = db.begin().await?; @@ -5383,6 +6013,10 @@ async fn archive_workspace( .await?; tx.commit().await?; + if let Some(prod) = dev_lock_parent { + windmill_common::workspaces::invalidate_protection_rules_cache(&prod); + } + Ok(format!( "Archived workspace {}, disabled {} schedules, canceled {} jobs and deleted {} tokens", &w_id, schedules_count, canceled_count, deleted_tokens_count @@ -5424,6 +6058,13 @@ async fn unarchive_workspace( authed: ApiAuthed, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + + // Unarchiving re-activates a soft-deleted workspace, so it must respect the + // same CE workspace-count cap as creating one. The archived workspace is + // deleted = true and thus excluded from the count until it is restored. + #[cfg(not(feature = "enterprise"))] + _check_nb_of_workspaces(&db).await?; + let mut tx = db.begin().await?; sqlx::query!("UPDATE workspace SET deleted = false WHERE id = $1", &w_id) .execute(&mut *tx) @@ -6020,6 +6661,17 @@ async fn change_workspace_color( } async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { + // On cloud, a fork's executions meter against its billing root, so report the root's usage here too; + // otherwise the free-execs indicator would show the fork's own (often 0) count while enforcement + // applies the root's shared quota. Gated on `*CLOUD_HOSTED` (not just the `cloud` feature, which is + // compiled into all EE builds): self-hosted doesn't meter usage this way. Off-fork it resolves to + // `w_id` itself anyway. + #[cfg(feature = "cloud")] + let w_id = if *CLOUD_HOSTED { + windmill_common::workspaces::get_billing_workspace_id(&db, &w_id).await? + } else { + w_id + }; let usage = sqlx::query_scalar!( " SELECT usage.usage FROM usage @@ -6224,6 +6876,119 @@ async fn list_protection_rules( )) } +/// Insert or replace a protection ruleset within an existing transaction. Unlike the +/// `create_protection_rule` handler (which rejects an existing name), this upserts, so it is safe to +/// call programmatically when designating a dev/prod pair. Callers MUST invalidate the +/// protection-rules cache (`invalidate_protection_rules_cache`) after the transaction commits. +async fn upsert_protection_rule( + tx: &mut Transaction<'_, Postgres>, + w_id: &str, + name: &str, + rules: ProtectionRules, + bypass_groups: &[String], + bypass_users: &[String], +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO workspace_protection_rule (workspace_id, name, rules, bypass_groups, bypass_users) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (workspace_id, name) + DO UPDATE SET rules = EXCLUDED.rules, + bypass_groups = EXCLUDED.bypass_groups, + bypass_users = EXCLUDED.bypass_users + "#, + w_id, + name, + rules.bits(), + bypass_groups, + bypass_users, + ) + .execute(&mut **tx) + .await?; + Ok(()) +} + +/// Lock a prod workspace by applying the reserved dev-workspace lock rule with the selected +/// restrictions (block direct deployment and/or ad-hoc forking). Non-admins are then funneled +/// through the one dev workspace; admins bypass the rules (their existing escape hatch). +async fn lock_prod_workspace( + tx: &mut Transaction<'_, Postgres>, + prod_w_id: &str, + block_deploy: bool, + block_forking: bool, +) -> Result<()> { + let mut rules = Vec::new(); + if block_deploy { + rules.push(ProtectionRuleKind::DisableDirectDeployment); + } + if block_forking { + rules.push(ProtectionRuleKind::DisableWorkspaceForking); + } + if rules.is_empty() { + return Ok(()); + } + upsert_protection_rule( + tx, + prod_w_id, + DEV_WORKSPACE_LOCK_RULE_NAME, + ProtectionRules::from(&rules), + &[], + &[], + ) + .await +} + +/// Error out if `parent_w_id` already has an active (non-archived) dev workspace. Mirrors the +/// partial unique index `workspace_canonical_dev_idx` with a friendly message. +async fn ensure_no_existing_dev_workspace(db: &DB, parent_w_id: &str) -> Result<()> { + let existing = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace AND deleted = false", + parent_w_id + ) + .fetch_optional(db) + .await?; + if let Some(existing) = existing { + return Err(Error::BadRequest(format!( + "Workspace '{}' already has a dev workspace ('{}'). Detach it before creating another.", + parent_w_id, existing + ))); + } + Ok(()) +} + +/// A dev workspace pairs with a root prod workspace; nesting dev workspaces (a dev of a dev) isn't +/// supported and would muddle the prod<->dev relationship. +async fn ensure_dev_parent_is_root(db: &DB, parent_w_id: &str) -> Result<()> { + let parent_is_fork = sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "is_fork!" FROM workspace WHERE id = $1"#, + parent_w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if parent_is_fork { + return Err(Error::BadRequest(format!( + "Cannot create a dev workspace of '{}' because it is itself a fork or dev workspace.", + parent_w_id + ))); + } + Ok(()) +} + +/// `dev_workspace_lock` is owned by the dev-workspace feature (attach/detach/archive/delete create and +/// remove it by name). Reserve it from the public protection-rule API so a user-managed rule can't +/// collide: otherwise the feature's name-based cleanup would clobber the user's rule, or a manual edit +/// could weaken the feature's lock. +fn reject_reserved_rule_name(name: &str) -> Result<()> { + if name == DEV_WORKSPACE_LOCK_RULE_NAME { + return Err(Error::BadRequest(format!( + "'{}' is a reserved protection-rule name managed by the dev workspace feature", + DEV_WORKSPACE_LOCK_RULE_NAME + ))); + } + Ok(()) +} + /// Create a new protection rule async fn create_protection_rule( authed: ApiAuthed, @@ -6232,6 +6997,7 @@ async fn create_protection_rule( Json(req): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&req.name)?; let mut tx = db.begin().await?; @@ -6306,6 +7072,7 @@ async fn update_protection_rule( Json(req): Json, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&rule_name)?; let mut tx = db.begin().await?; @@ -6380,6 +7147,7 @@ async fn delete_protection_rule( Path((w_id, rule_name)): Path<(String, String)>, ) -> Result { require_admin(authed.is_admin, &authed.username)?; + reject_reserved_rule_name(&rule_name)?; let mut tx = db.begin().await?; @@ -6437,6 +7205,27 @@ pub struct WorkspaceComparison { pub skipped_comparison: bool, pub diffs: Vec, pub summary: CompareSummary, + /// Items that exist in the diff but were dropped from `diffs` because they + /// are not visible to the caller (excluded from the partial deploy). Split + /// by direction: `hidden_ahead` lives in the fork, `hidden_behind` in the + /// parent. `by_kind`/`total` are always populated (aggregate, no names); + /// `items` (kind+path) is only filled for a caller who is admin of that side + /// — never leak the paths of items the ACL is hiding from a regular user. + pub hidden_ahead: HiddenItemsSummary, + pub hidden_behind: HiddenItemsSummary, +} + +#[derive(Serialize, Default)] +pub struct HiddenItemsSummary { + pub total: usize, + pub by_kind: std::collections::BTreeMap, + pub items: Vec, +} + +#[derive(Serialize)] +pub struct HiddenItem { + pub kind: String, + pub path: String, } #[derive(Serialize, Default)] @@ -6522,13 +7311,27 @@ async fn compare_workspaces( skipped_comparison, diffs: vec![], summary: Default::default(), + hidden_ahead: Default::default(), + hidden_behind: Default::default(), })); } + // Honor ws_specific at read time: a workspace-specific resource/variable keeps its own value per + // environment, so it must never appear in the normal diff (the per-item compare suppresses it, + // but a cached `has_changes=true` row is trusted without re-running that compare, so filter those + // here too). Seeding the initial copy onto a side that lacks it is a separate explicit action + // (the "Create in " button on the Workspace-specific list), not part of the diff. The row + // is left intact, so unpinning resurfaces it without a re-tally. let diff_items = sqlx::query_as!( WorkspaceDiffRow, "SELECT path, kind, ahead, behind, has_changes, exists_in_source, exists_in_fork FROM workspace_diff - WHERE source_workspace_id = $1 AND fork_workspace_id = $2", + WHERE source_workspace_id = $1 AND fork_workspace_id = $2 + AND NOT EXISTS ( + SELECT 1 FROM ws_specific ws + WHERE ws.path = workspace_diff.path + AND ws.item_kind = workspace_diff.kind + AND ws.workspace_id IN (workspace_diff.source_workspace_id, workspace_diff.fork_workspace_id) + )", source_workspace_id, fork_workspace_id, ) @@ -6783,12 +7586,72 @@ async fn compare_workspaces( .map(|s| s.behind) .fold(0, |acc, s| acc + s.try_into().unwrap_or(0)); + // Blast-radius guard for the "changes not visible to your user" warning + // (which hides the deploy button). The flag is a pure visibility guarantee — + // the deploy re-authorizes each item against the target workspace's + // create/update endpoints — so it is safe to force true for a caller who sees + // every item on BOTH sides, for whom any diff the filter dropped is provably a + // stale/phantom row, never a permission gap. + // + // It must be BOTH sides, not per-side: `filter_visible_diffs` keeps a modified + // or conflict row (one that exists in the source AND the fork) only when the + // caller can see it on both sides, so an ahead/conflict diff can be dropped for + // a source-side visibility gap even when the caller is a fork admin. Gating the + // ahead flag on fork-admin alone would then wrongly report "all ahead visible" + // and let the UI deploy from an incomplete comparison. So require admin of the + // source AND the fork (superadmin satisfies both), which guarantees full + // visibility of every item on every side. `fork_authed.is_admin` already folds + // in superadmin; `authed.is_admin` (source side) does not, so OR it in. + let is_super_admin = windmill_common::auth::is_super_admin_email(&db, &authed.email).await?; + let sees_all_items = is_super_admin || (authed.is_admin && fork_authed.is_admin); + let all_ahead_items_visible = all_ahead_items_visible || sees_all_items; + let all_behind_items_visible = all_behind_items_visible || sees_all_items; + + // Items dropped by the visibility filter (in confirmed_diffs but not in the + // returned visible_diffs). Surface what the partial deploy excludes: aggregate + // counts by kind for everyone, but kind+path only to a caller who `sees_all_items` + // (superadmin, or admin of the source AND the fork). For them a dropped item is + // provably a phantom/stale row, not an ACL-hidden secret, so no path leaks — + // fork-admin alone is not enough (a fork-deleted ahead item lives only in the + // parent, whose path a non-parent-admin must not see). + let visible_keys: HashSet<(&str, &str)> = visible_diffs + .iter() + .map(|d| (d.kind.as_str(), d.path.as_str())) + .collect(); + let mut hidden_ahead = HiddenItemsSummary::default(); + let mut hidden_behind = HiddenItemsSummary::default(); + for d in &confirmed_diffs { + if visible_keys.contains(&(d.kind.as_str(), d.path.as_str())) { + continue; + } + if d.ahead > 0 { + hidden_ahead.total += 1; + *hidden_ahead.by_kind.entry(d.kind.clone()).or_default() += 1; + if sees_all_items { + hidden_ahead + .items + .push(HiddenItem { kind: d.kind.clone(), path: d.path.clone() }); + } + } + if d.behind > 0 { + hidden_behind.total += 1; + *hidden_behind.by_kind.entry(d.kind.clone()).or_default() += 1; + if sees_all_items { + hidden_behind + .items + .push(HiddenItem { kind: d.kind.clone(), path: d.path.clone() }); + } + } + } + return Ok(Json(WorkspaceComparison { all_ahead_items_visible, all_behind_items_visible, skipped_comparison: false, diffs: visible_diffs, summary, + hidden_ahead, + hidden_behind, })); } @@ -7281,7 +8144,6 @@ async fn compare_two_resources( .fetch_optional(db) .await?; - // If either side is ws_specific, consider unchanged let source_ws_specific = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM ws_specific WHERE workspace_id = $1 AND item_kind = 'resource' AND path = $2)", source_workspace_id, @@ -7300,6 +8162,9 @@ async fn compare_two_resources( .await? .unwrap_or(false); + // A workspace-specific resource keeps its own value per environment, so it never appears in the + // diff (in either direction). Seeding the initial copy onto a side that lacks it is a separate + // explicit action ("Create in "), not a diff entry. if source_ws_specific || target_ws_specific { return Ok(ItemComparison { has_changes: false, @@ -7356,7 +8221,8 @@ async fn compare_two_variables( .fetch_one(db) .await?; - // If either side is ws_specific, consider unchanged + // A workspace-specific variable keeps its own value per environment, so it never appears in the + // diff. Seeding the initial copy onto a side that lacks it is a separate explicit action. if presence.src_ws || presence.tgt_ws { return Ok(ItemComparison { has_changes: false, @@ -7647,6 +8513,10 @@ struct CloudQuotas { apps: QuotaInfo, variables: QuotaInfo, resources: QuotaInfo, + /// Fork/dev workspaces under this workspace's billing root vs the per-seat cap. `limit` is 0 for a + /// non-premium root (forking is premium-only). Family-wide: resolves to the billing root, so it + /// reads the same whether viewed from the root or one of its forks. + forks: QuotaInfo, } async fn get_cloud_quotas( @@ -7727,12 +8597,32 @@ async fn get_cloud_quotas( .await? .unwrap_or(0); + // Fork/dev workspaces vs the per-seat cap, resolved to the billing root. Non-premium roots can't + // fork, so their allowance is 0. + #[cfg(feature = "cloud")] + let forks = { + use windmill_common::workspaces::{ + count_paid_seats, count_workspace_forks, get_billing_workspace_id, get_team_plan_status, + }; + let root = get_billing_workspace_id(&db, &w_id).await?; + let used = count_workspace_forks(&db, &root).await?; + let limit = if get_team_plan_status(&db, &root).await?.premium { + count_paid_seats(&db, &root).await?.max(1) * *MAX_FORKS_PER_SEAT + } else { + 0 + }; + QuotaInfo { used, limit, prunable: 0 } + }; + #[cfg(not(feature = "cloud"))] + let forks = QuotaInfo { used: 0, limit: 0, prunable: 0 }; + Ok(Json(CloudQuotas { scripts: QuotaInfo { used: scripts_used, limit: 5000, prunable: scripts_prunable }, flows: QuotaInfo { used: flows_used, limit: 1000, prunable: flows_prunable }, apps: QuotaInfo { used: apps_used, limit: 1000, prunable: apps_prunable }, variables: QuotaInfo { used: variables_used, limit: 10000, prunable: 0 }, resources: QuotaInfo { used: resources_used, limit: 10000, prunable: 0 }, + forks, })) } @@ -7870,6 +8760,19 @@ async fn list_ws_specific( .fetch_all(&mut *tx) .await?; tx.commit().await?; + // RLS gates membership/folder access, but a scoped API token must also be held to its read + // scopes — mirror the resource/variable list endpoints, which filter with these predicates so a + // token lacking `resources:read:*` / `variables:read:*` can't enumerate pinned paths it can't read. + let resource_allowed = build_scope_path_predicate(&authed, "resources", "read"); + let variable_allowed = build_scope_path_predicate(&authed, "variables", "read"); + let items = items + .into_iter() + .filter(|it| match it.item_kind.as_str() { + "resource" => resource_allowed(&it.path), + "variable" => variable_allowed(&it.path), + _ => false, + }) + .collect::>(); Ok(Json(items)) } @@ -7892,6 +8795,17 @@ async fn list_ws_specific_versions( ))); } + // A scoped API token must hold the read scope for this path, like the resource/variable read + // endpoints. Without the scope, report no versions rather than leaking the path's history. + let domain = if q.kind == "resource" { + "resources" + } else { + "variables" + }; + if !build_scope_path_predicate(&authed, domain, "read")(&q.path) { + return Ok(Json(vec![])); + } + let versions: Vec = sqlx::query_scalar!( r#"SELECT ws AS "ws!" FROM list_ws_specific_versions($1, $2, $3, $4)"#, &w_id, @@ -7904,3 +8818,155 @@ async fn list_ws_specific_versions( Ok(Json(versions)) } + +#[derive(Deserialize)] +struct SetWsSpecificBody { + item_kind: String, + path: String, + value: bool, +} + +/// Mark (or unmark) a single resource/variable as workspace-specific. Pinning +/// excludes it from the deploy diff so each environment keeps its own value +/// (see `compare_two_resources`/`compare_two_variables`). Set per-workspace, so +/// the compare page calls this once per side to flag both environments. +async fn set_ws_specific( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(body): Json, +) -> Result { + if body.item_kind != "resource" && body.item_kind != "variable" { + return Err(Error::BadRequest(format!( + "Invalid kind '{}'. Must be 'resource' or 'variable'", + body.item_kind + ))); + } + + // Reject a malformed path with a 400 before the auth check, which indexes the leading segments and + // would otherwise panic (500) on a path like `u` with no segment. Accept all three shared path + // shapes Windmill uses — `u/`, `f/`, `g/` (e.g. seeded `g/all/...` resources). + let segs: Vec<&str> = body.path.split('/').collect(); + if segs.len() < 2 || !matches!(segs[0], "u" | "f" | "g") || segs[1].is_empty() { + return Err(Error::BadRequest(format!( + "Invalid {} path: {}", + body.item_kind, body.path + ))); + } + + // Authorize like the resource/variable editors' own ws_specific toggle: + // actual write access to the item + token scope + the workspace deploy rules. + // `require_owner_of_path` is the real write gate (the resource editor uses it); + // `check_scopes` only constrains scoped tokens (it is a no-op for session/cookie + // logins). Together: a non-admin who can edit the item may pin it, while a + // read-only member is rejected and a locked workspace still blocks non-deployers. + // `require_is_writer` matches the resource/variable editors' write semantics (owner, folder + // writer, or item writer via extra_perms) — not owner-only. + let writer_query = if body.item_kind == "resource" { + "SELECT extra_perms FROM resource WHERE path = $1 AND workspace_id = $2" + } else { + "SELECT extra_perms FROM variable WHERE path = $1 AND workspace_id = $2" + }; + require_is_writer( + &authed, + &body.path, + &w_id, + db.clone(), + writer_query, + &body.item_kind, + ) + .await?; + check_scopes(&authed, || { + format!("{}s:write:{}", body.item_kind, body.path) + })?; + if let RuleCheckResult::Blocked(msg) = check_deploy_rules( + &w_id, + &authed.username, + &authed.groups, + authed.is_admin, + &db, + ) + .await? + { + return Err(Error::PermissionDenied(msg)); + } + + let mut tx = user_db.begin(&authed).await?; + + if body.value { + // Existence guard keeps a dangling marker from being created for a + // path absent in this workspace. + if body.item_kind == "resource" { + sqlx::query!( + "INSERT INTO ws_specific (workspace_id, item_kind, path) + SELECT $1::varchar, 'resource', $2::varchar + WHERE EXISTS (SELECT 1 FROM resource WHERE workspace_id = $1::varchar AND path = $2::varchar) + ON CONFLICT DO NOTHING", + w_id, + body.path, + ) + .execute(&mut *tx) + .await?; + // A resource owns its `$var:` secrets, so pin those too. + windmill_store::resources::mark_linked_variables_ws_specific( + &mut tx, &authed, &w_id, &body.path, + ) + .await?; + } else { + sqlx::query!( + "INSERT INTO ws_specific (workspace_id, item_kind, path) + SELECT $1::varchar, 'variable', $2::varchar + WHERE EXISTS (SELECT 1 FROM variable WHERE workspace_id = $1::varchar AND path = $2::varchar) + ON CONFLICT DO NOTHING", + w_id, + body.path, + ) + .execute(&mut *tx) + .await?; + } + } else { + // Unmark only this item; linked variables stay flagged (they may be + // referenced by other resources) — mirrors the resource-form toggle. + sqlx::query!( + "DELETE FROM ws_specific WHERE workspace_id = $1 AND item_kind = $2 AND path = $3", + w_id, + body.item_kind, + body.path, + ) + .execute(&mut *tx) + .await?; + // While pinned, the item's cached workspace_diff verdict was never recomputed (the compare + // read filter excludes it), so it may now be stale in either direction. Mark it NULL so the + // next compare re-evaluates from scratch — and the now-shared item reappears (or is dropped) + // correctly instead of being stuck on its pre-pin verdict. + sqlx::query!( + "UPDATE workspace_diff SET has_changes = NULL + WHERE path = $2 AND kind = $3 + AND ($1 IN (source_workspace_id, fork_workspace_id))", + w_id, + body.path, + body.item_kind, + ) + .execute(&mut *tx) + .await?; + } + + let value_str = body.value.to_string(); + audit_log( + &mut *tx, + &authed, + &format!("{}s.set_ws_specific", body.item_kind), + ActionKind::Update, + &w_id, + Some(&body.path), + Some([("value", value_str.as_str())].into()), + ) + .await?; + + tx.commit().await?; + Ok(format!( + "Set workspace-specific={} for {} {}", + body.value, body.item_kind, body.path + )) +} diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 3cb2b29f79..daf83171e4 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use windmill_api_auth::{require_super_admin, ApiAuthed}; -use windmill_common::workspaces::WM_FORK_PREFIX; use windmill_common::DB; use crate::workspaces::{ @@ -26,7 +25,7 @@ use windmill_common::{ db::UserDB, error::{Error, Result}, utils::require_admin, - workspaces::DataTable, + workspaces::{DataTable, DEV_WORKSPACE_LOCK_RULE_NAME, WM_FORK_PREFIX}, }; use windmill_queue::schedule::{get_schedule_opt, push_scheduled_job}; @@ -65,22 +64,42 @@ pub(crate) async fn change_workspace_id( old_id, rw.new_id ); - // Create new workspace with new id and name. A fork that keeps a wm-fork- - // id must carry its parent_workspace_id over, otherwise it becomes a - // parentless "fork of nothing" with no source to compare or merge against. - // A non-fork target id means the workspace is being promoted out of a fork, - // so the parent pointer is intentionally cleared. + // Create new workspace with new id and name. Fork lineage AND the dev designation are preserved + // from the source row, not inferred from the new id's prefix: a prefix-less fork (a dev or + // detached-dev workspace) would otherwise be silently promoted to a root workspace, and a dev + // would lose its flag — leaving its prod locked with no canonical dev. Promoting out of a fork is + // a separate, explicit action — a rename never does it implicitly. info!("Creating new workspace row"); - let new_is_fork = rw.new_id.starts_with(WM_FORK_PREFIX); + let old = sqlx::query!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!", is_dev_workspace + FROM workspace WHERE id = $1"#, + &old_id + ) + .fetch_optional(&mut *tx) + .await?; + let new_is_fork = old.as_ref().map(|o| o.has_parent).unwrap_or(false); + let new_is_dev = new_is_fork && old.as_ref().map(|o| o.is_dev_workspace).unwrap_or(false); + // Neutralize the old row's dev flag BEFORE inserting the new one: the move-and-archive archives + // the old row only later, so without this the new dev row and the not-yet-archived old dev row + // would momentarily both be active under the same parent and trip the one-dev-per-parent index. + if new_is_dev { + sqlx::query!( + "UPDATE workspace SET is_dev_workspace = false WHERE id = $1", + &old_id + ) + .execute(&mut *tx) + .await?; + } sqlx::query!( - "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id) + "INSERT INTO workspace (id, name, owner, deleted, premium, parent_workspace_id, is_dev_workspace) SELECT $1, $2, owner, false, premium, - CASE WHEN $4 THEN parent_workspace_id ELSE NULL END + CASE WHEN $4 THEN parent_workspace_id ELSE NULL END, $5 FROM workspace WHERE id = $3", &rw.new_id, &rw.new_name, &old_id, - new_is_fork + new_is_fork, + new_is_dev ) .execute(&mut *tx) .await?; @@ -257,6 +276,24 @@ pub(crate) async fn change_workspace_id( .execute(&mut *tx) .await?; + info!("Updating macro_definition table"); + sqlx::query!( + "UPDATE macro_definition SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + + info!("Updating macro_usage table"); + sqlx::query!( + "UPDATE macro_usage SET workspace_id = $1 WHERE workspace_id = $2", + &rw.new_id, + &old_id + ) + .execute(&mut *tx) + .await?; + info!("Updating deployment_metadata table"); sqlx::query!( "UPDATE deployment_metadata SET workspace_id = $1 WHERE workspace_id = $2", @@ -360,8 +397,18 @@ pub(crate) async fn change_workspace_id( // must follow the renamed parent to the new id, otherwise it is left // pointing at the soft-deleted old shell (whose data has moved here). info!("Re-parenting child forks to the new workspace id"); + let reparented_children: Vec = sqlx::query_scalar!( + "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2 RETURNING id", + &rw.new_id, + &old_id + ) + .fetch_all(&mut *tx) + .await?; + + // A dev/fork's `deploy_to` points at the prod root, so it must follow the rename too — otherwise + // the child re-parents to the new id but still deploys to the soft-deleted old shell. sqlx::query!( - "UPDATE workspace SET parent_workspace_id = $1 WHERE parent_workspace_id = $2", + "UPDATE workspace_settings SET deploy_to = $1 WHERE deploy_to = $2", &rw.new_id, &old_id ) @@ -696,11 +743,21 @@ pub(crate) async fn change_workspace_id( tx.commit().await?; + // The children's parent_workspace_id changed (old root -> new root); invalidate their fork-parent + // routing cache and their billing-workspace mapping so jobs route + meter under the renamed root + // rather than the old (archived) one, instead of waiting for the caches' TTLs. Deeper descendants + // (fork-of-fork) self-heal via the 60s billing-cache TTL. + for child in &reparented_children { + windmill_queue::tags::invalidate_fork_parent_cache(child); + #[cfg(feature = "cloud")] + windmill_common::workspaces::invalidate_billing_workspace_cache(child); + } + // Archive old workspace: disable schedules, cancel remaining jobs, set deleted=true // Note: schedules were already moved to new workspace, so this will find 0 schedules info!("Archiving old workspace"); let (_schedules_count, canceled_count, _deleted_tokens_count) = - archive_workspace_impl(&db, &old_id, &authed.username).await?; + archive_workspace_impl(&db, &old_id, &authed.username, None).await?; info!( "Workspace id change completed: moved {} to {}, archived old workspace", @@ -734,15 +791,65 @@ pub(crate) async fn delete_workspace( _ => Ok(w_id), }?; - if dwq.only_delete_forks.unwrap_or(false) && !w_id.starts_with(WM_FORK_PREFIX) { + let is_fork = workspace_is_fork(&db, &w_id).await?; + if dwq.only_delete_forks.unwrap_or(false) && !is_fork { return Err(Error::BadRequest( "Cannot delete this workspace because it is not a workspace fork.".to_string(), )); } let mut tx = db.begin().await?; - if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { - require_super_admin(&db, &authed.email).await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Deleting this workspace requires being the fork's owner or a superadmin".to_string(), + )); + } + + // Don't hard-delete a workspace that still has a dev workspace paired to it: the FK is + // ON DELETE SET NULL, which would orphan the (prefix-less) dev into a parentless, non-fork row + // its owner could no longer self-delete. Require detaching/deleting the dev first. Ordinary + // forks have no such guard — they keep their prefix and stay owner-deletable when orphaned. + // Archived devs (deleted = true) are included: they keep is_dev_workspace = true, so SET NULL on + // their parent would violate the `is_dev ⇒ has parent` CHECK and fail the whole delete with a 500. + if let Some(dev_id) = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + { + return Err(Error::BadRequest(format!( + "Cannot delete workspace '{}' because it has a dev workspace ('{}'). Detach or delete the dev workspace first.", + w_id, dev_id + ))); + } + + // Deleting an attached dev workspace removes the parent prod's dev_workspace_lock (below), so it + // must be a prod-admin action, not just the dev's own owner (dev ownership can diverge from + // prod's) — mirrors detach_dev_workspace, which is prod-admin gated. + if let Some(prod) = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + { + let is_prod_admin = sqlx::query_scalar!( + "SELECT is_admin FROM usr WHERE workspace_id = $1 AND email = $2", + &prod, + &authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if !is_prod_admin && !is_super_admin_email(&db, &authed.email).await? { + return Err(Error::PermissionDenied(format!( + "Deleting dev workspace '{w_id}' requires being an admin of its parent prod workspace '{prod}' (or a superadmin)" + ))); + } } sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id) @@ -763,6 +870,15 @@ pub(crate) async fn delete_workspace( sqlx::query!("DELETE FROM dependency_map WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; + sqlx::query!("DELETE FROM macro_usage WHERE workspace_id = $1", &w_id) + .execute(&mut *tx) + .await?; + sqlx::query!( + "DELETE FROM macro_definition WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; sqlx::query!("DELETE FROM v2_job_queue WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; @@ -920,10 +1036,41 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + // If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop + // the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod + // stays locked against direct deploy/forking with no dev workspace left to make changes in. + let dev_lock_parent: Option = sqlx::query_scalar!( + "SELECT parent_workspace_id FROM workspace WHERE id = $1 AND is_dev_workspace", + &w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + // Capture direct child forks before the delete: the FK is ON DELETE SET NULL, so they're about to + // be orphaned (their billing root changes from this workspace's root to themselves). We drop their + // cached mappings after commit alongside the deleted id itself. + let orphaned_children: Vec = sqlx::query_scalar!( + "SELECT id FROM workspace WHERE parent_workspace_id = $1", + &w_id + ) + .fetch_all(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace WHERE id = $1", &w_id) .execute(&mut *tx) .await?; + if let Some(ref parent) = dev_lock_parent { + sqlx::query!( + "DELETE FROM workspace_protection_rule WHERE workspace_id = $1 AND name = $2", + parent, + DEV_WORKSPACE_LOCK_RULE_NAME + ) + .execute(&mut *tx) + .await?; + } + // Record under the instance-level "admins" workspace. The per-workspace audit // rows are deleted along with the workspace, so this instance-level entry is the // only durable, superadmin-discoverable record of who deleted the workspace. @@ -939,6 +1086,26 @@ pub(crate) async fn delete_workspace( .await?; tx.commit().await?; + if let Some(parent) = dev_lock_parent { + windmill_common::workspaces::invalidate_protection_rules_cache(&parent); + } + + // Workspace ids are reusable after permanent deletion, so drop every cached mapping keyed by the + // deleted id (and any just-orphaned children) — otherwise a recreated id could inherit the gone + // workspace's state within the caches' lifetimes. This covers fork->parent (tag routing) and + // fork->root (billing), plus the premium/team-plan status: TEAM_PLAN_CACHE has no TTL and is only + // evicted by the premium-change NOTIFY, so without this a reused id would keep the old workspace's + // premium indefinitely (free forks/usage). Deeper (grandchild) descendants self-heal via the 60s + // billing TTL. + for id in std::iter::once(&w_id).chain(orphaned_children.iter()) { + windmill_queue::tags::invalidate_fork_parent_cache(id); + #[cfg(feature = "cloud")] + { + windmill_common::workspaces::invalidate_billing_workspace_cache(id); + windmill_common::workspaces::invalidate_team_plan_cache(id); + } + } + Ok(format!("Deleted workspace {}", &w_id)) } @@ -957,9 +1124,15 @@ pub async fn drop_forked_datatable_databases( Json(req): Json, ) -> Result>> { // Same permission check as delete_workspace: fork owner or super admin + let is_fork = workspace_is_fork(&db, &w_id).await?; let mut tx = db.begin().await?; - if !(w_id.starts_with(WM_FORK_PREFIX) && is_workspace_owner(&authed, &w_id, &mut tx).await?) { - require_super_admin(&db, &authed.email).await?; + if !(is_fork && is_workspace_owner(&authed, &w_id, &mut tx).await?) + && !is_super_admin_email(&db, &authed.email).await? + { + return Err(Error::PermissionDenied( + "Dropping forked datatable databases requires being the fork's owner or a superadmin" + .to_string(), + )); } tx.commit().await?; @@ -1103,3 +1276,21 @@ async fn is_workspace_owner( .await?; Ok(owner.map(|o| o == authed.email).unwrap_or(false)) } + +/// Whether a workspace is a fork or dev workspace. Both forks and dev workspaces set +/// `parent_workspace_id`, but a `wm-fork-` workspace can outlive its parent (the FK is +/// `ON DELETE SET NULL`), so also treat the prefix as fork-ness — otherwise an orphaned fork would +/// lose owner-self-delete. Used to gate owner-self-delete, which is permitted for forks/dev +/// workspaces but requires superadmin otherwise. +async fn workspace_is_fork(db: &DB, w_id: &str) -> Result { + if w_id.starts_with(WM_FORK_PREFIX) { + return Ok(true); + } + Ok(sqlx::query_scalar!( + r#"SELECT (parent_workspace_id IS NOT NULL) AS "has_parent!" FROM workspace WHERE id = $1"#, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or(false)) +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 6ce463a5aa..ebbc98e83e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,8 +10,8 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise"] +private = ["windmill-audit/private", "windmill-common/private", "windmill-api-auth/private", "windmill-store/private", "windmill-api-users/private", "windmill-api-workspaces/private", "windmill-api-groups/private", "windmill-api-configs/private", "windmill-api-settings/private", "windmill-api-assets/private", "windmill-api-agent-workers?/private", "windmill-trigger-kafka?/private", "windmill-trigger-postgres?/private", "windmill-trigger-mqtt?/private", "windmill-trigger-websocket?/private", "windmill-trigger-nats?/private", "windmill-trigger-sqs?/private", "windmill-trigger-gcp?/private", "windmill-trigger-azure?/private", "windmill-trigger-email?/private", "windmill-git-sync/private", "windmill-autoscaling?/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker?/enterprise", "windmill-api-auth/enterprise", "windmill-store/enterprise", "windmill-api-jobs/enterprise", "windmill-api-scripts/enterprise", "windmill-api-flows/enterprise", "windmill-api-users/enterprise", "windmill-api-workspaces/enterprise", "windmill-api-groups/enterprise", "windmill-api-configs/enterprise", "windmill-api-settings/enterprise", "windmill-api-schedule/enterprise", "windmill-api-agent-workers?/enterprise", "windmill-trigger/enterprise", "windmill-trigger-kafka?/enterprise", "windmill-trigger-postgres?/enterprise", "windmill-trigger-mqtt?/enterprise", "windmill-trigger-websocket?/enterprise", "windmill-trigger-email?/enterprise", "windmill-trigger-nats?/enterprise", "windmill-trigger-sqs?/enterprise", "windmill-trigger-gcp?/enterprise", "windmill-trigger-azure?/enterprise", "windmill-trigger-http?/enterprise", "windmill-native-triggers?/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "license"] stripe = [] run_inline = ["dep:windmill-worker", "windmill-api-configs/run_inline"] agent_worker_server = ["dep:windmill-worker", "dep:windmill-api-agent-workers"] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 7daddac726..75ab6f1240 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.742.0 + version: 1.746.0 title: Windmill API contact: @@ -1184,6 +1184,89 @@ paths: schema: type: string + /w/{workspace}/workspaces/attach_dev_workspace: + post: + summary: attach an existing workspace as the dev workspace of this (prod) workspace + operationId: attachDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + dev_workspace_id: + type: string + lock_prod_deploy: + type: boolean + lock_prod_forking: + type: boolean + required: + - dev_workspace_id + responses: + "200": + description: dev workspace attached + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/detach_dev_workspace: + post: + summary: detach the dev workspace from this (prod) workspace + operationId: detachDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + dev_workspace_id: + type: string + required: + - dev_workspace_id + responses: + "200": + description: dev workspace detached + content: + text/plain: + schema: + type: string + + /w/{workspace}/workspaces/get_dev_workspace: + get: + summary: this workspace's active canonical dev workspace, if any + operationId: getDevWorkspace + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: the dev workspace, or null if none + content: + application/json: + schema: + nullable: true + type: object + properties: + id: + type: string + name: + type: string + required: + - id + - name + /workspaces/exists: post: summary: exists workspace @@ -6483,12 +6566,15 @@ paths: $ref: "#/components/schemas/QuotaInfo" resources: $ref: "#/components/schemas/QuotaInfo" + forks: + $ref: "#/components/schemas/QuotaInfo" required: - scripts - flows - apps - variables - resources + - forks /w/{workspace}/workspaces/prune_versions: post: @@ -6583,6 +6669,42 @@ paths: items: type: string + /w/{workspace}/workspaces/set_ws_specific: + post: + summary: mark or unmark an item as workspace-specific + operationId: setWsSpecific + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + item_kind: + type: string + enum: + - resource + - variable + path: + type: string + value: + type: boolean + required: + - item_kind + - path + - value + responses: + "200": + description: workspace-specific flag updated + content: + text/plain: + schema: + type: string + /w/{workspace}/workspaces/public_app_rate_limit: post: summary: Set public app rate limit for this workspace @@ -20124,6 +20246,55 @@ paths: application/json: schema: {} + /w/{workspace}/job_helpers/storage_usage: + get: + summary: Get the storage usage of the workspace object storage, per configured storage. On Community Edition, also returns the workspace storage quota. + operationId: getStorageUsage + tags: + - helpers + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: refresh + in: query + description: recount usage by listing the storage instead of returning cached values + schema: + type: boolean + responses: + "200": + description: Storage usage + content: + application/json: + schema: + type: object + properties: + total_bytes: + type: integer + format: int64 + quota_bytes: + type: integer + format: int64 + description: only present on Community Edition, where workspace storage is capped + storages: + type: array + items: + type: object + properties: + storage: + type: string + bytes: + type: integer + format: int64 + computed_at: + type: string + format: date-time + required: + - storage + - bytes + - computed_at + required: + - total_bytes + - storages + /w/{workspace}/job_helpers/list_stored_files: get: summary: List the file keys available in a workspace object storage @@ -21607,6 +21778,20 @@ paths: in_pipeline: type: boolean description: True iff the script is a pipeline member (deployed with `// pipeline`). Omitted when false. + macros: + type: array + description: Macros this script provides to the workspace registry (deployed `// macros` library). Omitted when empty. + items: + type: object + required: [name, params, is_table] + properties: + name: + type: string + params: + type: string + description: verbatim parameter list + is_table: + type: boolean edges: type: array items: @@ -21672,6 +21857,56 @@ paths: $ref: "#/components/schemas/AssetUsageKind" runnable_path: type: string + macro_edges: + type: array + description: Macro-library → consumer edges (deploy-recorded call detection plus `// use`). Omitted when empty. + items: + type: object + required: [lib_path, consumer_path, macro_names, via_use] + properties: + lib_path: + type: string + consumer_path: + type: string + macro_names: + type: array + items: + type: string + via_use: + type: boolean + + /w/{workspace}/assets/macros: + get: + summary: List every workspace DuckDB macro (deployed `// macros` libraries) + operationId: listWorkspaceMacros + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: all registry macros, ordered by provider then name + content: + application/json: + schema: + type: array + items: + type: object + required: [name, params, body, is_table, provider_path] + properties: + name: + type: string + params: + type: string + description: verbatim parameter list + body: + type: string + description: verbatim body after AS [TABLE] + is_table: + type: boolean + provider_path: + type: string + description: path of the `// macros` library script /w/{workspace}/assets/pipelines: get: @@ -21700,6 +21935,90 @@ paths: format: int64 description: Number of pipeline-member scripts in the folder + /w/{workspace}/assets/partitions: + get: + summary: List materialized partitions for a ducklake asset + operationId: listAssetPartitions + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + responses: + "200": + description: per-partition materialization status + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/MaterializedPartition" + + /w/{workspace}/assets/partitions_in_range: + get: + summary: List expected partitions of a ducklake asset in a date range with their materialization status (enterprise) + operationId: listAssetPartitionsInRange + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + - name: from + in: query + required: true + description: Inclusive range start (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + - name: to + in: query + required: true + description: Inclusive range end (YYYY-MM-DD), local to the producer's partition tz + schema: + type: string + format: date + responses: + "200": + description: expected partitions in range with per-slice status — the missing/failed subset is the backfill worklist + content: + application/json: + schema: + $ref: "#/components/schemas/PartitionsInRange" + + /w/{workspace}/assets/asset_schemas: + get: + summary: List captured output-schema versions for a ducklake asset + operationId: listAssetSchemas + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path + in: query + required: true + description: The materialized ducklake asset path (`/
`) + schema: + type: string + responses: + "200": + description: captured schema versions, newest first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AssetSchemaVersion" + /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -23022,6 +23341,7 @@ components: [ openai, azure_openai, + azure_foundry, anthropic, mistral, deepseek, @@ -24152,6 +24472,9 @@ components: - $ref: "#/components/schemas/UserSource" is_service_account: type: boolean + non_member: + type: boolean + description: "True when this is a superadmin viewing a workspace they are not a member of (is_admin/role reflect the superadmin fallback, not an actual membership)." required: - email - username @@ -28101,6 +28424,8 @@ components: parent_workspace_id: type: string nullable: true + is_dev_workspace: + type: boolean created_by: type: string nullable: true @@ -28111,6 +28436,7 @@ components: - name - username - color + - is_dev_workspace - disabled required: - email @@ -28152,6 +28478,18 @@ components: new_dbname: type: string description: "New database name for the fork" + is_dev_workspace: + type: boolean + description: "Create the fork as a persistent dev workspace (id not required to carry the wm-fork- prefix; at most one per parent)" + lock_prod_deploy: + type: boolean + description: "When creating a dev workspace, lock the parent (prod) against direct deployment" + lock_prod_forking: + type: boolean + description: "When creating a dev workspace, prevent forking the parent (prod)" + copy_members: + type: boolean + description: "Copy the parent's members (users + group memberships) into the fork so the team can work in it" required: - id - name @@ -29527,6 +29865,8 @@ components: - skipped_comparison - diffs - summary + - hidden_ahead + - hidden_behind properties: all_ahead_items_visible: type: boolean @@ -29545,6 +29885,46 @@ components: summary: $ref: "#/components/schemas/CompareSummary" description: Summary statistics of the comparison + hidden_ahead: + $ref: "#/components/schemas/HiddenItemsSummary" + description: Ahead items excluded from `diffs` because they are not visible to the caller + hidden_behind: + $ref: "#/components/schemas/HiddenItemsSummary" + description: Behind items excluded from `diffs` because they are not visible to the caller + + HiddenItemsSummary: + type: object + required: + - total + - by_kind + - items + properties: + total: + type: integer + description: Total number of hidden items on this side + by_kind: + type: object + additionalProperties: + type: integer + description: Count of hidden items keyed by item kind (always populated) + items: + type: array + description: Kind and path of each hidden item; only populated when the caller is an admin of the relevant side (empty otherwise) + items: + $ref: "#/components/schemas/HiddenItem" + + HiddenItem: + type: object + required: + - kind + - path + properties: + kind: + type: string + description: Type of the hidden item + path: + type: string + description: Path of the hidden item WorkspaceItemDiff: type: object @@ -29820,6 +30200,86 @@ components: - ducklake - datatable - volume + MaterializedPartition: + type: object + required: [asset_kind, asset_path, partition, status, materialized_at] + properties: + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + partition: + type: string + status: + type: string + enum: [running, materialized, failed] + snapshot_id: + type: integer + format: int64 + nullable: true + row_count: + type: integer + format: int64 + nullable: true + job_id: + type: string + format: uuid + nullable: true + materialized_at: + type: string + format: date-time + error: + type: string + nullable: true + PartitionsInRange: + type: object + required: [producer_path, partition_kind, partitions] + properties: + producer_path: + type: string + description: the pipeline script that materializes the asset (managed `// materialize` target, or a partitioned writer using the SDK helpers) — the runnable a backfill launches + partition_kind: + type: string + enum: [daily, hourly, weekly, monthly, dynamic] + partitions: + type: array + items: + type: object + required: [partition, status] + properties: + partition: + type: string + status: + type: string + enum: [missing, running, materialized, failed] + AssetSchemaVersion: + type: object + required: [version, columns, captured_at] + properties: + version: + type: integer + format: int64 + columns: + type: array + items: + type: object + required: [name, type] + properties: + name: + type: string + type: + type: string + snapshot_id: + type: integer + format: int64 + nullable: true + job_id: + type: string + format: uuid + nullable: true + captured_at: + type: string + format: date-time Asset: type: object properties: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 08a41e2d1f..99d6156687 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -7,6 +7,11 @@ use std::{collections::HashMap, sync::Arc}; * Please see the included NOTICE for copyright information and * LICENSE-AGPL for a copy of the license. */ +#[cfg(all(feature = "parquet", not(feature = "enterprise")))] +use crate::job_helpers_oss::{ + bump_storage_usage, ce_upload_budget, reject_reserved_volume_key, + spawn_storage_usage_recount_floored, +}; use crate::{ auth::{get_end_user_email, OptTokened}, db::{ApiAuthed, DB}, @@ -415,13 +420,14 @@ async fn list_apps( "app.labels", "draft.path IS NOT NULL as is_draft", // Per-path draft owners as a JSON array; see scripts.rs for the rationale - // (admins-workspace identity fallback, legacy NULL-email row). + // (non-member superadmin identity fallback via `password`, legacy NULL-email row). // `app`/`raw_app` are separate draft kinds over one `app` table — match // either (like the `is_draft` join below), else a deployed raw app's draft // owners are dropped and the row shows "Draft" with no user badge. - "(SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END)) ORDER BY COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) NULLS LAST) \ + "(SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END)) ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) NULLS LAST) \ FROM draft d \ LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email \ + LEFT JOIN password p ON p.email = d.email AND p.super_admin = true \ WHERE d.workspace_id = app.workspace_id AND d.path = app.path AND d.typ IN ('app', 'raw_app')) as draft_users", "folder_labels(app.workspace_id, app.path) as inherited_labels", ]) @@ -3543,7 +3549,49 @@ async fn upload_s3_file_from_app( ]) .into(); - let _put_result = upload_file_from_req(s3_client, &file_key, request, options).await?; + // Only workspace storage is quota-metered; a custom-resource upload lands in + // the user's own bucket and is neither capped nor counted. An overwrite of an + // existing key only spends the difference over its current size. + let _is_workspace_storage = query.s3_resource_path.is_none(); + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + reject_reserved_volume_key(&file_key)?; + } + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + let (max_size, _existing_size) = if _is_workspace_storage { + let content_length = request + .headers() + .get(http::header::CONTENT_LENGTH) + .and_then(|h| h.to_str().ok()) + .and_then(|s| s.parse::().ok()); + let budget = ce_upload_budget(&db, &w_id, &s3_client, &file_key, content_length).await?; + (Some(budget.max_size), budget.existing_size) + } else { + (None, 0) + }; + #[cfg(any(not(feature = "parquet"), feature = "enterprise"))] + let max_size: Option = None; + + match upload_file_from_req(s3_client, &file_key, request, options, max_size).await { + Ok((_, _size)) => + { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + if _is_workspace_storage { + bump_storage_usage( + &db, + &w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64 - _existing_size, + ) + .await; + } + } + Err(e) => { + #[cfg(all(feature = "parquet", not(feature = "enterprise")))] + spawn_storage_usage_recount_floored(&db, &w_id); + return Err(e); + } + } let delete_token = jwt::encode_with_internal_secret(S3DeleteTokenClaims { file_key: file_key.clone(), diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 3793e94b02..dc8b36fbaf 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -90,6 +90,10 @@ impl RawWebhookArgs { db: &DB, w_id: &str, ) -> Result>, Error> { + #[cfg(not(feature = "enterprise"))] + use crate::job_helpers_oss::{ + bump_storage_usage, ce_storage_quota_remaining, spawn_storage_usage_recount_floored, + }; use crate::job_helpers_oss::{ get_random_file_name, get_workspace_s3_resource, upload_file_internal, }; @@ -139,8 +143,38 @@ impl RawWebhookArgs { .into_stream() .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); - upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) - .await?; + // file_key is always freshly random here, so this never + // overwrites an existing object; the full size is the delta. + #[cfg(not(feature = "enterprise"))] + let max_size = Some(ce_storage_quota_remaining(db, w_id, None).await? as usize); + #[cfg(feature = "enterprise")] + let max_size: Option = None; + + match upload_file_internal( + s3_client.clone(), + &file_key, + bytes_stream, + options, + max_size, + ) + .await + { + Ok((_, _size)) => { + #[cfg(not(feature = "enterprise"))] + bump_storage_usage( + db, + w_id, + windmill_object_store::DEFAULT_STORAGE, + _size as i64, + ) + .await; + } + Err(e) => { + #[cfg(not(feature = "enterprise"))] + spawn_storage_usage_recount_floored(db, w_id); + return Err(e); + } + } files.entry(name).or_insert(vec![]).push(serde_json::json!({ "s3": &file_key diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 9d4976489b..72791de78f 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -18,6 +18,7 @@ use windmill_common::{ db::UserDB, error::{Error, Result}, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, + users::resolve_username_to_email, variables::{build_crypt, encrypt}, }; @@ -171,11 +172,16 @@ fn list_drafts_query(all_users: bool) -> String { // draft author at this (path, kind), legacy NULL-email row surfaced as a // null username. Restricted to the shared full-page-editor kinds — drawer // kinds keep their drafts private, so we never reveal their authors. + // A superadmin authoring in a workspace they are not a member of has no `usr` + // row: fall back to their instance-derived username (`password.username`), or + // their email when derivation is disabled (`password.username` is NULL). This + // keeps the raw email out of the payload whenever a derived username exists. let draft_users = r#"CASE WHEN d.typ::text IN ('script', 'flow', 'app', 'raw_app') THEN ( - SELECT json_agg(json_build_object('username', COALESCE(u.username, CASE WHEN du.workspace_id = 'admins' THEN du.email END)) - ORDER BY COALESCE(u.username, CASE WHEN du.workspace_id = 'admins' THEN du.email END) NULLS LAST) + SELECT json_agg(json_build_object('username', COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END)) + ORDER BY COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN du.email END) NULLS LAST) FROM draft du LEFT JOIN usr u ON u.workspace_id = du.workspace_id AND u.email = du.email + LEFT JOIN password p ON p.email = du.email AND p.super_admin = true WHERE du.workspace_id = d.workspace_id AND du.path = d.path AND du.typ = d.typ ) ELSE NULL END"#; // Default lists the user's own drafts AND the legacy NULL-email rows; with @@ -578,20 +584,12 @@ async fn get_draft_for_user( // Username -> email, scoped to the workspace. None signals "fetch the // legacy NULL-email row" (distinct from a username with no draft, which - // 404s below). + // 404s below). Resolution falls back to the instance `password` table so a + // superadmin's draft (they are not a `usr` member of the workspace, and + // their username is their instance-derived one) still resolves. let owner_email: Option = if let Some(username) = &query.username { - let email = sqlx::query_scalar!( - r#"SELECT email FROM usr WHERE workspace_id = $1 AND username = $2"#, - &w_id, - username, - ) - .fetch_optional(&db) - .await?; - match email { + match resolve_username_to_email(&w_id, username, &db).await? { Some(e) => Some(e), - // The `admins` workspace has no `usr` rows (username IS the email - // there), so accept it as the owner email directly. - None if w_id == "admins" => Some(username.clone()), None => { return Err(Error::NotFound(format!( "no user with username {username} in workspace" @@ -805,7 +803,6 @@ async fn require_can_read_path( Err(Error::NotFound(format!("no draft visible at {path}"))) } - #[cfg(test)] mod tests { use super::strip_json_nul; @@ -841,7 +838,9 @@ mod tests { // "a" carries a real NUL; "b" carries the literal text backslash-u0000. // The value walk strips the former and leaves the latter intact — the // pathological case that needed a fallback in SQL is trivial in Rust. - let v = parsed(strip_json_nul(r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string())); + let v = parsed(strip_json_nul( + r#"{"a":"x\u0000y","b":"p\\u0000q"}"#.to_string(), + )); assert_eq!(v["a"], "xy"); assert_eq!(v["b"], "p\\u0000q"); } diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index 23f20459c3..c6d8e397f2 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -87,7 +87,8 @@ pub async fn upload_file_from_req( _file_key: &str, _req: axum::extract::Request, _options: PutMultipartOpts, -) -> error::Result { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) @@ -99,12 +100,77 @@ pub async fn upload_file_internal( _file_key: &str, _stream: impl Stream> + Unpin, _options: PutMultipartOpts, -) -> error::Result<()> { + _max_size: Option, +) -> error::Result<(PutResult, usize)> { Err(error::Error::internal_err( "Not implemented in Windmill's Open Source repository".to_string(), )) } +// These stubs stand in for the CE quota helpers in a pure-OSS build; their only +// callers (apps.rs / args.rs uploads) are `not(enterprise)`, so gate them the +// same way — an enterprise-without-private build compiles neither. +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_storage_quota_remaining( + _db: &DB, + _w_id: &str, + _exclude_upload_id: Option<&str>, +) -> error::Result { + Ok(i64::MAX) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn reject_reserved_volume_key(_file_key: &str) -> error::Result<()> { + Ok(()) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub struct CeUploadBudget { + pub max_size: usize, + pub existing_size: i64, +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn ce_upload_budget( + _db: &DB, + _w_id: &str, + _s3_client: &Arc, + _file_key: &str, + _content_length: Option, +) -> error::Result { + Ok(CeUploadBudget { max_size: usize::MAX, existing_size: 0 }) +} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub async fn bump_storage_usage(_db: &DB, _w_id: &str, _storage: &str, _delta: i64) {} + +#[cfg(all( + feature = "parquet", + not(feature = "private"), + not(feature = "enterprise") +))] +pub fn spawn_storage_usage_recount_floored(_db: &DB, _w_id: &str) {} + #[cfg(all(feature = "parquet", not(feature = "private")))] pub async fn download_s3_file_internal( _authed: OptJobAuthed, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 5192d27724..14cbb76c91 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7602,26 +7602,32 @@ async fn run_dynamic_select( )); } + if !is_valid_entrypoint_name(&request.entrypoint_function) { + return Err(error::Error::BadRequest(format!( + "Invalid entrypoint_function {:?}: must match \ + ^[A-Za-z_][A-Za-z0-9_]*$ (letters, digits and underscores, \ + not starting with a digit)", + request.entrypoint_function + ))); + } + + // Deployed scripts keep their normal deployed-run path (a `script` job, resolved below by + // push_script_job_by_path_into_queue). Deployed flows and inline snippets have no deployed + // runnable, so they run their dyn-select code as a `preview`; the flow carries its path and + // worker tag so its option-fetching job lines up with the script run. let dynamic_input: DynamicInput; + let runnable_path: Option; + let mut tag: Option = None; match request.runnable_ref { DynamicSelectRunnableRef::Deployed { path, runnable_kind } => match runnable_kind { RunnableKind::Script => { - if !is_valid_entrypoint_name(&request.entrypoint_function) { - return Err(error::Error::BadRequest(format!( - "Invalid entrypoint_function {:?}: must match \ - ^[A-Za-z_][A-Za-z0-9_]*$ (letters, digits and underscores, \ - not starting with a digit)", - request.entrypoint_function - ))); - } let mut script_args = request.args.unwrap_or_default(); script_args.insert( - "_ENTRYPOINT_OVERRIDE".to_string(), + ENTRYPOINT_OVERRIDE.to_string(), serde_json::value::to_raw_value(&request.entrypoint_function)?, ); - - let push_args = PushArgsOwned { extra: None, args: script_args.clone() }; + let push_args = PushArgsOwned { extra: None, args: script_args }; let (uuid, _, _) = push_script_job_by_path_into_queue( authed.clone(), @@ -7631,7 +7637,7 @@ async fn run_dynamic_select( w_id.clone(), StripPath(path), run_query.clone(), - push_args.clone(), + push_args, None, ) .await?; @@ -7639,13 +7645,40 @@ async fn run_dynamic_select( return Ok((StatusCode::CREATED, uuid.to_string()).into_response()); } RunnableKind::Flow => { - // Runs the deployed flow's dynamic-select code. Enforce the same - // path-scoped check the script branch gets via - // push_script_job_by_path_into_queue, so a token not scoped to this - // flow cannot trigger its code through dynamic select. + // Runs the deployed flow's dynamic-select code. Path-scoped so a token not + // scoped to this flow cannot trigger its code through dynamic select. check_scopes(&authed, || format!("jobs:run:flows:{path}"))?; let mut conn = user_db.clone().begin(&authed).await?; + // Read the flow's tag under RLS. This runs on every request (including the + // cache hit below), so it doubles as the access check and routes the + // option-fetching preview to the flow's worker group, matching the script branch. + let Some(flow_tag) = sqlx::query_scalar!( + "SELECT tag FROM flow WHERE workspace_id = $1 AND path = $2", + &w_id, + &path + ) + .fetch_optional(&mut *conn) + .await? + else { + conn.commit().await?; + let exists = sqlx::query_scalar!( + "SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2 LIMIT 1", + &w_id, + &path + ) + .fetch_optional(&db) + .await? + .is_some(); + if exists { + return Err(error::Error::NotAuthorized(format!( + "You are not authorized to access this flow: {path}" + ))); + } + return Err(Error::NotFound(format!("Flow not found at path {path}"))); + }; + tag = flow_tag; + let dynamic_input_res = match DYNAMIC_INPUT_CACHE.get(&format!("{}:{}", w_id, path)) { Some(cached) => cached.as_ref().clone(), @@ -7688,21 +7721,34 @@ async fn run_dynamic_select( conn.commit().await?; dynamic_input = dynamic_input_res; + runnable_path = Some(path); } }, DynamicSelectRunnableRef::Inline { code, lang: language } => { // Inline dynamic select runs arbitrary, request-supplied code; require the broad // jobs:run scope so a narrowly-scoped token cannot escape its scope. The Deployed - // branches are path-scoped instead (scripts via push_script_job_by_path_into_queue, - // flows via the check_scopes above). + // branches are path-scoped instead. check_scopes(&authed, || format!("jobs:run"))?; dynamic_input = DynamicInput { x_windmill_dyn_select_code: code, x_windmill_dyn_select_lang: language.unwrap_or_default(), }; + runnable_path = None; } } + // Same tag-permission gate a normal run gets (run_flow / push_script_job_by_path_into_queue): + // a caller allowed to read the flow must still be allowed to use its worker tag. No-op for + // inline (tag is None); the script branch checked this inside its helper and returned above. + check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; + + // Invoke the dyn-select entrypoint instead of `main`. + let mut args = request.args.unwrap_or_default(); + args.insert( + ENTRYPOINT_OVERRIDE.to_string(), + serde_json::value::to_raw_value(&request.entrypoint_function)?, + ); + let scheduled_for = run_query.get_scheduled_for(&db).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); @@ -7713,7 +7759,7 @@ async fn run_dynamic_select( JobPayload::Code(RawCode { hash: None, content: dynamic_input.x_windmill_dyn_select_code, - path: None, + path: runnable_path, language: dynamic_input.x_windmill_dyn_select_lang, lock: None, cache_ttl: None, @@ -7722,9 +7768,11 @@ async fn run_dynamic_select( concurrency_settings: ConcurrencySettings::default().into(), debouncing_settings: DebouncingSettings::default(), modules: None, + // RawCode.tag is ignored by the queue path (`JobPayload::Code` destructures it as + // `tag: _`); the effective tag is the `tag` argument to `push` below. tag: None, }), - PushArgs::from(&request.args.unwrap_or_default()), + PushArgs::from(&args), authed.display_username(), &authed.email, username_to_permissioned_as(&authed.username), @@ -7739,7 +7787,8 @@ async fn run_dynamic_select( false, None, true, - None, + // Deployed flow → the flow's worker tag; inline → `None` (language default). + tag, run_query.timeout, None, None, diff --git a/backend/windmill-api/src/mcp/utils.rs b/backend/windmill-api/src/mcp/utils.rs index 2a0c34ae8d..2184c1548f 100644 --- a/backend/windmill-api/src/mcp/utils.rs +++ b/backend/windmill-api/src/mcp/utils.rs @@ -382,12 +382,18 @@ pub fn build_query_string( .map(|value| { // Use the original name for the query parameter key let original_name = get_original_name(param_name, query_field_renames); - let value_str = value.to_string(); - let str_val = value_str.trim_matches('"'); + // For string values, use the raw content: to_string() would JSON-encode + // it, and stripping the outer quotes leaves inner quotes backslash-escaped + // (e.g. `{\"k\":\"v\"}`), which breaks downstream JSON parsing of params + // like `args`/`result`. Non-string values keep their JSON serialization. + let str_val = value + .as_str() + .map(|s| s.to_string()) + .unwrap_or_else(|| value.to_string()); format!( "{}={}", urlencoding::encode(&original_name), - urlencoding::encode(str_val) + urlencoding::encode(&str_val) ) }) }) @@ -624,4 +630,55 @@ mod tests { .expect("legitimate path should substitute"); assert_eq!(result, "/w/dev/scripts/get/p/u/alice/my_script"); } + + fn single_query_schema(param: &str) -> Option { + Some(json!({ + "type": "object", + "properties": { param: { "type": "string" } } + })) + } + + #[test] + fn build_query_string_preserves_json_string_content() { + // A string param carrying JSON (e.g. the `args` filter on listJobs) must be + // emitted as its raw content so the backend can `serde_json::from_str` it. + let mut args = serde_json::Map::new(); + args.insert("args".to_string(), json!("{\"key\":\"val\"}")); + + let qs = build_query_string(&args, &single_query_schema("args"), &None); + + // No backslash escaping: %5C must not appear; the encoded braces/quotes are exact. + assert_eq!(qs, "?args=%7B%22key%22%3A%22val%22%7D"); + assert!( + !qs.contains("%5C"), + "must not contain backslash escapes: {qs}" + ); + } + + #[test] + fn build_query_string_keeps_non_string_serialization() { + let mut args = serde_json::Map::new(); + args.insert("per_page".to_string(), json!(42)); + assert_eq!( + build_query_string(&args, &single_query_schema("per_page"), &None), + "?per_page=42" + ); + + let mut args = serde_json::Map::new(); + args.insert("running".to_string(), json!(true)); + assert_eq!( + build_query_string(&args, &single_query_schema("running"), &None), + "?running=true" + ); + } + + #[test] + fn build_query_string_encodes_plain_string() { + let mut args = serde_json::Map::new(); + args.insert("path".to_string(), json!("u/alice/my script")); + assert_eq!( + build_query_string(&args, &single_query_schema("path"), &None), + "?path=u%2Falice%2Fmy%20script" + ); + } } diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index c186f59141..0757ef6894 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -10,6 +10,41 @@ pub use windmill_parser::asset_parser::{ }; pub use windmill_types::assets::*; +// --- Workspace DuckDB macro registry cache (worker hot path) --- +// Every DuckDB job reads the `macro_definition` registry; cascades can run +// many jobs per second. Primary invalidation is the +// `notify_macro_registry_change` event, emitted transactionally with every +// registry mutation and dispatched to this cache in main.rs — the TTL is a +// backstop for mutation paths that don't emit (manual SQL edits). + +#[derive(Clone, Debug, sqlx::FromRow)] +pub struct MacroRegistryEntry { + pub name: String, + pub params: String, + pub body: String, + pub is_table_macro: bool, + pub provider_path: String, +} + +#[derive(Clone)] +pub struct ExpiringMacroRegistry { + pub rows: std::sync::Arc>, + pub expires_at: std::time::Instant, +} + +pub const MACRO_REGISTRY_TTL: std::time::Duration = std::time::Duration::from_secs(60); + +// Tests sharing one process across several databases must disable the cache: +// it is keyed by workspace id alone, and a notify from one DB can't evict +// entries populated from another (same hazard as DEPLOYED_SCRIPT_CACHE_DISABLED). +pub static MACRO_REGISTRY_CACHE_DISABLED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +lazy_static::lazy_static! { + pub static ref MACRO_REGISTRY_CACHE: quick_cache::sync::Cache = + quick_cache::sync::Cache::new(1000); +} + #[derive(sqlx::Type, Debug, Clone, Copy, PartialEq)] #[sqlx(type_name = "SCRIPT_TRIGGER_KIND", rename_all = "lowercase")] pub enum ScriptTriggerKind { diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index 5c1cedc6d8..bd6bf1ef9d 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -413,9 +413,18 @@ async fn fetch_authed_from_permissioned_as_inner( }) } } else { + // Bare (no `u/`|`g/` prefix) permissioned_as is reached for superadmins + // whose identifier is their email (they are not a workspace member). Use + // the instance-derived username when available so no email leaks + // downstream as the acting username. + let username = if is_super_admin && permissioned_as == email { + crate::usernames::get_instance_username_or_fallback_to_email(&mut *conn, email).await? + } else { + permissioned_as.to_string() + }; Ok(Authed { email: email.to_string(), - username: permissioned_as.to_string(), + username, is_admin: is_super_admin, is_operator: true, groups: vec![], diff --git a/backend/windmill-common/src/ee_oss.rs b/backend/windmill-common/src/ee_oss.rs index e68bc00c6e..26b9c87b7e 100644 --- a/backend/windmill-common/src/ee_oss.rs +++ b/backend/windmill-common/src/ee_oss.rs @@ -20,6 +20,7 @@ lazy_static::lazy_static! { pub static ref LICENSE_KEY: arc_swap::ArcSwap = arc_swap::ArcSwap::from_pointee("".to_string()); pub static ref LICENSE_OFFLINE_METADATA: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref LICENSE_OFFLINE_OVER_CU_CAP: AtomicBool = AtomicBool::new(false); + pub static ref LICENSE_OFFLINE_OVER_SEAT_CAP: AtomicBool = AtomicBool::new(false); pub static ref LICENSE_OFFLINE_LAST_STATUS: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(None); pub static ref LICENSE_OFFLINE_LAST_CHECKED_AT: arc_swap::ArcSwap>> = arc_swap::ArcSwap::from_pointee(None); } @@ -62,6 +63,14 @@ pub async fn check_seat_cap_for_new_user( Ok(None) } +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub async fn check_seat_cap_for_reactivation( + _db: &DB, + _email: &str, +) -> anyhow::Result> { + Ok(None) +} + #[cfg(all(feature = "enterprise", not(feature = "private")))] pub async fn compute_instance_hash(_db: &DB) -> anyhow::Result> { // Implementation is not open source diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index be336a84e1..baa3e4b500 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -268,6 +268,10 @@ lazy_static::lazy_static! { pub static ref INSTANCE_NAME: String = rd_string(5); pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + // Latest non-archived version per (workspace, path) for bundle cache keying — + // looser predicate than DEPLOYED_SCRIPT_HASH_CACHE (no lock requirement), so + // the two must not share entries. See get_latest_script_hash_for_import_cached. + pub static ref IMPORTED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); pub static ref DYNAMIC_INPUT_CACHE: Cache> = Cache::new(1000); pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); @@ -564,6 +568,132 @@ mod validate_dbname_tests { } } +#[cfg(test)] +mod pg_tls_tests { + use super::PgDatabase; + + // A syntactically valid (self-signed) certificate, used only to exercise the + // "root certificate supplied" branch — its contents are never validated here. + const VALID_PEM: &str = "-----BEGIN CERTIFICATE-----\n\ +MIIDETCCAfmgAwIBAgIUX/yHsMoWBljFzJr5Xh7V2I6ykMEwDQYJKoZIhvcNAQEL\n\ +BQAwGDEWMBQGA1UEAwwNd2luZG1pbGwtdGVzdDAeFw0yNjA2MjkwOTUwNTlaFw0z\n\ +NjA2MjYwOTUwNTlaMBgxFjAUBgNVBAMMDXdpbmRtaWxsLXRlc3QwggEiMA0GCSqG\n\ +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvF2hMw8adQGG6EnDk8GsOIoHT+kLN1W0F\n\ +yYFwH1wGVmzVP1YNfUts8aQfMtl/ZjW7SQlvKeK+18id4fVNYvZpbFhj66IsKMOU\n\ +MnJHcC6X/IAdhANyhM1fcrS6YupanAKOhLPk4HYRD5tGI4Y1vzTnQKGffIZ0bof7\n\ +3GtCiJLv8wrJKszeoKPtdFazdW+CYePbFq3Owc7HMo8CwA7A5TsgcowELhCfYwZv\n\ +Pn/9v+NDHQO0jJclH7qK221RkbqZGD+nPJ4rUm7oRi0vfApBQZ0FFJZjiki/Kg2+\n\ +RACb6Ud/LOeRBerKQHbN8KeYnGafCaIC4s/XytVwxAz+kgK1qyl7AgMBAAGjUzBR\n\ +MB0GA1UdDgQWBBRo2Jby4SZlrwMNbhA4bswZcBNRyjAfBgNVHSMEGDAWgBRo2Jby\n\ +4SZlrwMNbhA4bswZcBNRyjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\n\ +A4IBAQBlED+FQW3GB3Wa1NdVN252vihuFNnbq81yvhf4T7dfAxwkxI9jiM+ZWCw2\n\ +g59FbLupj8Rwun5gE2H/9M8ZunISdlwaMH5nyDJlbRjttPfY1cEoyGEY+UXIslfg\n\ +BoiI5rOtz9R2qurxEic1VtEVfXhEuWwCG86vCBDdHrL/qqqUJEx/P8qyC7uVc8XC\n\ +uclnJVL7x1ax0jTmEPur9K+DQn2ws01mzpq2QwSunibpDL5D5xM1oYekv0tQFEkT\n\ +ta9ELulniZau8zUAtwqwecxodzl+KO8NYj0a9PGgAM64dMqkRtRA8P4UP350Nag3\n\ ++hOq1qpWD7yPVyycx/KCilICOKVf\n\ +-----END CERTIFICATE-----\n"; + + fn pg(sslmode: Option<&str>, root_cert: Option<&str>) -> PgDatabase { + PgDatabase { + host: "db.example.com".to_string(), + user: Some("u".to_string()), + password: Some("p".to_string()), + port: Some(5432), + sslmode: sslmode.map(|s| s.to_string()), + dbname: "mydb".to_string(), + root_certificate_pem: root_cert.map(|s| s.to_string()), + accept_invalid_certs: None, + use_iam_auth: None, + region: None, + } + } + + /// Whether the connector enforces certificate verification for the given config. + fn verifies( + sslmode: Option<&str>, + root_cert: Option<&str>, + accept_invalid_certs: Option, + ) -> bool { + let mut builder = native_tls::TlsConnector::builder(); + PgDatabase::configure_pg_tls_verification( + &mut builder, + sslmode, + root_cert, + accept_invalid_certs, + ) + .unwrap() + } + + #[test] + fn verify_modes_enforce_verification_when_explicitly_requested() { + // accept_invalid_certs=Some(false) is what newly created resources carry: it + // verifies even with no custom cert (against the OS trust store). + assert!(verifies(Some("verify-full"), None, Some(false))); + assert!(verifies(Some("verify-ca"), None, Some(false))); + assert!(verifies(Some("verify-full"), Some(""), Some(false))); + assert!(verifies(Some("verify-full"), Some(VALID_PEM), Some(false))); + assert!(verifies(Some("verify-ca"), Some(VALID_PEM), Some(false))); + } + + #[test] + fn verify_modes_unset_fall_back_to_legacy_behavior() { + // Unset (None): verify iff a root cert is present — preserves the behavior of + // resources that predate the flag (incl. git-synced), so upgrades don't break. + assert!(!verifies(Some("verify-full"), None, None)); + assert!(!verifies(Some("verify-ca"), None, None)); + assert!(!verifies(Some("verify-full"), Some(""), None)); + assert!(verifies(Some("verify-full"), Some(VALID_PEM), None)); + assert!(verifies(Some("verify-ca"), Some(VALID_PEM), None)); + } + + #[test] + fn accept_invalid_certs_true_disables_verification_for_verify_modes() { + assert!(!verifies(Some("verify-full"), Some(VALID_PEM), Some(true))); + assert!(!verifies(Some("verify-ca"), None, Some(true))); + } + + #[test] + fn accept_invalid_certs_is_ignored_outside_verify_modes() { + // require never consults the flag: it verifies iff a cert is present, and + // encrypts-without-verifying otherwise, regardless of accept_invalid_certs. + assert!(!verifies(Some("require"), None, Some(false))); + assert!(!verifies(Some("require"), None, Some(true))); + assert!(!verifies(None, None, Some(true))); + assert!(verifies(Some("require"), Some(VALID_PEM), Some(true))); + assert!(verifies(Some("require"), Some(VALID_PEM), None)); + } + + #[test] + fn invalid_pem_is_rejected() { + let mut builder = native_tls::TlsConnector::builder(); + let err = PgDatabase::configure_pg_tls_verification( + &mut builder, + Some("verify-full"), + Some("not a certificate"), + Some(false), + ); + assert!(err.is_err()); + } + + #[test] + fn to_uri_collapses_verify_modes_for_tokio_postgres() { + // to_uri() feeds tokio-postgres, which only parses disable/prefer/require; + // verify-* therefore map to require there (verification is connector-driven). + for mode in ["require", "verify-ca", "verify-full"] { + assert!( + pg(Some(mode), None).to_uri().contains("sslmode=require"), + "{mode} should map to sslmode=require in to_uri()" + ); + } + assert!(pg(Some("disable"), None) + .to_uri() + .contains("sslmode=disable")); + assert!(pg(Some("allow"), None).to_uri().contains("sslmode=prefer")); + assert!(pg(None, None).to_uri().contains("sslmode=prefer")); + } +} + #[derive(Serialize, Debug)] pub struct PrepareQueryColumnInfo { pub name: String, @@ -588,6 +718,12 @@ pub struct PgDatabase { pub sslmode: Option, pub dbname: String, pub root_certificate_pem: Option, + /// Only meaningful for sslmode verify-ca/verify-full. `Some(true)` accepts any + /// server certificate (no chain or hostname check); `Some(false)` enforces + /// verification. `None` falls back to legacy behavior — verify only when a root + /// certificate is present — so resources that predate this flag (including + /// git-synced ones, whose source never sets it) keep working unchanged. + pub accept_invalid_certs: Option, pub use_iam_auth: Option, pub region: Option, } @@ -676,10 +812,94 @@ impl PgDatabase { } } + /// True when sslmode requests verification (verify-ca/verify-full) but the + /// effective configuration disables it, so the server's identity is not + /// checked. Mirrors the verify-* decision in `configure_pg_tls_verification`. + pub fn verify_mode_skips_verification(&self) -> bool { + matches!( + self.sslmode.as_deref(), + Some("verify-ca") | Some("verify-full") + ) && self.accept_invalid_certs.unwrap_or( + self.root_certificate_pem + .as_deref() + .unwrap_or("") + .is_empty(), + ) + } + + /// Configure certificate and hostname verification on a native-tls connector + /// according to the requested Postgres `sslmode`. The crates.io tokio-postgres + /// build only parses disable/prefer/require, so verify-ca and verify-full are + /// enforced here, on the connector, rather than through the connection URI. + /// + /// verify-full — verify the certificate chain AND that it matches the host. + /// verify-ca — verify the chain only; libpq does not check the hostname. + /// require / other — encrypt without verifying identity, unless a root + /// certificate is supplied (then verify the chain). + /// + /// `accept_invalid_certs` only applies to verify-ca/verify-full: `Some(true)` + /// accepts any certificate, `Some(false)` enforces verification, and `None` + /// falls back to the legacy behavior — verify only when a root certificate is + /// present — so resources predating the flag (including git-synced ones, whose + /// source never sets it) keep working unchanged. Verification uses the OS trust + /// store plus any supplied root certificate. Returns false when the connector + /// was set to accept any certificate, so callers can surface that an unverified + /// connection is being made. + fn configure_pg_tls_verification( + builder: &mut native_tls::TlsConnectorBuilder, + sslmode: Option<&str>, + root_certificate_pem: Option<&str>, + accept_invalid_certs: Option, + ) -> Result { + use native_tls::Certificate; + + let custom_root = match root_certificate_pem { + Some(pem) if !pem.is_empty() => Some( + Certificate::from_pem(pem.as_bytes()) + .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, + ), + _ => None, + }; + + match sslmode { + Some("verify-full") | Some("verify-ca") => { + // Unset falls back to the legacy behavior: verify iff a cert is present. + if accept_invalid_certs.unwrap_or(custom_root.is_none()) { + builder + .danger_accept_invalid_certs(true) + .danger_accept_invalid_hostnames(true); + return Ok(false); + } + if let Some(cert) = custom_root { + builder.add_root_certificate(cert); + } + if sslmode == Some("verify-ca") { + // verify-ca verifies the chain but, per libpq, not the hostname. + builder.danger_accept_invalid_hostnames(true); + } + Ok(true) + } + _ => { + // "require": accept_invalid_certs does not apply. Encrypt but do not + // verify identity, unless an explicit root certificate was supplied + // (then verify the chain). + if let Some(cert) = custom_root { + builder.add_root_certificate(cert); + Ok(true) + } else { + builder + .danger_accept_invalid_certs(true) + .danger_accept_invalid_hostnames(true); + Ok(false) + } + } + } + } + async fn connect_inner( &self, ) -> Result<(tokio_postgres::Client, TokioPgConnection), error::Error> { - use native_tls::{Certificate, TlsConnector}; + use native_tls::TlsConnector; use postgres_native_tls::MakeTlsConnector; use tokio_postgres::tls::NoTls; let ssl_mode_is_require = matches!( @@ -690,21 +910,17 @@ impl PgDatabase { if ssl_mode_is_require { tracing::info!("Creating new connection"); let mut connector = TlsConnector::builder(); - if let Some(root_certificate_pem) = &self.root_certificate_pem { - if !root_certificate_pem.is_empty() { - connector.add_root_certificate( - Certificate::from_pem(root_certificate_pem.as_bytes()).map_err(|e| { - error::Error::BadConfig(format!("Invalid Certs: {e:#}")) - })?, - ); - } else { - connector.danger_accept_invalid_certs(true); - connector.danger_accept_invalid_hostnames(true); - } - } else { - connector - .danger_accept_invalid_certs(true) - .danger_accept_invalid_hostnames(true); + Self::configure_pg_tls_verification( + &mut connector, + self.sslmode.as_deref(), + self.root_certificate_pem.as_deref(), + self.accept_invalid_certs, + )?; + if self.verify_mode_skips_verification() { + tracing::warn!( + "Postgres connection with sslmode={} is not verifying the server certificate (accept_invalid_certs is set, or no root certificate is configured and the resource predates that flag). Set accept_invalid_certs=false or provide root_certificate_pem to enforce verification.", + self.sslmode.as_deref().unwrap_or("") + ); } let (client, connection) = tokio::time::timeout( @@ -759,23 +975,16 @@ impl PgDatabase { error::Error::InternalErr(format!("IAM token generation failed: {e:#}")) })?; - // RDS IAM auth requires SSL + // RDS IAM auth requires SSL. let mut connector = TlsConnector::builder(); - if let Some(root_certificate_pem) = &self.root_certificate_pem { - if !root_certificate_pem.is_empty() { - connector.add_root_certificate( - native_tls::Certificate::from_pem(root_certificate_pem.as_bytes()) - .map_err(|e| error::Error::BadConfig(format!("Invalid Certs: {e:#}")))?, - ); - } else { - connector.danger_accept_invalid_certs(true); - connector.danger_accept_invalid_hostnames(true); - } - } else { - tracing::warn!("IAM RDS auth without root certificate: TLS certificate verification is disabled. Consider providing root_certificate_pem for production use."); - connector - .danger_accept_invalid_certs(true) - .danger_accept_invalid_hostnames(true); + let verified = Self::configure_pg_tls_verification( + &mut connector, + self.sslmode.as_deref(), + self.root_certificate_pem.as_deref(), + self.accept_invalid_certs, + )?; + if !verified { + tracing::warn!("IAM RDS auth without certificate verification: TLS certificate verification is disabled. Provide root_certificate_pem (and set sslmode=verify-full) to enforce verification."); } tracing::info!("Creating new IAM RDS connection to {}", &self.host); @@ -840,6 +1049,7 @@ impl PgDatabase { dbname, sslmode, root_certificate_pem: None, + accept_invalid_certs: None, use_iam_auth: None, region: None, }) @@ -1396,6 +1606,46 @@ pub async fn get_latest_script_hash<'e, E: sqlx::PgExecutor<'e>>( return Ok(hash); } +/// Latest non-archived hash for an imported `path`, for bundle cache keying. +/// MUST select the same row as the bundler's content endpoint +/// (`raw_script_by_path_internal`: `archived = false ORDER BY created_at DESC`, +/// no lock predicate) — a stricter filter here would let the key point at an +/// older version than the content that gets inlined. Cached with the same +/// freshness contract as that endpoint's `RAW_SCRIPT_LATEST_HASH_CACHE`: +/// evicted by `notify_runnable_version_change` events, 60s TTL fallback. +pub async fn get_latest_script_hash_for_import_cached( + db: &DB, + w_id: &str, + script_path: &str, +) -> error::Result> { + let use_cache = !DEPLOYED_SCRIPT_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + let cache_key = (w_id.to_string(), script_path.to_string()); + if use_cache { + if let Some(cached) = IMPORTED_SCRIPT_HASH_CACHE.get(&cache_key) { + if cached.expires_at > std::time::Instant::now() { + return Ok(Some(cached.id)); + } + } + } + let hash = sqlx::query_scalar!( + "SELECT hash FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(db) + .await?; + if let (true, Some(hash)) = (use_cache, hash) { + IMPORTED_SCRIPT_HASH_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: hash, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + } + Ok(hash) +} + pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>( db_authed: Option>>, db: E, diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index 620cbe2a5f..7e0c1f3f55 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -237,18 +237,22 @@ async fn fetch_other_drafts_users( kind: UserDraftItemKind, path: &str, ) -> Result> { - // The `admins` workspace has no `usr` rows (username IS the email there), - // so fall back to `d.email` when the join misses, else a real teammate's - // draft renders as a phantom "Legacy draft". The genuine NULL-email legacy - // row keeps `username = None` (its `d.email` is NULL, so the CASE yields NULL). + // A superadmin authoring in a workspace they are not a member of has no `usr` + // row: fall back to their instance-derived username (`password.username`), or + // their email when derivation is disabled. Else a real teammate's draft renders + // as a phantom "Legacy draft". The genuine NULL-email legacy row keeps + // `username = None` (no `usr`/`password` match and `d.email` is NULL). let rows = sqlx::query_as!( OtherDraftUser, - r#"SELECT COALESCE(u.username, CASE WHEN d.workspace_id = 'admins' THEN d.email END) as "username?", + r#"SELECT COALESCE(u.username, p.username, CASE WHEN p.email IS NOT NULL THEN d.email END) as "username?", d.created_at as "draft_saved_at!" FROM draft d LEFT JOIN usr u ON u.workspace_id = d.workspace_id AND u.email = d.email + LEFT JOIN password p + ON p.email = d.email + AND p.super_admin = true WHERE d.workspace_id = $1 AND d.path = $2 AND d.typ = $3 diff --git a/backend/windmill-common/src/usernames.rs b/backend/windmill-common/src/usernames.rs index 2501841cac..6fde2a8186 100644 --- a/backend/windmill-common/src/usernames.rs +++ b/backend/windmill-common/src/usernames.rs @@ -94,6 +94,38 @@ pub async fn generate_instance_username_for_all_users(db: &DB) -> error::Result< Ok(()) } +/// Resolve the username to use for a user (typically a superadmin) accessing a +/// workspace they are not a member of. When instance-wide username derivation is +/// enabled (`automate_username_creation`, the default on almost all instances), +/// `password.username` is populated, so we return that derived username instead +/// of leaking the raw email address downstream (as job/script author, audit +/// actor, etc.). Falls back to the email when no derived username exists (the few +/// instances that keep `automate_username_creation` disabled). +/// +/// The hot callers (token auth, `fetch_api_authed`) are already behind their own +/// 120s auth caches, so this deliberately does not add another email->username +/// cache (which would only help the cold JWT-mint / job-perms paths while risking +/// cross-DB contamination in the shared-process integration tests). +/// +/// A DB error is propagated rather than swallowed into the email fallback: falling +/// back to the email on a transient failure would reintroduce the very email leak +/// this path exists to prevent, so callers fail closed instead. +pub async fn get_instance_username_or_fallback_to_email<'e, E>( + db: E, + email: &str, +) -> error::Result +where + E: sqlx::PgExecutor<'e>, +{ + let derived = sqlx::query_scalar!("SELECT username FROM password WHERE email = $1", email) + .fetch_optional(db) + .await? + .flatten(); + // No derived username (automate_username_creation disabled) → fall back to the + // email. This is the only legitimate fallback; a query error propagates above. + Ok(derived.unwrap_or_else(|| email.to_string())) +} + pub async fn get_instance_username_or_create_pending<'c>( tx: &mut Transaction<'c, Postgres>, email: &str, diff --git a/backend/windmill-common/src/users.rs b/backend/windmill-common/src/users.rs index d40b5cd19c..738dc84d14 100644 --- a/backend/windmill-common/src/users.rs +++ b/backend/windmill-common/src/users.rs @@ -46,8 +46,34 @@ lazy_static::lazy_static! { const EMAIL_CACHE_TTL_SECS: u64 = 60; +/// Resolve a workspace-scoped username to its email. +/// +/// Members are found in `usr`. A superadmin acting in a workspace they are *not* +/// a member of has no `usr` row; they carry either their instance-derived +/// username (`password.username`, when `automate_username_creation` is enabled) +/// or their email (when it is disabled), so fall back to `password` on both, +/// gated on `super_admin` since only superadmins can act without membership. +/// Returns `None` when the username resolves to nobody. +pub async fn resolve_username_to_email<'c>( + workspace_id: &str, + username: &str, + db: impl sqlx::PgExecutor<'c>, +) -> crate::error::Result> { + Ok(sqlx::query_scalar!( + "SELECT COALESCE( + (SELECT email FROM usr WHERE workspace_id = $1 AND username = $2), + (SELECT email FROM password WHERE (username = $2 OR email = $2) AND super_admin = true) + )", + workspace_id, + username + ) + .fetch_optional(db) + .await? + .flatten()) +} + /// Get email from permissioned_as string. -/// - "u/{username}" → lookup email from usr table (cached) +/// - "u/{username}" → resolve via [`resolve_username_to_email`] (cached) /// - "g/{group}" → "group-{group}@windmill.dev" /// - raw email → return as-is pub async fn get_email_from_permissioned_as<'c>( @@ -62,14 +88,9 @@ pub async fn get_email_from_permissioned_as<'c>( return Ok(email); } } - let email = sqlx::query_scalar!( - "SELECT email FROM usr WHERE username = $1 AND workspace_id = $2", - username, - workspace_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| format!("{}@unknown.windmill.dev", username)); + let email = resolve_username_to_email(workspace_id, username, db) + .await? + .unwrap_or_else(|| format!("{}@unknown.windmill.dev", username)); let key = (workspace_id.to_string(), username.to_string()); EMAIL_CACHE.insert(key, (email.clone(), std::time::Instant::now())); Ok(email) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index cebbf729be..969f3897e4 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -180,7 +180,37 @@ pub const WM_FORK_PREFIX: &str = "wm-fork-"; /// layer because the actual branch creation runs in a deferred git-sync worker job — without /// this check, the API returns 200 and the failure only surfaces later in the worker. pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> { - if !id.starts_with(WM_FORK_PREFIX) { + validate_workspace_branch_id(id, true) +} + +/// Like [`validate_fork_workspace_id`] but does not require the `wm-fork-` prefix. Used for dev +/// workspaces, whose id is an ordinary (prefix-less) workspace id but must still be git-branch-safe +/// because it is interpolated into a `wm-fork//` branch name like any fork. +pub fn validate_dev_workspace_id(id: &str) -> error::Result<()> { + validate_workspace_branch_id(id, false) +} + +/// The `workspace.name` column is `character varying(50)`, so a name longer than 50 characters +/// triggers a raw `value too long for type character varying(50)` SQL error on insert. Validate +/// up front to return a clear message instead. +pub fn validate_workspace_name(name: &str) -> error::Result<()> { + if name.chars().count() > 50 { + return Err(Error::BadRequest(format!( + "Workspace name is too long ({} chars). Maximum length is 50 characters.", + name.chars().count() + ))); + } + Ok(()) +} + +fn validate_workspace_branch_id(id: &str, require_fork_prefix: bool) -> error::Result<()> { + if id.is_empty() { + return Err(Error::BadRequest( + "Workspace id cannot be empty".to_string(), + )); + } + + if require_fork_prefix && !id.starts_with(WM_FORK_PREFIX) { return Err(Error::BadRequest(format!( "The id `{}` is invalid for a forked workspace. It should be prefixed by {}", id, WM_FORK_PREFIX @@ -189,8 +219,9 @@ pub fn validate_fork_workspace_id(id: &str) -> error::Result<()> { if id.len() > 50 { return Err(Error::BadRequest(format!( - "Fork workspace id `{}` is too long ({} chars). Maximum length is 50 characters (including the '{}' prefix).", - id, id.len(), WM_FORK_PREFIX + "Workspace id `{}` is too long ({} chars). Maximum length is 50 characters.", + id, + id.len() ))); } @@ -317,9 +348,215 @@ lazy_static::lazy_static! { pub static ref PUBLIC_APP_RATE_LIMIT_CACHE: Cache, i64)> = Cache::new(1000); } +#[cfg(feature = "cloud")] +lazy_static::lazy_static! { + // Maps a workspace id to its root (billing) ancestor. Value: (root_id, expiry_timestamp). + // Reparenting (attach/detach dev) is rare and self-heals via the 60s TTL, so a brief stale + // mapping only mis-attributes usage for <60s across other instances. + pub static ref BILLING_WORKSPACE_CACHE: Cache = Cache::new(5000); +} + +/// Resolve the "billing" workspace for `w_id`: the root ancestor of the fork/dev chain (the +/// workspace whose plan and usage a fork draws from). Returns `w_id` unchanged for a standalone +/// workspace, an unknown id, or a (malformed) cyclic chain. +/// +/// Unauthenticated metering helper: it only reads the parent chain and returns another workspace id, +/// so callers must already be authorized for `w_id` (or run in trusted server-side code); `w_id` is +/// expected to be a server-side id, not raw user input. +#[cfg(feature = "cloud")] +pub async fn get_billing_workspace_id(db: &crate::DB, w_id: &str) -> Result { + let now = chrono::Utc::now().timestamp(); + if let Some((root, expiry)) = BILLING_WORKSPACE_CACHE.get(w_id) { + if expiry > now { + return Ok(root); + } + } + + // The depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so a + // truncated (root-not-found) result — which would fall back to `w_id` and mis-attribute billing — + // is unreachable for any real hierarchy; only a malformed cycle could hit it. + let root = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT id AS "id!" FROM chain WHERE parent_workspace_id IS NULL LIMIT 1 + "#, + w_id + ) + .fetch_optional(db) + .await + .map_err(|e| Error::internal_err(format!("resolving billing workspace for {w_id}: {e:#}")))? + .unwrap_or_else(|| w_id.to_string()); + + BILLING_WORKSPACE_CACHE.insert(w_id.to_string(), (root.clone(), now + 60)); + Ok(root) +} + +/// Invalidate the billing-workspace mapping for a workspace (call after reparenting it). +#[cfg(feature = "cloud")] +pub fn invalidate_billing_workspace_cache(w_id: &str) { + BILLING_WORKSPACE_CACHE.remove(w_id); +} + +/// Invalidate the cached team-plan (premium/past-due) status for a workspace. `TEAM_PLAN_CACHE` has +/// no TTL — it's only evicted by the premium-change NOTIFY — so call this when a workspace id is +/// permanently deleted, otherwise a reused id could inherit the old workspace's premium status. +#[cfg(feature = "cloud")] +pub fn invalidate_team_plan_cache(w_id: &str) { + TEAM_PLAN_CACHE.remove(w_id); +} + +/// Depth of `w_id` in its fork chain: 0 for a root (no parent), 1 for a direct fork, and so on. Walks +/// the parent chain up to the root. The recursion bound is a cycle-safety backstop set well above the +/// enforced `MAX_FORK_DEPTH`; a (malformed) cyclic chain saturates it and so reads as "too deep", +/// which safely rejects rather than allows. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_chain_depth(db: &crate::DB, w_id: &str) -> Result { + let depth = sqlx::query_scalar!( + r#" + WITH RECURSIVE chain AS ( + SELECT id, parent_workspace_id, 0 AS depth + FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.parent_workspace_id, chain.depth + 1 + FROM workspace w + JOIN chain ON w.id = chain.parent_workspace_id + WHERE chain.depth < 20 + ) + SELECT COALESCE(MAX(depth), 0)::bigint AS "depth!" FROM chain + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork depth for {w_id}: {e:#}")))?; + Ok(depth) +} + +/// Height of the fork subtree rooted at `w_id`: 0 when it has no live child forks, 1 with direct +/// children, and so on. Used so that attaching a candidate which already has its own child forks can't +/// push the family past the depth limit. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn fork_subtree_height(db: &crate::DB, w_id: &str) -> Result { + // The `deleted` filter is applied in the outer aggregation (not the recursive step, matching + // count_workspace_forks) so a live descendant under a soft-deleted intermediate is still measured + // at its true depth rather than pruned — otherwise the height could be underestimated and let the + // resulting chain exceed the depth limit. + let height = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COALESCE(MAX(depth) FILTER (WHERE NOT deleted), 0)::bigint AS "height!" FROM tree + "#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("computing fork subtree height for {w_id}: {e:#}")))?; + Ok(height) +} + +/// Ids of every fork/dev workspace anywhere under `w_id` (excludes `w_id` itself), including live +/// descendants beneath a soft-deleted intermediate. Used to invalidate per-workspace caches for a +/// whole subtree after its ancestor is reparented. +/// +/// Unauthenticated helper: reads workspace hierarchy for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +pub async fn list_fork_descendants(db: &crate::DB, w_id: &str) -> Result> { + let ids = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT id AS "id!" FROM tree WHERE id != $1 + "#, + w_id + ) + .fetch_all(db) + .await + .map_err(|e| Error::internal_err(format!("listing fork descendants of {w_id}: {e:#}")))?; + Ok(ids) +} + +/// Count non-deleted fork/dev workspaces anywhere under `root` (excludes `root` itself). +/// +/// Unauthenticated metering helper: it reads workspace hierarchy for any `root` id, so callers must +/// already be authorized for that workspace (or run in trusted server-side code). `root` is expected +/// to be a server-resolved id, never raw user input. +#[cfg(feature = "cloud")] +pub async fn count_workspace_forks(db: &crate::DB, root: &str) -> Result { + // The `deleted` filter is on the outer SELECT (not the recursive step) so that a live sub-fork + // whose intermediate parent was soft-deleted is still counted rather than pruned with it. The + // depth bound is a cycle-safety backstop kept well above the enforced `MAX_FORK_DEPTH`, so + // descendants are never silently dropped from the cap count. + let count = sqlx::query_scalar!( + r#" + WITH RECURSIVE tree AS ( + SELECT id, deleted, 0 AS depth FROM workspace WHERE id = $1 + UNION ALL + SELECT w.id, w.deleted, tree.depth + 1 FROM workspace w + JOIN tree ON w.parent_workspace_id = tree.id + WHERE tree.depth < 20 + ) + SELECT COUNT(DISTINCT id) AS "count!" FROM tree WHERE id != $1 AND NOT deleted + "#, + root + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting forks of {root}: {e:#}")))?; + Ok(count) +} + +/// Approximate paid seats of a workspace as `ceil(developers + operators/2)`, excluding disabled and +/// service-account members. Reuses billing's author/operator weighting, but counts provisioned +/// members rather than the active-user population billing meters, so it only ever loosens the fork +/// cap (never blocks a paid seat) — good enough for a soft guardrail. +/// +/// Unauthenticated metering helper: reads member counts for any `w_id`, so callers must already be +/// authorized for that workspace (or run in trusted server-side code). +#[cfg(feature = "cloud")] +pub async fn count_paid_seats(db: &crate::DB, w_id: &str) -> Result { + let row = sqlx::query!( + r#"SELECT + COUNT(*) FILTER (WHERE NOT operator AND NOT disabled AND NOT is_service_account) AS "developers!", + COUNT(*) FILTER (WHERE operator AND NOT disabled AND NOT is_service_account) AS "operators!" + FROM usr WHERE workspace_id = $1"#, + w_id + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("counting paid seats of {w_id}: {e:#}")))?; + Ok(((row.developers as f64) + 0.5 * (row.operators as f64)).ceil() as i64) +} + #[cfg(feature = "cloud")] pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result { - let cached = TEAM_PLAN_CACHE.get(_w_id); + // A fork/dev workspace draws its plan from the root (billing) workspace. Resolve to the root and + // key the cache by it: the premium-change NOTIFY is keyed by the workspace whose premium row + // changed (the root), so keying by root keeps invalidation correct and lets forks share it. + let billing_w_id = get_billing_workspace_id(_db, _w_id).await?; + let cached = TEAM_PLAN_CACHE.get(&billing_w_id); if let Some(cached) = cached { return Ok(cached); } @@ -338,7 +575,7 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result Result Result, id: &str, parent: Option<&str>, deleted: bool) { + sqlx::query( + "INSERT INTO workspace (id, name, owner, parent_workspace_id, deleted) + VALUES ($1, $1, 'test-user', $2, $3)", + ) + .bind(id) + .bind(parent) + .bind(deleted) + .execute(db) + .await + .expect("insert workspace"); + // The resolver caches per id (60s TTL) in a process-global cache shared across tests, so drop + // any stale mapping for this id before the test reads it. + invalidate_billing_workspace_cache(id); +} + +async fn insert_member( + db: &Pool, + w_id: &str, + email: &str, + operator: bool, + disabled: bool, + is_service_account: bool, +) { + // `usr.username` has a `proper_username` check (no `@`), so derive one from the email prefix. + let username = email.split('@').next().unwrap(); + sqlx::query( + "INSERT INTO usr (workspace_id, email, username, is_admin, operator, disabled, is_service_account, role) + VALUES ($1, $2, $3, false, $4, $5, $6, 'Developer')", + ) + .bind(w_id) + .bind(email) + .bind(username) + .bind(operator) + .bind(disabled) + .bind(is_service_account) + .execute(db) + .await + .expect("insert usr"); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_resolves_to_root(db: Pool) { + insert_ws(&db, "bwt-root", None, false).await; + insert_ws(&db, "bwt-fork", Some("bwt-root"), false).await; + insert_ws(&db, "bwt-grandchild", Some("bwt-fork"), false).await; + + assert_eq!( + get_billing_workspace_id(&db, "bwt-root").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-fork").await.unwrap(), + "bwt-root" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwt-grandchild") + .await + .unwrap(), + "bwt-root" + ); + // Unknown / orphaned ids resolve to themselves. + assert_eq!( + get_billing_workspace_id(&db, "bwt-missing").await.unwrap(), + "bwt-missing" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_workspace_survives_cycles(db: Pool) { + insert_ws(&db, "bwc-a", None, false).await; + insert_ws(&db, "bwc-b", Some("bwc-a"), false).await; + // Introduce a cycle a -> b -> a; no row has a NULL parent, so resolution falls back to the input + // and the depth guard keeps it from looping forever. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'bwc-b' WHERE id = 'bwc-a'") + .execute(&db) + .await + .unwrap(); + invalidate_billing_workspace_cache("bwc-a"); + invalidate_billing_workspace_cache("bwc-b"); + + assert_eq!( + get_billing_workspace_id(&db, "bwc-a").await.unwrap(), + "bwc-a" + ); + assert_eq!( + get_billing_workspace_id(&db, "bwc-b").await.unwrap(), + "bwc-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn paid_seats_and_fork_count(db: Pool) { + insert_ws(&db, "seat-root", None, false).await; + // 2 developers + 2 operators counted -> ceil(2 + 0.5*2) = 3. + insert_member(&db, "seat-root", "dev1@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "dev2@w.dev", false, false, false).await; + insert_member(&db, "seat-root", "op1@w.dev", true, false, false).await; + insert_member(&db, "seat-root", "op2@w.dev", true, false, false).await; + // These must NOT count towards seats. + insert_member(&db, "seat-root", "disabled@w.dev", false, true, false).await; + insert_member(&db, "seat-root", "svc@w.dev", false, false, true).await; + + assert_eq!(count_paid_seats(&db, "seat-root").await.unwrap(), 3); + + insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; + insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; + // A deleted fork itself is not counted... + insert_ws(&db, "seat-fork-deleted", Some("seat-root"), true).await; + // ...but a live sub-fork under it still is (deleted filter is on the outer SELECT, not the walk). + insert_ws(&db, "seat-deleted-child", Some("seat-fork-deleted"), false).await; + // A grandchild fork still counts. + insert_ws(&db, "seat-fork1-child", Some("seat-fork1"), false).await; + + // Live: fork1, fork2, fork1-child, deleted-child -> 4 (seat-fork-deleted excluded). + assert_eq!(count_workspace_forks(&db, "seat-root").await.unwrap(), 4); + // A standalone workspace has no forks. + assert_eq!(count_workspace_forks(&db, "seat-fork2").await.unwrap(), 0); + + // list_fork_descendants returns every descendant id (deleted included, for cache invalidation): + // fork1, fork2, fork-deleted, deleted-child, fork1-child -> 5. + let mut descendants = list_fork_descendants(&db, "seat-root").await.unwrap(); + descendants.sort(); + assert_eq!( + descendants, + vec![ + "seat-deleted-child", + "seat-fork-deleted", + "seat-fork1", + "seat-fork1-child", + "seat-fork2", + ] + ); + assert!(list_fork_descendants(&db, "seat-fork2") + .await + .unwrap() + .is_empty()); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn billing_cache_invalidation_reflects_reparent(db: Pool) { + insert_ws(&db, "inv-root-a", None, false).await; + insert_ws(&db, "inv-root-b", None, false).await; + insert_ws(&db, "inv-fork", Some("inv-root-a"), false).await; + + // Resolve + cache: fork -> root-a. + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // Reparent in the DB, as delete+recreate-under-another-root (or attach) would. + sqlx::query("UPDATE workspace SET parent_workspace_id = 'inv-root-b' WHERE id = 'inv-fork'") + .execute(&db) + .await + .unwrap(); + + // The cached mapping survives until invalidated (this is the staleness the delete/attach/rename + // paths must clear). + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-a" + ); + + // After invalidation (what delete_workspace / attach_dev_workspace / change_workspace_id now call), + // it re-resolves to the new root. + invalidate_billing_workspace_cache("inv-fork"); + assert_eq!( + get_billing_workspace_id(&db, "inv-fork").await.unwrap(), + "inv-root-b" + ); +} + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn fork_depth_and_subtree_height(db: Pool) { + // Chain: root -> f1 -> f2 -> f3 + insert_ws(&db, "fd-root", None, false).await; + insert_ws(&db, "fd-f1", Some("fd-root"), false).await; + insert_ws(&db, "fd-f2", Some("fd-f1"), false).await; + insert_ws(&db, "fd-f3", Some("fd-f2"), false).await; + + // Depth walks up to the root: root = 0, each fork adds one. + assert_eq!(fork_chain_depth(&db, "fd-root").await.unwrap(), 0); + assert_eq!(fork_chain_depth(&db, "fd-f1").await.unwrap(), 1); + assert_eq!(fork_chain_depth(&db, "fd-f3").await.unwrap(), 3); + // An unknown id has no chain, so depth 0 (treated as a root by the guard). + assert_eq!(fork_chain_depth(&db, "fd-missing").await.unwrap(), 0); + + // Height walks down: the deepest live descendant below the node. + assert_eq!(fork_subtree_height(&db, "fd-root").await.unwrap(), 3); + assert_eq!(fork_subtree_height(&db, "fd-f2").await.unwrap(), 1); + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // A deleted leaf doesn't add to the height. + insert_ws(&db, "fd-f3-del", Some("fd-f3"), true).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 0); + + // ...but a LIVE descendant below a soft-deleted intermediate still counts at its true depth + // (the walk traverses through the deleted node; only the aggregation filters deleted). + insert_ws(&db, "fd-f3-live-gc", Some("fd-f3-del"), false).await; + assert_eq!(fork_subtree_height(&db, "fd-f3").await.unwrap(), 2); +} diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index f3ee59cab7..054f892220 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -635,7 +635,9 @@ fn duckdb_value_to_json_value( .ok_or_else(|| "Could not convert to f64".to_string())?, ), duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), - duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), + duckdb::types::Value::Timestamp(unit, ts) => { + serde_json::Value::String(duckdb_timestamp_to_iso(unit, ts)) + } duckdb::types::Value::Text(s) if type_alias.as_deref().unwrap_or_default() == "JSON" => { serde_json::from_str(&s) .map_err(|e| format!("Error parsing JSON text: {}", e.to_string()))? @@ -646,8 +648,15 @@ fn duckdb_value_to_json_value( .map(|byte| serde_json::Value::Number(byte.into())) .collect(), ), - duckdb::types::Value::Date32(d) => serde_json::Value::Number(d.into()), - duckdb::types::Value::Time64(_, t) => serde_json::Value::String(t.to_string()), + duckdb::types::Value::Date32(d) => { + match chrono::DateTime::from_timestamp(i64::from(d) * 86_400, 0) { + Some(dt) => serde_json::Value::String(dt.date_naive().to_string()), + None => serde_json::Value::Number(d.into()), + } + } + duckdb::types::Value::Time64(unit, t) => { + serde_json::Value::String(duckdb_time_to_iso(unit, t).unwrap_or_else(|| t.to_string())) + } duckdb::types::Value::Interval { months, days, nanos } => serde_json::json!({ "months": months, "days": days, @@ -688,6 +697,105 @@ fn duckdb_value_to_json_value( Ok(json_value) } +// DuckDB surfaces TIMESTAMP[_S/_MS/_NS] values as a raw count since the epoch; +// stringifying that count leaks values like "1782974022218435" into job +// results. Render ISO-8601 instead (the Postgres executor's +// "2024-01-15T10:30:00" shape); a count outside chrono's representable range +// falls back to the raw number. +fn duckdb_timestamp_to_iso(unit: duckdb::types::TimeUnit, ts: i64) -> String { + let dt = match unit { + duckdb::types::TimeUnit::Second => chrono::DateTime::from_timestamp(ts, 0), + duckdb::types::TimeUnit::Millisecond => chrono::DateTime::from_timestamp_millis(ts), + duckdb::types::TimeUnit::Microsecond => chrono::DateTime::from_timestamp_micros(ts), + duckdb::types::TimeUnit::Nanosecond => Some(chrono::DateTime::from_timestamp_nanos(ts)), + }; + dt.map(|dt| dt.naive_utc().format("%Y-%m-%dT%H:%M:%S%.f").to_string()) + .unwrap_or_else(|| ts.to_string()) +} + +// Same story for TIME: a raw count since midnight. None when out of range +// (caller falls back to the raw number). +fn duckdb_time_to_iso(unit: duckdb::types::TimeUnit, t: i64) -> Option { + let (secs, nanos) = match unit { + duckdb::types::TimeUnit::Second => (t, 0), + duckdb::types::TimeUnit::Millisecond => (t / 1_000, (t % 1_000) * 1_000_000), + duckdb::types::TimeUnit::Microsecond => (t / 1_000_000, (t % 1_000_000) * 1_000), + duckdb::types::TimeUnit::Nanosecond => (t / 1_000_000_000, t % 1_000_000_000), + }; + chrono::NaiveTime::from_num_seconds_from_midnight_opt( + u32::try_from(secs).ok()?, + u32::try_from(nanos).ok()?, + ) + .map(|t| t.format("%H:%M:%S%.f").to_string()) +} + +#[cfg(test)] +mod temporal_json_tests { + use super::*; + use duckdb::types::TimeUnit; + + #[test] + fn timestamp_micros_renders_iso() { + // 2026-07-01 23:13:42.218435 UTC + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Microsecond, 1_782_947_622_218_435), + "2026-07-01T23:13:42.218435" + ); + } + + #[test] + fn timestamp_seconds_renders_iso_without_subseconds() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, 1_735_689_600), + "2025-01-01T00:00:00" + ); + } + + #[test] + fn out_of_range_timestamp_falls_back_to_raw() { + assert_eq!( + duckdb_timestamp_to_iso(TimeUnit::Second, i64::MAX), + i64::MAX.to_string() + ); + } + + #[test] + fn time_micros_renders_iso() { + assert_eq!( + duckdb_time_to_iso(TimeUnit::Microsecond, 37_800_500_000).as_deref(), + Some("10:30:00.500") + ); + } + + #[test] + fn temporal_values_render_iso_through_real_query() { + let conn = duckdb::Connection::open_in_memory().unwrap(); + let mut stmt = conn + .prepare( + "SELECT TIMESTAMP '2026-07-01 23:13:42.218435' AS ts, + TIMESTAMPTZ '2026-07-01 23:13:42+00' AS tstz, + TIMESTAMP_NS '2026-07-01 23:13:42.218435678' AS ts_ns, + DATE '2026-07-01' AS d, + TIME '10:30:00' AS t", + ) + .unwrap(); + let mut rows = stmt.query([]).unwrap(); + let row = rows.next().unwrap().unwrap(); + let json_of = |i: usize| { + let v: duckdb::types::Value = row.get(i).unwrap(); + duckdb_value_to_json_value(v, &None).unwrap() + }; + assert_eq!(json_of(0), serde_json::json!("2026-07-01T23:13:42.218435")); + assert_eq!(json_of(1), serde_json::json!("2026-07-01T23:13:42")); + assert_eq!( + json_of(2), + serde_json::json!("2026-07-01T23:13:42.218435678") + ); + assert_eq!(json_of(3), serde_json::json!("2026-07-01")); + assert_eq!(json_of(4), serde_json::json!("10:30:00")); + } +} + fn json_value_to_duckdb_value( json_value: &serde_json::Value, arg_type: &str, diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index cf30efe116..553eb69917 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -494,6 +494,23 @@ fn build_azure_blob_client( return Ok(Arc::new(store)); } +/// Whether a GCS `service_account_key` carries no static credentials, in which case the client +/// should fall back to the instance's ambient credentials (GKE Workload Identity / metadata server) +/// instead of being handed an unparseable key. Besides an empty/whitespace string, the settings UI +/// stores "no key" as an empty JSON object `{}` (and `serde_json` may yield `null`), so treat those +/// as absent too. Shared with the connectivity-test SSRF guard so both agree on what "no key" means. +pub fn gcs_service_account_key_is_blank(service_account_key: &str) -> bool { + let trimmed = service_account_key.trim(); + if trimmed.is_empty() { + return true; + } + match serde_json::from_str::(trimmed) { + Ok(serde_json::Value::Null) => true, + Ok(serde_json::Value::Object(map)) => map.is_empty(), + _ => false, + } +} + #[cfg(feature = "parquet")] async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result> { let gcs_resource = gcs_resource_ref.clone(); @@ -509,7 +526,12 @@ async fn build_gcs_client(gcs_resource_ref: &GcsResource) -> error::Result { // Update user usage for non-premium workspaces if !team_plan_status.premium { @@ -5124,8 +5133,13 @@ async fn push_inner<'c, 'd>( ) -> Result<(Uuid, Transaction<'c, Postgres>), Error> { #[cfg(feature = "cloud")] if *CLOUD_HOSTED { + // A fork/dev workspace draws its plan and usage from the root (billing) workspace, so its + // executions are metered against the parent's quota/bill. Resolves to `workspace_id` itself + // for a standalone workspace (no behavior change off-fork). + let billing_w_id = + windmill_common::workspaces::get_billing_workspace_id(db, workspace_id).await?; let team_plan_status = - windmill_common::workspaces::get_team_plan_status(db, workspace_id).await?; + windmill_common::workspaces::get_team_plan_status(db, &billing_w_id).await?; // we track only non flow steps let (workspace_usage, user_usage) = if !matches!( job_payload, @@ -5134,12 +5148,12 @@ async fn push_inner<'c, 'd>( // Check current usage with SELECT (fast, no row locks) // Only check user usage for non-premium workspaces let (current_workspace_usage, current_user_usage) = - check_usage_limits(db, workspace_id, email, !team_plan_status.premium).await?; + check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?; // Spawn async task to update usage counters in the background increment_usage_async( db.clone(), - workspace_id.to_string(), + billing_w_id.clone(), if !team_plan_status.premium { Some(email.to_string()) } else { @@ -5242,7 +5256,7 @@ async fn push_inner<'c, 'd>( WHERE is_workspace IS TRUE AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND id = $1", - workspace_id + billing_w_id ) .fetch_optional(db) .await? @@ -5269,6 +5283,13 @@ async fn push_inner<'c, 'd>( ))); } + // These two burst guards intentionally stay keyed to `workspace_id`, not the + // billing root: the shared caps are the monthly usage above (metered to the + // root) and the per-user in-queue/concurrent guards further up (keyed by email, + // global across the whole family). Keying these to the root would count the + // root's own queue rather than this workspace's load or the family's; a true + // family-shared burst cap would need a family-wide subquery, not worth the + // hot-path cost for this downgrade-only soft guard. let in_queue_workspace = sqlx::query_scalar!( "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1", workspace_id diff --git a/backend/windmill-queue/src/tags.rs b/backend/windmill-queue/src/tags.rs index a0cc3a305e..d80e50d8b2 100644 --- a/backend/windmill-queue/src/tags.rs +++ b/backend/windmill-queue/src/tags.rs @@ -8,13 +8,21 @@ const FORK_PARENT_CACHE_TTL_SECS: u64 = 300; lazy_static::lazy_static! { // Cache of fork workspace id -> (parent_workspace_id, cached_at). - // `parent_workspace_id` is essentially immutable once a fork is created, so a multi-minute TTL - // is safe. `None` means the lookup found no parent (or the DB call failed); we still cache it - // briefly so that forks missing a parent do not hammer the DB. + // `parent_workspace_id` is stable for the lifetime of a fork EXCEPT across attach/detach of a + // dev workspace, which set/keep it; those paths call `invalidate_fork_parent_cache` so routing + // doesn't lag. `None` means the lookup found no parent (or the DB call failed); we still cache + // it briefly so that forks missing a parent do not hammer the DB. static ref FORK_PARENT_CACHE: quick_cache::sync::Cache, std::time::Instant)> = quick_cache::sync::Cache::new(500); } +/// Drop the cached fork->parent mapping for a workspace. Call after mutating `parent_workspace_id` +/// (attaching/detaching a dev workspace) so per-workspace job tags resolve to the new parent +/// immediately instead of after the cache TTL. +pub fn invalidate_fork_parent_cache(workspace_id: &str) { + FORK_PARENT_CACHE.remove(workspace_id); +} + /// Returns `Some(effective_workspace_tag_id)` if jobs of `workspace_id` should use workspace- /// specific tags, where `effective_workspace_tag_id` is the string embedded in the tag. For forks, /// this is always the parent workspace id, optionally suffixed with `-fork` (controlled by the @@ -26,16 +34,14 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool) -> Optio return None; } - let is_fork = workspace_id.starts_with(WM_FORK_PREFIX); - - // For forks, always resolve to the parent workspace id; regular workspaces avoid the lookup. - let effective_ws_id: String = if is_fork { - lookup_fork_parent(workspace_id, db) - .await - .unwrap_or_else(|| workspace_id.to_string()) // no parent found -> fall back to fork's own id - } else { - workspace_id.to_string() - }; + // Resolve to the parent workspace id when the workspace is a fork or dev workspace (both set + // parent_workspace_id). The lookup caches its `None` result, so non-forks stay cheap after warmup + // (and the common case is already short-circuited by the global toggle above). + let parent = lookup_fork_parent(workspace_id, db).await; + // A `wm-fork-` workspace can outlive its parent (the FK is `ON DELETE SET NULL`), so keep + // treating the prefix as fork-ness for the `-fork` suffix even when the parent link is gone. + let is_fork = parent.is_some() || workspace_id.starts_with(WM_FORK_PREFIX); + let effective_ws_id: String = parent.unwrap_or_else(|| workspace_id.to_string()); // Whitelist check is against the resolved (parent) id so that including a parent in the // whitelist transparently covers all of its forks. @@ -62,8 +68,10 @@ pub async fn per_workspace_tag(workspace_id: &str, db: &Pool) -> Optio }) } -/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set (or the -/// DB lookup failed). Backed by a short-TTL cache to avoid a DB round-trip per job push. +/// Returns the parent workspace id for a fork, or `None` if the fork has no parent set. Backed by a +/// short-TTL cache to avoid a DB round-trip per job push. A transient DB error returns `None` for +/// this call but is NOT cached, so the next push retries instead of misrouting a (prefix-less) dev +/// workspace's jobs for the whole TTL. async fn lookup_fork_parent(fork_id: &str, db: &Pool) -> Option { if let Some((parent, cached_at)) = FORK_PARENT_CACHE.get(fork_id) { if cached_at.elapsed().as_secs() < FORK_PARENT_CACHE_TTL_SECS { @@ -78,8 +86,11 @@ async fn lookup_fork_parent(fork_id: &str, db: &Pool) -> Option Some(parent), - _ => None, + Ok(opt) => opt.flatten(), + Err(e) => { + tracing::warn!("failed to look up fork parent for {fork_id}: {e:#}"); + return None; + } }; FORK_PARENT_CACHE.insert( diff --git a/backend/windmill-store/src/resources.rs b/backend/windmill-store/src/resources.rs index 7da6d75f4b..98f91065e4 100644 --- a/backend/windmill-store/src/resources.rs +++ b/backend/windmill-store/src/resources.rs @@ -1088,21 +1088,48 @@ async fn create_resource( .execute(&db) .await?; } - sqlx::query!( - "INSERT INTO resource - (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) - VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", - w_id, - resource.path, - raw_json as sqlx::types::Json<&RawValue>, - resource.description, - resource.resource_type, - authed.username, - resource.labels.as_deref() as Option<&[String]> - ) - .execute(&mut *tx) - .await?; + if update_if_exists { + sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) + VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = EXCLUDED.value, description = EXCLUDED.description, resource_type = EXCLUDED.resource_type, edited_at = now(), labels = EXCLUDED.labels", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username, + resource.labels.as_deref() as Option<&[String]> + ) + .execute(&mut *tx) + .await?; + } else { + // Create-only (the default): DO NOTHING + a row-count guard, so a path that appears between + // check_path_conflict above and this insert is rejected rather than overwritten. A plain + // DO UPDATE here would clobber a concurrently-created resource, breaking create-only callers + // (e.g. Compare & Deploy "Create in "). + let inserted = sqlx::query!( + "INSERT INTO resource + (workspace_id, path, value, description, resource_type, created_by, edited_at, labels) + VALUES ($1, $2, $3, $4, $5, $6, now(), $7) ON CONFLICT (workspace_id, path) DO NOTHING", + w_id, + resource.path, + raw_json as sqlx::types::Json<&RawValue>, + resource.description, + resource.resource_type, + authed.username, + resource.labels.as_deref() as Option<&[String]> + ) + .execute(&mut *tx) + .await?; + if inserted.rows_affected() == 0 { + return Err(Error::BadRequest(format!( + "Resource {} already exists", + resource.path + ))); + } + } // Mirror update_resource: Some(true) inserts, Some(false) clears (only // meaningful on the upsert path, since a pure create has no existing row), @@ -1385,7 +1412,12 @@ fn collect_var_refs(value: &serde_json::Value, out: &mut Vec) { } } -async fn mark_linked_variables_ws_specific( +/// Marks every variable referenced by the resource at `resource_path` as workspace-specific. +/// +/// AUTH CONTRACT: this mutates `ws_specific` and does NOT check authorization itself. The caller +/// MUST verify that `authed` has write access to the resource at `resource_path` in `w_id` (e.g. via +/// `require_owner_of_path`) before calling it. +pub async fn mark_linked_variables_ws_specific( tx: &mut Transaction<'_, Postgres>, authed: &ApiAuthed, w_id: &str, diff --git a/backend/windmill-trigger-websocket/src/listener.rs b/backend/windmill-trigger-websocket/src/listener.rs index d46faf1b1a..e34fe6e4f6 100644 --- a/backend/windmill-trigger-websocket/src/listener.rs +++ b/backend/windmill-trigger-websocket/src/listener.rs @@ -11,7 +11,7 @@ use serde::Deserialize; use serde_json::value::RawValue; use std::{borrow::Cow, collections::HashMap, sync::Arc}; use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use tokio_tungstenite::{tungstenite, tungstenite::Message, MaybeTlsStream, WebSocketStream}; use windmill_common::{ error::{to_anyhow, Error, Result}, jobs::JobTriggerKind, @@ -22,13 +22,30 @@ use windmill_common::{ }; use windmill_queue::PushArgsOwned; use windmill_trigger::filter::{check_filters, Filter}; -use windmill_trigger::listener::ListeningTrigger; +use windmill_trigger::listener::{update_rw_lock, ListeningTrigger}; use windmill_trigger::trigger_helpers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, }; use windmill_trigger::Listener; +const MAX_CONNECT_ATTEMPTS: u32 = 5; + +/// Whether a failed WebSocket connect is worth retrying: network-level IO +/// errors and HTTP 5xx/429 handshake responses are typically transient (edge +/// proxies like Cloudflare return 502/520 sporadically), while other errors +/// (bad URL, protocol mismatch, other 4xx) point at configuration and would +/// fail identically on every attempt. +fn is_transient_connect_error(err: &tungstenite::Error) -> bool { + match err { + tungstenite::Error::Io(_) => true, + tungstenite::Error::Http(resp) => { + resp.status().is_server_error() || resp.status().as_u16() == 429 + } + _ => false, + } +} + async fn send_initial_messages( listening_trigger: &ListeningTrigger, writer: &mut SplitSink>, Message>, @@ -176,12 +193,50 @@ impl Listener for WebsocketTrigger { validate_websocket_url_for_ssrf(&connect_url).await?; - let connection = connect_async_with_proxy(&*connect_url) - .await - .map(|conn| Some(conn)) - .map_err(|err| to_anyhow(err).into()); - - connection + // Gateway endpoints are often fronted by an edge proxy (e.g. Cloudflare) + // that sporadically answers the upgrade request with a transient 5xx + // instead of `101 Switching Protocols`, and a `get_consumer` error + // disables the trigger until a human re-enables it — so retry transient + // failures with backoff before giving up. The caller runs `loop_ping` + // concurrently so `last_server_ping` stays alive across the sleeps, and + // killpill cancels this future between awaits. + let mut attempt = 0; + loop { + attempt += 1; + match connect_async_with_proxy(&*connect_url).await { + Ok(conn) => return Ok(Some(conn)), + // Only retry in trigger mode: a failed connect there disables the + // trigger until a human re-enables it, while capture mode is an + // interactive test where instant feedback beats resilience (an + // `Io` error can also be a permanent misconfiguration, e.g. a + // typo'd host, which should surface immediately when iterating). + Err(err) + if listening_trigger.trigger_mode + && attempt < MAX_CONNECT_ATTEMPTS + && is_transient_connect_error(&err) => + { + let delay_secs = 1u64 << attempt; + tracing::warn!( + "Transient error connecting to WebSocket for trigger {} (attempt {}/{}), retrying in {}s: {}", + listening_trigger.path, + attempt, + MAX_CONNECT_ATTEMPTS, + delay_secs, + err + ); + update_rw_lock( + err_message.clone(), + Some(format!( + "Connection attempt {}/{} failed ({}), retrying in {}s...", + attempt, MAX_CONNECT_ATTEMPTS, err, delay_secs + )), + ) + .await; + tokio::time::sleep(std::time::Duration::from_secs(delay_secs)).await; + } + Err(err) => return Err(to_anyhow(err).into()), + } + } } async fn consume( &self, diff --git a/backend/windmill-trigger-websocket/tests/connect_retry.rs b/backend/windmill-trigger-websocket/tests/connect_retry.rs new file mode 100644 index 0000000000..2d21512692 --- /dev/null +++ b/backend/windmill-trigger-websocket/tests/connect_retry.rs @@ -0,0 +1,153 @@ +//! End-to-end tests for the websocket trigger connect-retry behavior: a mock +//! TCP server rejects the websocket upgrade with a configurable HTTP status a +//! number of times before completing a real handshake, and the tests assert +//! which failures `get_consumer` retries. +//! +//! This lives in an integration-test binary (own process) because it sets +//! ALLOW_PRIVATE_WEBSOCKET_URLS — the mock server listens on 127.0.0.1, which +//! the SSRF check blocks — and that process-global env var must not leak into +//! the crate's unit tests, which assert loopback URLs are rejected. + +use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, +}; + +use tokio::{ + io::{AsyncBufReadExt, AsyncWriteExt, BufReader}, + net::TcpListener, + sync::{broadcast, RwLock}, +}; +use windmill_trigger::{listener::ListeningTrigger, Listener}; +use windmill_trigger_websocket::{ + WebsocketConfig, WebsocketTrigger, ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, +}; + +/// Mock server: rejects the first `failures` upgrade requests with +/// `status_line` and closes, then completes real websocket handshakes and +/// parks the connection open. Returns the bound address and the +/// connection-attempt counter. +async fn mock_ws_server( + failures: u32, + status_line: &'static str, +) -> (std::net::SocketAddr, Arc) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let attempts = Arc::new(AtomicU32::new(0)); + let served = attempts.clone(); + tokio::spawn(async move { + loop { + let (mut socket, _) = listener.accept().await.unwrap(); + let n = served.fetch_add(1, Ordering::SeqCst); + if n < failures { + // Drain the request head, then reject the upgrade. + let mut reader = BufReader::new(&mut socket); + let mut line = String::new(); + loop { + line.clear(); + let read = reader.read_line(&mut line).await.unwrap_or(0); + if read == 0 || line == "\r\n" { + break; + } + } + socket + .write_all( + format!("HTTP/1.1 {status_line}\r\nContent-Length: 0\r\n\r\n").as_bytes(), + ) + .await + .ok(); + } else if let Ok(ws) = tokio_tungstenite::accept_async(socket).await { + tokio::spawn(async move { + let _open = ws; + std::future::pending::<()>().await + }); + } + } + }); + (addr, attempts) +} + +fn trigger(url: String, trigger_mode: bool) -> ListeningTrigger { + ListeningTrigger { + path: "f/test/ws".to_string(), + is_flow: false, + workspace_id: "test".to_string(), + edited_by: "test".to_string(), + permissioned_as: "u/test".to_string(), + trigger_config: WebsocketConfig { + url, + filters: vec![], + filter_logic: "and".to_string(), + initial_messages: None, + url_runnable_args: None, + can_return_message: false, + can_return_error_result: false, + heartbeat: None, + }, + script_path: "f/test/script".to_string(), + trigger_mode, + error_handling: None, + suspended_mode: false, + } +} + +async fn get_consumer_result( + lt: &ListeningTrigger, + err_message: Arc>>, +) -> windmill_common::error::Result::Consumer>> { + // The static-URL path of `get_consumer` never touches the DB; a lazy pool + // satisfies the signature without a running postgres. + let db: windmill_common::DB = + sqlx::Pool::connect_lazy("postgres://unused:unused@127.0.0.1:1/unused").unwrap(); + let (_killpill_tx, killpill_rx) = broadcast::channel::<()>(1); + WebsocketTrigger + .get_consumer(&db, lt, err_message, killpill_rx) + .await +} + +#[tokio::test] +async fn transient_502s_are_retried_until_the_upgrade_succeeds() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(2, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), true); + let err_message = Arc::new(RwLock::new(None)); + + let consumer = get_consumer_result(<, err_message.clone()) + .await + .expect("connect should succeed after retries"); + + assert!(consumer.is_some(), "expected an established connection"); + assert_eq!(attempts.load(Ordering::SeqCst), 3); + // Retry progress was reported through the shared status lock. + let status = err_message.read().await.clone().unwrap(); + assert!(status.contains("attempt 2/5"), "got status: {status}"); + assert!(status.contains("502"), "got status: {status}"); +} + +#[tokio::test] +async fn non_transient_http_errors_fail_on_the_first_attempt() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "404 Not Found").await; + let lt = trigger(format!("ws://{addr}"), true); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("a 404 upgrade response should not be retried"); + + assert!(err.to_string().contains("404"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn capture_mode_fails_fast_even_on_transient_errors() { + std::env::set_var(ALLOW_PRIVATE_WEBSOCKET_URLS_ENV, "true"); + let (addr, attempts) = mock_ws_server(u32::MAX, "502 Bad Gateway").await; + let lt = trigger(format!("ws://{addr}"), false); + + let err = get_consumer_result(<, Arc::new(RwLock::new(None))) + .await + .expect_err("capture mode should surface the first failure immediately"); + + assert!(err.to_string().contains("502"), "got error: {err}"); + assert_eq!(attempts.load(Ordering::SeqCst), 1); +} diff --git a/backend/windmill-trigger/src/handler.rs b/backend/windmill-trigger/src/handler.rs index 65072bc9ab..ed3b89fc65 100644 --- a/backend/windmill-trigger/src/handler.rs +++ b/backend/windmill-trigger/src/handler.rs @@ -905,6 +905,26 @@ async fn delete_trigger( tx.commit().await?; + // Reset the fork/parent workspace_diff tally for this path, exactly as + // create/update and every other kind's delete does. Without this a deleted + // trigger leaves its cached `has_changes=true` diff row behind: the compare + // trusts it (triggers aren't re-validated like scripts/flows), then drops it + // as it no longer exists in the table — a phantom "ahead" item that reads as + // "changes not visible to your user" and hides the deploy button, even for + // superadmins. Re-tallying sets has_changes=NULL so the next compare + // re-evaluates and corrects/removes the row. + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(path.to_string(), None), + Some(format!("{} '{}' deleted", T::DEPLOYMENT_NAME, path)), + true, + None, + ) + .await?; + // Trigger gone for everyone: wipe ALL users' drafts at this path; see scripts.rs. delete_all_drafts_for_path(&db, &workspace_id, T::user_draft_item_kind(), path).await?; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 2758fe85ff..292246c62c 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -938,6 +938,143 @@ remote_tmp={job_dir}/.ansible/tmp Ok(()) } +/// Read a colon-separated path list (e.g. `roles_path`, `collections_path`) from +/// the `[defaults]` section of an ansible.cfg. Returns the raw entries as written, +/// unresolved. Deliberately minimal: no inline-comment or continuation handling, +/// which ansible's configparser also does not apply to these values. `key` and `=` +/// or `:` as the delimiter are both matched (Python configparser accepts either), +/// with the value itself split on `:` (`os.pathsep`). +fn parse_ansible_cfg_path_list(content: &str, key: &str) -> Option> { + let mut in_defaults = false; + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.starts_with('[') && trimmed.ends_with(']') { + in_defaults = trimmed[1..trimmed.len() - 1] + .trim() + .eq_ignore_ascii_case("defaults"); + continue; + } + if !in_defaults || trimmed.starts_with('#') || trimmed.starts_with(';') { + continue; + } + // configparser delimits key/value on the first `=` or `:`, whichever + // comes first; the remaining `:` in the value are path separators. + let sep = trimmed.find('=').into_iter().chain(trimmed.find(':')).min(); + if let Some(sep) = sep { + let (k, rest) = trimmed.split_at(sep); + let v = &rest[1..]; + if k.trim().eq_ignore_ascii_case(key) { + let entries: Vec = v + .split(':') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + return (!entries.is_empty()).then_some(entries); + } + } + } + None +} + +/// Prepend Windmill's dependency install dir to the repo cfg's declared path list. +/// Relative entries from the repo cfg are resolved against `cfg_dir` to match how +/// ansible resolves them relative to the config file's own directory. +fn resolve_and_prepend_path( + base: String, + repo_entries: Option>, + cfg_dir: &str, +) -> String { + let mut paths = vec![base]; + if let Some(entries) = repo_entries { + for e in entries { + if e.starts_with('/') || e.starts_with('~') { + paths.push(e); + } else { + paths.push(format!("{cfg_dir}/{e}")); + } + } + } + paths.join(":") +} + +/// Build the environment overrides applied when delegating to a git repo that +/// ships its own ansible.cfg. See the call site for the layering rationale. +async fn build_ansible_cfg_override_envs( + cfg_path: &str, + job_dir: &str, + vault_password_file_exists: bool, + reqs: Option<&AnsibleRequirements>, +) -> error::Result> { + let mut envs = vec![ + ("ANSIBLE_CONFIG".to_string(), cfg_path.to_string()), + // Runtime-bound: reference the ephemeral job dir, cannot be set statically. + ("ANSIBLE_HOME".to_string(), format!("{job_dir}/.ansible")), + ( + "ANSIBLE_LOCAL_TEMP".to_string(), + format!("{job_dir}/.ansible/tmp"), + ), + ( + "ANSIBLE_REMOTE_TEMP".to_string(), + format!("{job_dir}/.ansible/tmp"), + ), + ]; + + // Vault: Windmill manages the secret, so its config wins over the repo cfg. + if vault_password_file_exists { + envs.push(( + "ANSIBLE_VAULT_PASSWORD_FILE".to_string(), + format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}"), + )); + } + if let Some(vault_ids) = reqs.map(|r| &r.vault_id).filter(|v| !v.is_empty()) { + // Defense in depth: entries are validated at parse time, but re-check + // here since they are interpolated raw into the env value. + for vault_id in vault_ids { + validate_vault_id(vault_id)?; + } + envs.push(( + "ANSIBLE_VAULT_IDENTITY_LIST".to_string(), + vault_ids.join(","), + )); + } + + // Dependency search paths: additive. Windmill installs galaxy roles into + // `{job_dir}/roles` and collections into `{job_dir}`; prepend those to the + // repo cfg's declared paths so both Windmill-installed and repo deps resolve. + let cfg_dir = std::path::Path::new(cfg_path) + .parent() + .and_then(|p| p.to_str()) + .unwrap_or(job_dir); + let cfg_content = tokio::fs::read_to_string(cfg_path).await.map_err(|e| { + windmill_common::error::Error::internal_err(format!( + "Failed to read delegated ansible.cfg at `{cfg_path}`: {e}" + )) + })?; + + envs.push(( + "ANSIBLE_ROLES_PATH".to_string(), + resolve_and_prepend_path( + format!("{job_dir}/roles"), + parse_ansible_cfg_path_list(&cfg_content, "roles_path"), + cfg_dir, + ), + )); + envs.push(( + "ANSIBLE_COLLECTIONS_PATH".to_string(), + resolve_and_prepend_path( + job_dir.to_string(), + // Also probe the deprecated plural ini alias; env vars replace (not + // merge) the cfg value, so a repo using it would otherwise be dropped. + parse_ansible_cfg_path_list(&cfg_content, "collections_path") + .or_else(|| parse_ansible_cfg_path_list(&cfg_content, "collections_paths")), + cfg_dir, + ), + )); + + Ok(envs) +} + pub async fn get_git_ssh_cmd( reqs: &AnsibleRequirements, job_dir: &str, @@ -1136,6 +1273,9 @@ pub async fn handle_ansible_job( let mut nsjail_extra_mounts = vec![]; let mut playbook_override = None; + // Absolute path of a repo-provided `ansible.cfg` to use as the effective + // config, set when `delegate_to_git_repo.ansible_cfg` is provided. + let mut ansible_config_override: Option = None; if let Some(r) = reqs.as_ref() { nsjail_extra_mounts = create_file_resources( @@ -1301,6 +1441,23 @@ pub async fn handle_ansible_job( inventories.push(format!("{}/{}", &repo.target_path, inv)); } + if let Some(cfg_rel) = delegated_git_repo.ansible_cfg.as_ref() { + let cfg_rel = interpolate_template( + cfg_rel, + interpolated_args.as_ref(), + "delegate_to_git_repo.ansible_cfg", + )?; + validate_relative_path(&cfg_rel, "delegate_to_git_repo.ansible_cfg")?; + let cfg_path = format!("{}/{}/{}", job_dir, &repo.target_path, cfg_rel); + if !tokio::fs::try_exists(&cfg_path).await.unwrap_or(false) { + return Err(windmill_common::error::Error::BadRequest(format!( + "delegate_to_git_repo.ansible_cfg: no ansible.cfg found in the cloned repo at `{}/{}`", + &repo.target_path, cfg_rel + ))); + } + ansible_config_override = Some(cfg_path); + } + if delegated_git_repo.install_requirements { install_requirements_from_cloned_repo( &repo.target_path, @@ -1450,6 +1607,32 @@ pub async fn handle_ansible_job( create_ansible_cfg(reqs.as_ref(), job_dir, vault_password_file_exists)?; + // When the run delegates to a git repo that ships its own ansible.cfg, that + // file becomes the effective config (ansible loads exactly one config file and + // does not merge). These env vars layer Windmill's runtime-bound settings back + // on top — env vars outrank ansible.cfg. Only applied on the non-sandboxed + // path: git-repo delegation clones into `job_dir` which the nsjail profile does + // not mount, so it already requires DISABLE_NSJAIL. + let ansible_env_overrides = match ansible_config_override.as_ref() { + Some(cfg_path) => { + if is_sandboxing_enabled() { + tracing::warn!( + "delegate_to_git_repo.ansible_cfg is set but sandboxing is enabled; \ + git-repo delegation requires DISABLE_NSJAIL, the ansible.cfg override \ + will not take effect" + ); + } + build_ansible_cfg_override_envs( + cfg_path, + job_dir, + vault_password_file_exists, + reqs.as_ref(), + ) + .await? + } + None => vec![], + }; + let mut reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; let additional_python_paths_folders = additional_python_paths.join(":"); @@ -1560,6 +1743,7 @@ fi .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) + .envs(ansible_env_overrides) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1856,4 +2040,230 @@ mod tests { assert!(create_ansible_cfg(Some(&reqs), job_dir, false).is_err()); assert!(!dir.path().join("ansible.cfg").exists()); } + + #[test] + fn test_parse_ansible_cfg_path_list() { + let cfg = "\ +[defaults] +roles_path = roles:extra/roles +collections_path=/opt/collections +host_key_checking = False + +[inventory] +roles_path = ignored/section +"; + assert_eq!( + parse_ansible_cfg_path_list(cfg, "roles_path"), + Some(vec!["roles".to_string(), "extra/roles".to_string()]) + ); + assert_eq!( + parse_ansible_cfg_path_list(cfg, "collections_path"), + Some(vec!["/opt/collections".to_string()]) + ); + // Keys only in another section are not picked up. + assert_eq!(parse_ansible_cfg_path_list(cfg, "library"), None); + } + + #[test] + fn test_parse_ansible_cfg_path_list_ignores_comments() { + let cfg = "\ +[defaults] +# roles_path = commented +; roles_path = also_commented +"; + assert_eq!(parse_ansible_cfg_path_list(cfg, "roles_path"), None); + } + + #[test] + fn test_parse_ansible_cfg_path_list_colon_delimiter() { + // configparser accepts `:` as a key/value delimiter, and the value can + // itself be a `:`-separated list. + let cfg = "\ +[defaults] +roles_path: my_roles +collections_path : a/col:b/col +"; + assert_eq!( + parse_ansible_cfg_path_list(cfg, "roles_path"), + Some(vec!["my_roles".to_string()]) + ); + assert_eq!( + parse_ansible_cfg_path_list(cfg, "collections_path"), + Some(vec!["a/col".to_string(), "b/col".to_string()]) + ); + } + + #[test] + fn test_resolve_and_prepend_path() { + // No repo entries: only Windmill's install dir. + assert_eq!( + resolve_and_prepend_path("/job/roles".to_string(), None, "/job/repo"), + "/job/roles" + ); + // Relative repo entries resolve against the cfg dir; absolute/~ kept as-is. + assert_eq!( + resolve_and_prepend_path( + "/job/roles".to_string(), + Some(vec![ + "roles".to_string(), + "/abs/roles".to_string(), + "~/r".to_string() + ]), + "/job/repo/config" + ), + "/job/roles:/job/repo/config/roles:/abs/roles:~/r" + ); + } + + fn ansible_playbook_available() -> bool { + std::process::Command::new("ansible-playbook") + .arg("--version") + .output() + .is_ok() + } + + /// End-to-end: with a delegated repo that ships its own `ansible.cfg` pointing + /// `roles_path` at an in-repo directory, the override env vars must make the + /// real `ansible-playbook` resolve a role it otherwise cannot. Requires the + /// `ansible-playbook` binary; self-skips when absent (e.g. standard CI). Run on + /// a worker devbox with `cargo test -p windmill-worker --features python`. + #[tokio::test] + async fn test_ansible_cfg_override_resolves_repo_roles_e2e() { + if !ansible_playbook_available() { + eprintln!( + "SKIP test_ansible_cfg_override_resolves_repo_roles_e2e: ansible-playbook not found on PATH" + ); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo = dir.path().join(DELEGATE_GIT_REPO_TARGET); + let role_tasks = repo.join("my_roles/greet/tasks"); + std::fs::create_dir_all(&role_tasks).unwrap(); + + std::fs::write( + repo.join("ansible.cfg"), + "[defaults]\nroles_path = my_roles\n", + ) + .unwrap(); + std::fs::write( + role_tasks.join("main.yml"), + "- debug:\n msg: \"hello from greet role\"\n", + ) + .unwrap(); + let play = "- hosts: localhost\n connection: local\n gather_facts: false\n roles:\n - greet\n"; + std::fs::write(repo.join("play.yml"), play).unwrap(); + + // Windmill's own generated cfg (the negative-control config that exists today). + create_ansible_cfg(None, job_dir, false).unwrap(); + + let playbook = format!("{DELEGATE_GIT_REPO_TARGET}/play.yml"); + let run = |envs: Vec<(String, String)>| { + std::process::Command::new("ansible-playbook") + .arg(&playbook) + .current_dir(job_dir) + .envs(envs) + .output() + .unwrap() + }; + + // Negative control: today's behavior (Windmill cfg via cwd, no override) — + // the role lives in the repo subdir and is not found. + let cfg_path = repo.join("ansible.cfg"); + let before = run(vec![]); + assert!( + !before.status.success(), + "without the override the repo role must NOT resolve; stdout={}", + String::from_utf8_lossy(&before.stdout) + ); + + // With the override: ANSIBLE_CONFIG points at the repo cfg and roles_path + // is honored, so the role runs. + let envs = + build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) + .await + .unwrap(); + let after = run(envs); + let stdout = String::from_utf8_lossy(&after.stdout); + assert!( + after.status.success() && stdout.contains("hello from greet role"), + "with the override the repo role must resolve; status={:?} stdout={stdout} stderr={}", + after.status, + String::from_utf8_lossy(&after.stderr) + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join("delegate_git_repository"); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + std::fs::write(&cfg_path, "[defaults]\nroles_path = my_roles\n").unwrap(); + let cfg_path = cfg_path.to_str().unwrap(); + + let reqs = AnsibleRequirements { + vault_id: vec!["dev@vault_pass.txt".to_string()], + ..Default::default() + }; + let envs = build_ansible_cfg_override_envs(cfg_path, job_dir, true, Some(&reqs)) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + + assert_eq!( + map.get("ANSIBLE_CONFIG").map(|s| s.as_str()), + Some(cfg_path) + ); + assert_eq!( + map.get("ANSIBLE_HOME"), + Some(&format!("{job_dir}/.ansible")) + ); + assert_eq!( + map.get("ANSIBLE_VAULT_PASSWORD_FILE"), + Some(&format!("{job_dir}/{WINDMILL_ANSIBLE_PASSWORD_FILENAME}")) + ); + assert_eq!( + map.get("ANSIBLE_VAULT_IDENTITY_LIST").map(|s| s.as_str()), + Some("dev@vault_pass.txt") + ); + // Windmill's `{job_dir}/roles` is prepended to the repo cfg's own path, + // which is resolved against the cfg directory. + assert_eq!( + map.get("ANSIBLE_ROLES_PATH"), + Some(&format!( + "{job_dir}/roles:{}/my_roles", + repo_dir.to_str().unwrap() + )) + ); + // No collections_path in the repo cfg → only Windmill's job dir. + assert_eq!( + map.get("ANSIBLE_COLLECTIONS_PATH").map(|s| s.as_str()), + Some(job_dir) + ); + } + + #[tokio::test] + async fn test_build_ansible_cfg_override_envs_collections_paths_alias() { + let dir = tempfile::tempdir().unwrap(); + let job_dir = dir.path().to_str().unwrap(); + let repo_dir = dir.path().join("delegate_git_repository"); + std::fs::create_dir_all(&repo_dir).unwrap(); + let cfg_path = repo_dir.join("ansible.cfg"); + // Deprecated plural alias must still be picked up so the repo's collections + // are not silently dropped when the env override replaces the cfg value. + std::fs::write(&cfg_path, "[defaults]\ncollections_paths = my_cols\n").unwrap(); + + let envs = + build_ansible_cfg_override_envs(cfg_path.to_str().unwrap(), job_dir, false, None) + .await + .unwrap(); + let map: std::collections::HashMap<_, _> = envs.into_iter().collect(); + assert_eq!( + map.get("ANSIBLE_COLLECTIONS_PATH"), + Some(&format!("{job_dir}:{}/my_cols", repo_dir.to_str().unwrap())) + ); + } } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 1e4f4aafd6..f6b093f766 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,6 +1,11 @@ #[cfg(feature = "deno_core")] use std::time::Instant; -use std::{collections::HashMap, fs, process::Stdio}; +use std::{ + collections::{HashMap, HashSet}, + fs, + process::Stdio, + sync::Arc, +}; use base64::Engine; use itertools::Itertools; @@ -27,9 +32,10 @@ use crate::{ NSJAIL_AVAILABLE, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; use windmill_common::{ + cache, client::AuthedClient, jobs::JobKind, - scripts::{id_to_codebase_info, CodebaseInfo, ScriptLang}, + scripts::{id_to_codebase_info, CodebaseInfo, ScriptHash, ScriptLang}, utils::WarnAfterExt, workspace_dependencies::WorkspaceDependenciesPrefetched, }; @@ -44,7 +50,6 @@ use tokio::io::AsyncReadExt; use windmill_common::{ error::{self, Result}, - get_latest_hash_for_path, worker::{write_file, Connection, DISABLE_BUNDLING}, DB, }; @@ -1242,16 +1247,101 @@ pub fn ensure_bundle_output_exists(bundle_path: &str) -> Result<()> { pub const BUN_BUNDLE_OBJECT_STORE_PREFIX: &str = "bun_bundle/"; -async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) -> Result { - let script_hash = get_latest_hash_for_path(db, w_id, script_path, false).await?; - let last_updated_at = sqlx::query_scalar!( - "SELECT created_at FROM script WHERE workspace_id = $1 AND hash = $2", - w_id, - script_hash.0 .0 +// A script version's relative-import list never changes (content is immutable +// per hash), so parses are memoized without any invalidation. +lazy_static::lazy_static! { + static ref RELATIVE_IMPORTS_PER_HASH: quick_cache::sync::Cache>> = + quick_cache::sync::Cache::new(1000); +} + +const MAX_TRANSITIVE_IMPORT_PATHS: usize = 256; + +/// `(path, latest hash)` for the whole transitive closure of relative imports +/// of `inner_content` — the set of scripts whose code gets inlined into the +/// bundle, so all of them must key the bundle cache. Resolution goes through +/// `IMPORTED_SCRIPT_HASH_CACHE` (notify-evicted, 60s TTL fallback, same +/// version-selection predicate as the loader's content endpoint) and the +/// per-hash script/parse caches, so steady state costs no DB queries. +async fn collect_transitive_import_versions( + db: &DB, + w_id: &str, + script_path: &str, + inner_content: &str, +) -> Vec<(String, i64)> { + let conn = Connection::from(db.clone()); + let mut queue = crate::worker_lockfiles::extract_relative_imports( + inner_content, + script_path, + &Some(ScriptLang::Bun), ) - .fetch_one(db) - .await?; - Ok(last_updated_at.to_string()) + .unwrap_or_default(); + let mut visited: HashSet = HashSet::new(); + let mut versions: Vec<(String, i64)> = vec![]; + while let Some(path) = queue.pop() { + if !visited.insert(path.clone()) { + continue; + } + if visited.len() > MAX_TRANSITIVE_IMPORT_PATHS { + tracing::warn!( + "transitive relative-import closure of {script_path} exceeds \ + {MAX_TRANSITIVE_IMPORT_PATHS} scripts; bundle cache key covers only the first \ + {MAX_TRANSITIVE_IMPORT_PATHS}" + ); + break; + } + let hash = match windmill_common::get_latest_script_hash_for_import_cached(db, w_id, &path) + .await + { + Ok(Some(hash)) => hash, + // Not a deployed script at this path (deleted, or not a script): + // excluded from the key, matching what the bundler can inline. + Ok(None) => continue, + Err(e) => { + tracing::warn!( + "could not resolve import {path} while computing bundle cache key for \ + {script_path}: {e:#}" + ); + continue; + } + }; + versions.push((path.clone(), hash)); + let imports = match RELATIVE_IMPORTS_PER_HASH.get(&hash) { + Some(imports) => imports, + None => match cache::script::fetch(&conn, ScriptHash(hash)).await { + Ok((data, meta)) => { + let imports = Arc::new(match meta.language { + Some(ScriptLang::Bun) + | Some(ScriptLang::Bunnative) + | Some(ScriptLang::Deno) => { + crate::worker_lockfiles::extract_relative_imports( + &data.code, + &path, + &meta.language, + ) + .unwrap_or_default() + } + _ => vec![], + }); + RELATIVE_IMPORTS_PER_HASH.insert(hash, imports.clone()); + imports + } + // A fetch error is transient, not a property of the (immutable) + // content — memoizing it would drop this subtree from the key + // until worker restart. Skip caching and retry next run. + Err(e) => { + tracing::warn!( + "could not fetch import {path} (hash {hash}) while computing bundle \ + cache key for {script_path}: {e:#}" + ); + Arc::new(vec![]) + } + }, + }; + queue.extend(imports.iter().cloned()); + } + // deterministic key regardless of traversal order + versions.sort(); + versions } pub async fn compute_bundle_local_and_remote_path( @@ -1265,16 +1355,13 @@ pub async fn compute_bundle_local_and_remote_path( let mut input_src = format!("{inner_content}{lock}",); if let Some(db) = db { - let relative_imports = crate::worker_lockfiles::extract_relative_imports( - &inner_content, - script_path, - &Some(ScriptLang::Bun), - ); - for path in relative_imports.unwrap_or_default() { - if let Ok(updated_at) = get_script_import_updated_at(&db, w_id, &path).await { - input_src.push_str(&path); - input_src.push_str(&updated_at.to_string()); - } + // The bundle inlines the whole transitive relative-import closure, so a + // new deployed version of ANY script in it must change the key. + for (path, hash) in + collect_transitive_import_versions(db, w_id, script_path, inner_content).await + { + input_src.push_str(&path); + input_src.push_str(&hash.to_string()); } }; diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e4b94acabf..09e3158200 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -195,7 +195,21 @@ fn build_materialized_query( for s in plan.setup.iter_mut() { *s = substitute(s); } - let strategy = if m.append { + let strategy = if m.scd2 { + // SCD2 needs a natural key to identify an entity across versions, and its + // diff/close/open shape has no partition-scoped form in v1. + let key = m.unique_key.clone().ok_or_else(|| { + Error::ExecutionErr( + "materialize scd2: requires a natural key — add `key=`".to_string(), + ) + })?; + if partitioned { + return Err(Error::ExecutionErr( + "materialize scd2: `// partitioned` is not supported with scd2 in v1".to_string(), + )); + } + MaterializeStrategy::Scd2 { key, track: m.track.clone(), close_deleted: m.close_deleted } + } else if m.append { MaterializeStrategy::Append } else if let Some(uk) = m.unique_key { MaterializeStrategy::Merge { unique_key: uk } @@ -275,6 +289,469 @@ fn classify_wrap_or_err(query: &str) -> Result Result>> { + use windmill_common::assets::{ + ExpiringMacroRegistry, MACRO_REGISTRY_CACHE, MACRO_REGISTRY_CACHE_DISABLED, + MACRO_REGISTRY_TTL, + }; + // Cloud: unbounded workspace count makes the 1000-entry per-workspace + // cache churn instead of hit, and its per-entry memory (full registry + // text) is tenant-controlled — skip it there; self-hosted keeps the fast + // path. + let use_cache = !*windmill_common::worker::CLOUD_HOSTED + && !MACRO_REGISTRY_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + if use_cache { + if let Some(e) = MACRO_REGISTRY_CACHE.get(w_id) { + if e.expires_at > std::time::Instant::now() { + return Ok(e.rows); + } + } + } + let rows = sqlx::query_as!( + MacroRow, + "SELECT name, params, body, is_table_macro, provider_path FROM macro_definition WHERE workspace_id = $1", + w_id + ) + .fetch_all(db) + .await?; + let rows = std::sync::Arc::new(rows); + if use_cache { + MACRO_REGISTRY_CACHE.insert( + w_id.to_string(), + ExpiringMacroRegistry { + rows: rows.clone(), + expires_at: std::time::Instant::now() + MACRO_REGISTRY_TTL, + }, + ); + } + Ok(rows) +} + +// Selection pass: seed = macros the (comment-stripped, transformed) blocks +// call, plus every macro of each `// use` library; then the transitive +// closure over macro bodies. Local definitions win — their names are removed +// from the injectable set entirely, so a later workspace-library deploy can +// never silently replace a script's own macro. Pure and separate from +// `plan_macro_injection` so the shell knows which provider libraries to +// fetch (their setup statements are injected too) before planning. +fn select_workspace_macros( + blocks: &[String], + registry: &[MacroRow], + use_libs: &[String], +) -> Result> { + use std::collections::{BTreeMap, HashSet}; + use windmill_parser::duckdb_macros::{detect_macro_calls, locally_defined_macro_names}; + + let local = locally_defined_macro_names(blocks); + let all_names: HashSet = registry + .iter() + .map(|r| r.name.clone()) + .filter(|n| !local.contains(n)) + .collect(); + let by_name: BTreeMap<&str, &MacroRow> = + registry.iter().map(|r| (r.name.as_str(), r)).collect(); + + let mut selected: HashSet = HashSet::new(); + for path in use_libs { + let lib_names: Vec<&str> = registry + .iter() + .filter(|r| &r.provider_path == path) + .map(|r| r.name.as_str()) + .collect(); + if lib_names.is_empty() { + return Err(Error::ExecutionErr(format!( + "`// use {path}`: no deployed macro library at this path" + ))); + } + selected.extend( + lib_names + .into_iter() + .filter(|n| all_names.contains(*n)) + .map(String::from), + ); + } + selected.extend(detect_macro_calls(&blocks.join("\n"), &all_names)); + + // Transitive closure over macro bodies (a selected macro may call others). + let mut frontier: Vec = selected.iter().cloned().collect(); + while let Some(name) = frontier.pop() { + if let Some(row) = by_name.get(name.as_str()) { + for dep in detect_macro_calls(&row.body, &all_names) { + if selected.insert(dep.clone()) { + frontier.push(dep); + } + } + } + } + Ok(selected) +} + +// Fixpoint over library-level `// use`: a library may declare `// use` for +// dynamic calls its macro bodies make (string-hidden from lexical detection, +// e.g. inside `query('…')`). Those declarations are honored transitively — +// consuming a macro from lib B pulls in whatever B `// use`s, so the dynamic +// dependency stays encapsulated in the library instead of leaking to every +// consumer. `lib_uses` maps provider path → its parsed `// use` list; the +// shell grows it lazily and re-resolves until no new library appears. +// Returns the selected macro names plus the effective `// use` list +// (consumer's own first, in annotation order, then discovered libs). +fn resolve_macro_selection( + blocks: &[String], + registry: &[MacroRow], + consumer_use_libs: &[String], + lib_uses: &std::collections::BTreeMap>, +) -> Result<(std::collections::HashSet, Vec)> { + let mut effective: Vec = Vec::new(); + for p in consumer_use_libs { + if !effective.contains(p) { + effective.push(p.clone()); + } + } + loop { + let selected = select_workspace_macros(blocks, registry, &effective)?; + let mut relevant: Vec = effective.clone(); + for name in &selected { + if let Some(r) = registry.iter().find(|r| &r.name == name) { + if !relevant.contains(&r.provider_path) { + relevant.push(r.provider_path.clone()); + } + } + } + let mut grew = false; + for lib in &relevant { + for u in lib_uses.get(lib).map(Vec::as_slice).unwrap_or(&[]) { + if !effective.contains(u) { + effective.push(u.clone()); + grew = true; + } + } + } + if !grew { + return Ok((selected, effective)); + } + } +} + +// Emit the blocks to inject: first every relevant library's setup statements +// (a macro body may reference its own lib's ATTACH, and DuckDB bind-checks +// at CREATE — so setup is injected for the provider of every selected macro, +// not just `// use` libs), then the selected definitions in dependency order. +// `lib_bodies` carries the parsed deployed content per provider path; setup +// order = `// use` libs in annotation order, then remaining providers sorted. +// Exact-duplicate setup statements across libraries are deduped (two libs +// attaching the same catalog the same way must not double-ATTACH). Pure — +// fetches happen in `inject_workspace_macros` — so this is unit-testable. +fn plan_macro_injection( + selected: &std::collections::HashSet, + registry: &[MacroRow], + use_libs: &[String], + lib_bodies: &std::collections::BTreeMap< + String, + Vec, + >, +) -> Result> { + use std::collections::{BTreeMap, BTreeSet, HashSet}; + use windmill_parser::duckdb_macros::{macro_create_statement, topo_order_macros, LibStatement}; + + let by_name: BTreeMap<&str, &MacroRow> = + registry.iter().map(|r| (r.name.as_str(), r)).collect(); + + // Providers whose setup must run: every `// use` lib (annotation order — + // even with zero selected macros, the explicit `use` carries its ATTACH + // side effects), then the provider of each selected macro (sorted). + let mut providers: Vec<&str> = use_libs.iter().map(String::as_str).collect(); + let selected_providers: BTreeSet<&str> = selected + .iter() + .filter_map(|n| by_name.get(n.as_str()).map(|r| r.provider_path.as_str())) + .collect(); + for p in selected_providers { + if !providers.contains(&p) { + providers.push(p); + } + } + + let mut injected: Vec = Vec::new(); + let mut seen_setup: HashSet = HashSet::new(); + for provider in providers { + let Some(statements) = lib_bodies.get(provider) else { + return Err(Error::ExecutionErr(format!( + "macro library `{provider}` has registry entries but its deployed script could \ + not be loaded" + ))); + }; + for s in statements { + if let LibStatement::Setup(stmt) = s { + let stmt = format!("{};", stmt.trim_end_matches(';').trim_end()); + if seen_setup.insert(stmt.clone()) { + injected.push(stmt); + } + } + } + } + + if selected.is_empty() { + return Ok(injected); + } + let defs: BTreeMap = selected + .iter() + .filter_map(|n| by_name.get(n.as_str()).map(|r| (n.clone(), r.body.clone()))) + .collect(); + let order = topo_order_macros(selected, &defs).map_err(Error::ExecutionErr)?; + for name in order { + let r = by_name.get(name.as_str()).ok_or_else(|| { + Error::ExecutionErr(format!("workspace macro `{name}` has no registry row")) + })?; + injected.push(macro_create_statement( + &r.name, + &r.params, + r.is_table_macro, + &r.body, + )); + } + Ok(injected) +} + +// Weave the injected blocks into the user's statement list. DuckDB +// bind-checks macro bodies at CREATE, in both directions: +// - an injected definition that a LOCAL macro calls must land *before* +// that local definition (pulling its own injected dependencies with it); +// - an injected definition that references a local macro must stay *after* +// it — a conflict between the two requirements is an error, not a +// silent mis-ordering. +// The default slot (no local interplay) is after the leading prefix of setup +// statements and local definitions, as before. Injected setup statements +// (library ATTACHes) go at the earliest slot any injected definition landed +// on: injected bodies are self-contained w.r.t. their own library's setup +// and never depend on the consumer's setup. +fn weave_macro_blocks(blocks: Vec, injected: Vec) -> Result> { + if injected.is_empty() { + return Ok(blocks); + } + use std::collections::HashSet; + use windmill_parser::duckdb_macros::{ + detect_macro_calls, is_macro_definition, parse_macro_definition, + }; + use windmill_parser::sql_materialize::{classify_block, BlockClass}; + + // Default slot: insert before the first block that is neither setup nor + // a local macro definition (i.e. after the whole leading prefix). + let default_slot = blocks + .iter() + .position(|b| !matches!(classify_block(b), BlockClass::Setup) && !is_macro_definition(b)) + .unwrap_or(blocks.len()); + + // The user's own macro definitions, as placement anchors. + let locals: Vec<(usize, windmill_parser::duckdb_macros::ParsedMacro)> = blocks + .iter() + .enumerate() + .filter_map(|(i, b)| parse_macro_definition(b).map(|m| (i, m))) + .collect(); + let local_names: HashSet = locals.iter().map(|(_, m)| m.name.clone()).collect(); + + // Plan emits [lib setup…, definitions in topo order…]. + let (setup, defs): (Vec, Vec) = + injected.into_iter().partition(|s| !is_macro_definition(s)); + let def_metas: Vec = defs + .iter() + .map(|s| { + parse_macro_definition(s).ok_or_else(|| { + Error::ExecutionErr(format!( + "internal: generated macro statement failed to re-parse: {}", + s.chars().take(80).collect::() + )) + }) + }) + .collect::>()?; + let def_names: HashSet = def_metas.iter().map(|m| m.name.clone()).collect(); + + // Per-injected-definition slot bounds from the local anchors. `slot = i` + // means "insert before block i". + let mut max_slot: Vec = vec![default_slot; defs.len()]; + let mut min_slot: Vec = vec![0; defs.len()]; + for (li, lm) in &locals { + let local_calls = detect_macro_calls(&lm.body, &def_names); + let referenced_locals_of: Vec = def_metas + .iter() + .enumerate() + .filter(|(_, dm)| detect_macro_calls(&dm.body, &local_names).contains(&lm.name)) + .map(|(di, _)| di) + .collect(); + for (di, dm) in def_metas.iter().enumerate() { + if local_calls.contains(&dm.name) { + max_slot[di] = max_slot[di].min(*li); + } + } + for di in referenced_locals_of { + min_slot[di] = min_slot[di].max(*li + 1); + } + } + // An injected definition must not land after any injected definition + // that depends on it: propagate upper bounds backwards through the topo + // order (dependents come later in `defs`). + let mut eff_slot = max_slot.clone(); + for i in (0..defs.len()).rev() { + for j in (i + 1)..defs.len() { + if detect_macro_calls(&def_metas[j].body, &def_names).contains(&def_metas[i].name) { + eff_slot[i] = eff_slot[i].min(eff_slot[j]); + } + } + if min_slot[i] > eff_slot[i] { + return Err(Error::ExecutionErr(format!( + "workspace macro `{}` and this script's own macro definitions have conflicting \ + order requirements (a local macro calls it while it references a local macro \ + defined later); reorder the local definitions", + def_metas[i].name + ))); + } + } + // Library setup runs before everything: the statements are self-contained + // (validated plain/non-managed at lib deploy, within-lib order preserved) + // and ANY consumer statement may depend on them at bind time — including + // a leading local macro definition that shadows a lib macro (so no def is + // injected) but still references the lib's ATTACHed catalog. + let mut out: Vec = Vec::with_capacity(blocks.len() + defs.len() + setup.len()); + out.extend(setup); + for slot in 0..=blocks.len() { + for (di, d) in defs.iter().enumerate() { + if eff_slot[di] == slot { + out.push(d.clone()); + } + } + if let Some(b) = blocks.get(slot) { + out.push(b.clone()); + } + } + Ok(out) +} + +// Workspace-macro injection (`// macros` libraries): fetch the registry, plan +// the needed `CREATE OR REPLACE TEMP MACRO` blocks and splice them into the +// job's statement list. Late-bound: every run reads the current registry, so a +// lib redeploy applies to the next run. On agent (Http) workers — no DB — the +// implicit path silently degrades (a called macro then fails with DuckDB's +// clear Catalog Error) but an explicit `// use` hard-errors like custom tests. +async fn inject_workspace_macros( + conn: &Connection, + w_id: &str, + is_macro_lib: bool, + use_libs: &[String], + blocks: Vec, +) -> Result> { + if is_macro_lib { + // A library run executes its own definitions; nothing to inject. + return Ok(blocks); + } + let db = match conn { + Connection::Sql(db) => db, + Connection::Http(_) => { + if !use_libs.is_empty() { + return Err(Error::ExecutionErr( + "`// use` requires a server worker (not supported on agent workers in v1)" + .to_string(), + )); + } + return Ok(blocks); + } + }; + let registry = fetch_macro_registry(db, w_id).await?; + if registry.is_empty() && use_libs.is_empty() { + return Ok(blocks); + } + if select_workspace_macros(&blocks, ®istry, use_libs)?.is_empty() && use_libs.is_empty() { + return Ok(blocks); + } + // Every relevant provider's deployed body is fetched: its setup + // statements are injected ahead of the definitions (a macro body may + // reference its own lib's ATTACH, which must run before the injected + // CREATE binds), and its own `// use` declarations are honored + // transitively — so the loop fetches lazily and re-resolves until no new + // library appears. Content fetches are cached by hash. + let mut lib_bodies: std::collections::BTreeMap< + String, + Vec, + > = Default::default(); + let mut lib_uses: std::collections::BTreeMap> = Default::default(); + let (selected, effective_use) = loop { + let (selected, effective) = + resolve_macro_selection(&blocks, ®istry, use_libs, &lib_uses)?; + let mut relevant: Vec = effective.clone(); + for name in &selected { + if let Some(r) = registry.iter().find(|r| &r.name == name) { + if !relevant.contains(&r.provider_path) { + relevant.push(r.provider_path.clone()); + } + } + } + let missing: Vec = relevant + .into_iter() + .filter(|l| !lib_bodies.contains_key(l)) + .collect(); + if missing.is_empty() { + break (selected, effective); + } + for path in missing { + let content = fetch_macro_lib_body(conn, w_id, &path).await?; + let statements = windmill_parser::duckdb_macros::parse_macro_library(&content) + .map_err(|e| { + Error::ExecutionErr(format!("macro library `{path}`: invalid content: {e}")) + })?; + lib_uses.insert( + path.clone(), + windmill_parser::asset_parser::parse_pipeline_annotations(&content).use_libs, + ); + lib_bodies.insert(path, statements); + } + }; + let injected = plan_macro_injection(&selected, ®istry, &effective_use, &lib_bodies)?; + weave_macro_blocks(blocks, injected) +} + +// Fetch a macro library's deployed body — for its setup statements; the +// macro definitions themselves come from the registry. Used both for `// use` +// libs and for the provider of every selected macro. Same server-worker +// fetch path as custom data tests. +async fn fetch_macro_lib_body(conn: &Connection, w_id: &str, path: &str) -> Result { + let Connection::Sql(db) = conn else { + return Err(Error::ExecutionErr( + "workspace macros require a server worker (not supported on agent workers in v1)" + .to_string(), + )); + }; + let hash = windmill_common::get_latest_script_hash(db, path, w_id) + .await? + .ok_or_else(|| { + Error::ExecutionErr(format!( + "macro library `{path}`: no deployed script found at this path" + )) + })?; + let content = + crate::get_script_content_by_hash(&windmill_common::scripts::ScriptHash(hash), w_id, conn) + .await?; + if !matches!( + content.language, + Some(windmill_common::scripts::ScriptLang::DuckDb) + ) { + return Err(Error::ExecutionErr(format!( + "macro library `{path}`: must be a DuckDB `// macros` library (got language {:?})", + content.language + ))); + } + Ok(content.content) +} + // Pull a named i64 field (`snapshot_id` / `rows`) out of the trailing summary // read — which in wrap mode is the job result. Shape-tolerant (object / array / // nested), returns None if absent (literal mode, or capture failed). @@ -559,6 +1036,11 @@ pub async fn do_duckdb( // `s3://` URIs — at run time. let sig = parse_duckdb_sig(query)?.args; + // `// macros` / `// use` also come off the ORIGINAL script text (the + // materialize rewrite strips the annotation comments). Drives the + // workspace-macro injection after the ATTACH-transform pass below. + let macro_ann = windmill_parser::asset_parser::parse_pipeline_annotations(query); + let materialized_query; let query: &str = match &materialize { Some((Some(rewritten), _)) => { @@ -670,6 +1152,19 @@ pub async fn do_duckdb( v }; + // Workspace macros: splice `CREATE OR REPLACE TEMP MACRO` blocks for + // registry macros this script calls (plus whole `// use` libraries) + // after the setup/ATTACH prefix — post-transform, so macro bodies can + // reference the attached catalogs when DuckDB bind-checks the CREATE. + let query_block_list = inject_workspace_macros( + conn, + &job.workspace_id, + macro_ann.macros, + ¯o_ann.use_libs, + query_block_list, + ) + .await?; + let base_internal_url = client.base_internal_url.clone(); let w_id = job.workspace_id.clone(); let job_dir = job_dir.to_string(); @@ -1446,6 +1941,374 @@ pub struct Arg { mod tests { use super::*; + fn mrow(name: &str, params: &str, body: &str, is_table: bool, provider: &str) -> MacroRow { + MacroRow { + name: name.to_string(), + params: params.to_string(), + body: body.to_string(), + is_table_macro: is_table, + provider_path: provider.to_string(), + } + } + + fn blocks(list: &[&str]) -> Vec { + list.iter().map(|s| s.to_string()).collect() + } + + // Mimics the shell: resolve (incl. transitive library `// use`), parse the + // given lib sources, plan. `libs` maps provider path → deployed source + // (must cover every relevant provider). + fn select_and_plan( + b: &[String], + registry: &[MacroRow], + use_libs: &[&str], + libs: &[(&str, &str)], + ) -> Result> { + use windmill_parser::asset_parser::parse_pipeline_annotations; + use windmill_parser::duckdb_macros::parse_macro_library; + let use_libs: Vec = use_libs.iter().map(|s| s.to_string()).collect(); + let lib_uses = libs + .iter() + .map(|(p, src)| (p.to_string(), parse_pipeline_annotations(src).use_libs)) + .collect(); + let (selected, effective) = resolve_macro_selection(b, registry, &use_libs, &lib_uses)?; + let lib_bodies = libs + .iter() + .map(|(p, src)| (p.to_string(), parse_macro_library(src).unwrap())) + .collect(); + plan_macro_injection(&selected, registry, &effective, &lib_bodies) + } + + #[test] + fn macro_injection_detects_and_orders_transitively() { + // consumer calls `outer`; `outer` calls `inner` — both injected, inner first. + let registry = vec![ + mrow("outer", "a", "inner(a) + 1", false, "f/lib/m"), + mrow("inner", "a", "a * 2", false, "f/lib/m"), + mrow("unused", "a", "a", false, "f/lib/m"), + ]; + let b = blocks(&["ATTACH 'x.duckdb' AS ext;", "SELECT outer(1);"]); + let injected = select_and_plan(&b, ®istry, &[], &[("f/lib/m", "")]).unwrap(); + assert_eq!( + injected, + vec![ + "CREATE OR REPLACE TEMP MACRO inner(a) AS a * 2;".to_string(), + "CREATE OR REPLACE TEMP MACRO outer(a) AS inner(a) + 1;".to_string(), + ] + ); + } + + #[test] + fn implicit_call_injects_provider_setup() { + // A macro whose body references its own lib's ATTACH must carry that + // setup even on the implicit (detection) path — DuckDB bind-checks the + // body at CREATE, so `ext` must be attached first. + let registry = vec![mrow( + "lookup", + "k", + "(SELECT v FROM ext.kv WHERE key = k)", + false, + "f/lib/m", + )]; + let b = blocks(&["SELECT lookup('a');"]); + let injected = select_and_plan( + &b, + ®istry, + &[], + &[( + "f/lib/m", + "ATTACH 'ext.duckdb' AS ext;\nCREATE MACRO lookup(k) AS (SELECT v FROM ext.kv WHERE key = k);", + )], + ) + .unwrap(); + assert_eq!(injected[0], "ATTACH 'ext.duckdb' AS ext;"); + assert!(injected[1].contains("TEMP MACRO lookup(k)")); + } + + #[test] + fn provider_lib_use_is_honored_transitively() { + // Lib B's macro calls `base_macro` inside a string (invisible to + // lexical detection), so B declares `// use f/lib/base`. A consumer + // that merely calls B's macro must still get base's whole library — + // the dynamic dependency is encapsulated in B, not leaked to every + // consumer. + let registry = vec![ + mrow( + "str_macro", + "", + "(SELECT v FROM query('SELECT base_macro() AS v'))", + false, + "f/lib/b", + ), + mrow("base_macro", "", "42", false, "f/lib/base"), + ]; + let b = blocks(&["SELECT str_macro();"]); + let injected = select_and_plan( + &b, + ®istry, + &[], + &[ + ( + "f/lib/b", + "-- macros\n-- use f/lib/base\nCREATE MACRO str_macro() AS (SELECT v FROM query('SELECT base_macro() AS v'));", + ), + ("f/lib/base", "-- macros\nCREATE MACRO base_macro() AS 42;"), + ], + ) + .unwrap(); + assert!( + injected + .iter() + .any(|s| s.contains("TEMP MACRO base_macro()")), + "{injected:?}" + ); + assert!(injected + .iter() + .any(|s| s.contains("TEMP MACRO str_macro()"))); + // Both defs are injected; string-hidden deps carry no topo edge, so + // their relative order falls back to name-sorted ties (deterministic + // for this input — the assertion pins the current behavior). + let base_idx = injected + .iter() + .position(|s| s.contains("TEMP MACRO base_macro()")) + .unwrap(); + let str_idx = injected + .iter() + .position(|s| s.contains("TEMP MACRO str_macro()")) + .unwrap(); + assert!(base_idx < str_idx, "{injected:?}"); + } + + #[test] + fn duplicate_setup_across_libs_is_deduped() { + // Two libs attaching the same catalog identically must not double-ATTACH. + let registry = vec![ + mrow("m1", "a", "a", false, "f/lib/one"), + mrow("m2", "a", "a", false, "f/lib/two"), + ]; + let b = blocks(&["SELECT m1(1), m2(2);"]); + let src = "ATTACH 'ext.duckdb' AS ext;\nCREATE MACRO m1(a) AS a;"; + let src2 = "ATTACH 'ext.duckdb' AS ext;\nCREATE MACRO m2(a) AS a;"; + let injected = select_and_plan( + &b, + ®istry, + &[], + &[("f/lib/one", src), ("f/lib/two", src2)], + ) + .unwrap(); + assert_eq!( + injected + .iter() + .filter(|s| s.starts_with("ATTACH 'ext.duckdb'")) + .count(), + 1 + ); + } + + #[test] + fn macro_injection_empty_when_nothing_called() { + let registry = vec![mrow("m", "a", "a", false, "f/lib/m")]; + let b = blocks(&["SELECT 1;"]); + assert!(select_workspace_macros(&b, ®istry, &[]) + .unwrap() + .is_empty()); + } + + #[test] + fn local_definition_wins_over_registry() { + // A consumer defining its own `dbl` must never get the registry's + // version injected — a library deploy can't change this job. + let registry = vec![mrow("dbl", "a", "a * 10", false, "f/lib/m")]; + let b = blocks(&["CREATE TEMP MACRO dbl(a) AS a * 2;", "SELECT dbl(4);"]); + assert!(select_workspace_macros(&b, ®istry, &[]) + .unwrap() + .is_empty()); + } + + #[test] + fn use_lib_setup_survives_when_no_macros_selected() { + // Every lib macro shadowed locally — the explicit `// use` must still + // carry the lib's setup statements (its ATTACH side effects). + let registry = vec![mrow("dbl", "a", "a * 10", false, "f/lib/m")]; + let b = blocks(&["CREATE TEMP MACRO dbl(a) AS a * 2;", "SELECT dbl(4);"]); + let injected = select_and_plan( + &b, + ®istry, + &["f/lib/m"], + &[( + "f/lib/m", + "ATTACH 'ext.duckdb' AS ext;\nCREATE MACRO dbl(a) AS a * 10;", + )], + ) + .unwrap(); + assert_eq!(injected, vec!["ATTACH 'ext.duckdb' AS ext;".to_string()]); + } + + #[test] + fn weave_lands_after_local_definition_it_references() { + // Injected bodies may call a local macro (local-wins excludes it from + // the registry set), so the leading local CREATE must run first. + let b = blocks(&[ + "ATTACH 'x' AS a;", + "CREATE MACRO local_dbl(a) AS a * 2;", + "SELECT registry_m(1);", + ]); + let out = weave_macro_blocks( + b, + vec!["CREATE OR REPLACE TEMP MACRO registry_m(a) AS local_dbl(a) + 1;".into()], + ) + .unwrap(); + assert_eq!( + out[2], + "CREATE OR REPLACE TEMP MACRO registry_m(a) AS local_dbl(a) + 1;" + ); + } + + #[test] + fn weave_lands_before_local_definition_that_calls_it() { + // The inverse direction: a LOCAL macro whose body calls a registry + // macro bind-checks at its own CREATE, so the injected definition + // (and its library setup) must come first. + let b = blocks(&[ + "CREATE MACRO outer(x) AS shared_inner(x) + 1;", + "SELECT outer(1);", + ]); + let out = weave_macro_blocks( + b, + vec![ + "ATTACH 'ext.duckdb' AS ext;".into(), + "CREATE OR REPLACE TEMP MACRO shared_inner(x) AS x * 2;".into(), + ], + ) + .unwrap(); + assert_eq!( + out, + blocks(&[ + "ATTACH 'ext.duckdb' AS ext;", + "CREATE OR REPLACE TEMP MACRO shared_inner(x) AS x * 2;", + "CREATE MACRO outer(x) AS shared_inner(x) + 1;", + "SELECT outer(1);", + ]) + ); + } + + #[test] + fn weave_pulls_injected_dependencies_before_the_calling_local() { + // outer(local) calls injected `mid`, whose body calls injected `base`: + // both must precede the local definition, base before mid. + let b = blocks(&["CREATE MACRO outer(x) AS mid(x) + 1;", "SELECT outer(1);"]); + let out = weave_macro_blocks( + b, + vec![ + "CREATE OR REPLACE TEMP MACRO base(x) AS x * 2;".into(), + "CREATE OR REPLACE TEMP MACRO mid(x) AS base(x) + 1;".into(), + ], + ) + .unwrap(); + let pos = |needle: &str| out.iter().position(|s| s.contains(needle)).unwrap(); + assert!(pos("MACRO base(x)") < pos("MACRO mid(x)")); + assert!(pos("MACRO mid(x)") < pos("MACRO outer(x)")); + } + + #[test] + fn weave_conflicting_local_order_errors() { + // local_a calls injected X; X references local_b, defined after + // local_a — unsatisfiable, must error rather than silently mis-order. + let b = blocks(&[ + "CREATE MACRO local_a(x) AS conflicted(x);", + "CREATE MACRO local_b(x) AS x * 3;", + "SELECT local_a(1);", + ]); + let err = weave_macro_blocks( + b, + vec!["CREATE OR REPLACE TEMP MACRO conflicted(x) AS local_b(x) + 1;".into()], + ) + .unwrap_err() + .to_string(); + assert!(err.contains("conflicting"), "{err}"); + } + + #[test] + fn use_lib_injects_all_macros_and_setup() { + let registry = vec![ + mrow("m1", "a", "a", false, "f/lib/m"), + mrow("m2", "", "SELECT 1", true, "f/lib/m"), + ]; + let b = blocks(&["SELECT 'no calls here';"]); + let injected = select_and_plan( + &b, + ®istry, + &["f/lib/m"], + &[( + "f/lib/m", + "ATTACH 'ext.duckdb' AS ext;\nCREATE MACRO m1(a) AS a;\nCREATE MACRO m2() AS TABLE SELECT 1;", + )], + ) + .unwrap(); + assert_eq!(injected[0], "ATTACH 'ext.duckdb' AS ext;"); + assert!(injected[1..].iter().any(|s| s.contains("TEMP MACRO m1(a)"))); + assert!(injected[1..] + .iter() + .any(|s| s.contains("TEMP MACRO m2() AS TABLE"))); + } + + #[test] + fn use_lib_without_registry_rows_errors() { + let b = blocks(&["SELECT 1;"]); + let err = select_workspace_macros(&b, &[], &["f/lib/gone".to_string()]) + .unwrap_err() + .to_string(); + assert!(err.contains("no deployed macro library"), "{err}"); + } + + #[test] + fn weave_lands_after_setup_prefix() { + let b = blocks(&[ + "INSTALL ducklake;", + "ATTACH 'ducklake:postgres:...' AS lake (DATA_PATH 's3://x');", + "CREATE TABLE IF NOT EXISTS lake.t AS SELECT 1;", + "SELECT dbl(1);", + ]); + let out = weave_macro_blocks( + b, + vec!["CREATE OR REPLACE TEMP MACRO dbl(a) AS a * 2;".into()], + ) + .unwrap(); + assert_eq!(out[2], "CREATE OR REPLACE TEMP MACRO dbl(a) AS a * 2;"); + assert_eq!(out.len(), 5); + } + + #[test] + fn weave_setup_runs_before_all_user_blocks() { + // Injected library setup is self-contained and anything in the + // consumer may depend on it at bind time — it always goes first. + let out = weave_macro_blocks(blocks(&["ATTACH 'x' AS a;"]), blocks(&["m;"])).unwrap(); + assert_eq!(out, blocks(&["m;", "ATTACH 'x' AS a;"])); + let out = weave_macro_blocks(vec![], blocks(&["m;"])).unwrap(); + assert_eq!(out, blocks(&["m;"])); + } + + #[test] + fn weave_use_setup_precedes_local_shadow_that_references_it() { + // `// use` with every lib macro shadowed locally: no defs are + // injected, but the local shadow's body references the lib's + // ATTACHed catalog — the injected setup must still run first or the + // local CREATE bind-fails. + let b = blocks(&[ + "CREATE TEMP MACRO kv_lookup(k) AS (SELECT v FROM ext.kv WHERE key = k);", + "SELECT kv_lookup('a');", + ]); + let out = weave_macro_blocks(b, vec!["ATTACH 'ext.duckdb' AS ext;".into()]).unwrap(); + assert_eq!( + out, + blocks(&[ + "ATTACH 'ext.duckdb' AS ext;", + "CREATE TEMP MACRO kv_lookup(k) AS (SELECT v FROM ext.kv WHERE key = k);", + "SELECT kv_lookup('a');", + ]) + ); + } + #[test] fn cgroup_bytes_unlimited_or_invalid_returns_none() { assert_eq!(cgroup_bytes_to_duckdb_memory_limit(0), None); @@ -1520,6 +2383,121 @@ mod tests { assert!(!rewritten.contains("-- $file")); } + // A `// partitioned` script referencing `$partition` needs no manual + // `-- $partition (text)` declaration: the parser auto-declares the arg, so + // the executor binds the injected `partition` job arg instead of failing + // with duckdb's "Wrong number of parameters" at prepare time. + #[test] + fn partitioned_auto_declares_partition_arg() { + let script = "// partitioned daily\n\ + // materialize ducklake://main/sales_daily\n\ + SELECT $partition AS day, count(*) AS n FROM dl.sales WHERE day = $partition"; + + let sig = parse_duckdb_sig(script).expect("sig parses").args; + let partition_arg = sig + .iter() + .find(|a| a.name == "partition") + .expect("`partition` auto-declared"); + assert_eq!(partition_arg.otyp.as_deref(), Some("text")); + + // The wrapped query keeps the `$partition` references so the parsed sig + // binds them at run time. + let (rewritten, _) = build_materialized_query( + script, + Some("2026-07-02"), + &std::collections::HashMap::new(), + ) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("managed mode rewrites the query"); + assert!( + rewritten.contains("$partition"), + "wrapped query must keep the `$partition` reference, got:\n{rewritten}" + ); + } + + // SCD2 managed mode wraps the SELECT into the diff → close-old → open-new + // shape (unit-covered in the parser's codegen tests); here we pin the + // executor-level wiring: the natural key flows through and the wrap is + // generated (not the manual track-only path). + #[test] + fn materialize_scd2_wraps_with_history() { + // Primary spelling: `key=history` on a merge. + let script = "-- materialize ducklake://main/dim key=id history track=name\n\ + SELECT id, name FROM dl.src"; + let (rewritten, _) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("scd2 is managed — must rewrite"); + assert!( + rewritten.contains("valid_from"), + "adds SCD2 columns:\n{rewritten}" + ); + assert!(rewritten.contains("is_current")); + assert!( + rewritten.contains("_wm_scd2_changed"), + "captures changed keys" + ); + assert!( + rewritten.contains("UPDATE _wm_target.dim SET valid_to"), + "closes prior version" + ); + assert!( + rewritten.contains("CREATE VIEW IF NOT EXISTS _wm_target.dim_current"), + "emits the consumer-convenience current view" + ); + // default is soft-delete — no deleted-key set without `deletes=close` + assert!(!rewritten.contains("_wm_scd2_deleted")); + assert!(!rewritten.contains("MERGE INTO")); + } + + #[test] + fn materialize_scd2_deletes_close_wires_through() { + let script = "-- materialize ducklake://main/dim key=id history deletes=close\n\ + SELECT id, name FROM dl.src"; + let (rewritten, _) = + build_materialized_query(script, None, &std::collections::HashMap::new()) + .expect("materialize builds") + .expect("materialize present"); + let rewritten = rewritten.expect("scd2 is managed — must rewrite"); + assert!( + rewritten.contains("_wm_scd2_deleted"), + "deletes=close adds the vanished-key set + close:\n{rewritten}" + ); + } + + #[test] + fn materialize_scd2_requires_key() { + let script = "-- materialize scd2 ducklake://main/dim\nSELECT id, name FROM dl.src"; + let err = match build_materialized_query(script, None, &std::collections::HashMap::new()) { + Err(e) => e, + Ok(_) => panic!("scd2 without key must error"), + }; + assert!( + format!("{err}").contains("requires a natural key"), + "got: {err}" + ); + } + + #[test] + fn materialize_scd2_rejects_partitioned() { + let script = "-- pipeline\n-- partitioned daily\n\ + -- materialize scd2 ducklake://main/dim key=id\nSELECT id, name FROM dl.src"; + let err = match build_materialized_query( + script, + Some("2026-07-01"), + &std::collections::HashMap::new(), + ) { + Err(e) => e, + Ok(_) => panic!("partitioned + scd2 must error"), + }; + assert!( + format!("{err}").contains("not supported with scd2"), + "got: {err}" + ); + } + // Tests for parse_attach_db_resource function #[test] fn test_parse_attach_db_resource_postgres_res_prefix() { diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index a1a26d0630..98c1b5bba7 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -615,6 +615,25 @@ pub async fn do_postgresql( return Err(Error::BadRequest("Missing database argument".to_string())); }; + // Surface in the job logs (not just the worker logs) when a verify-ca/verify-full + // resource is connecting without actually verifying the server certificate, so the + // person running the query can see and fix the misconfiguration. + if database.verify_mode_skips_verification() { + windmill_queue::append_logs( + &job.id, + &job.workspace_id, + format!( + "warning: sslmode={} but the server's TLS certificate is not being verified \ + (accept_invalid_certs is enabled, or no root certificate is configured). Set \ + accept_invalid_certs to false or provide root_certificate_pem to verify the \ + server identity.\n", + database.sslmode.as_deref().unwrap_or("") + ), + conn, + ) + .await; + } + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let collection_strategy = if annotations.raw_output || annotations.return_last_result { // raw_output emits a single envelope from the last statement, so the @@ -629,10 +648,26 @@ pub async fn do_postgresql( // Include use_iam_auth in cache key to distinguish IAM vs non-IAM connections to the same host. // The cache key is static (doesn't include the token), which is correct because PostgreSQL // connections remain valid after initial auth — fresh tokens are generated on cache miss. + // + // to_uri() collapses require/verify-ca/verify-full to the same string, so the TLS verification + // inputs are folded into the key separately. Without this a connection established under a + // weaker sslmode (or a different root cert) could be reused for a stricter request, undoing + // the verification configured in PgDatabase::configure_pg_tls_verification. + let tls_disc = { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + database.sslmode.hash(&mut h); + database.root_certificate_pem.hash(&mut h); + database.accept_invalid_certs.hash(&mut h); + h.finish() + }; + // to_uri() already ends with `?sslmode=...`, so append further key segments + // with `&` to keep database_string a well-formed URI (it is only ever a cache + // key, but a malformed one would mislead anyone who later logs or parses it). let database_string = if use_iam_auth { - format!("{}?iam=true", database.to_uri()) + format!("{}&iam=true&tls={tls_disc:x}", database.to_uri()) } else { - database.to_uri() + format!("{}&tls={tls_disc:x}", database.to_uri()) }; let database_string_clone = database_string.clone(); diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index f8cc748bdb..e3d8172bfe 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.742.0"; +export const VERSION = "v1.746.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/bun.lock b/cli/bun.lock index c94551beb3..50bcc48222 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -19,6 +19,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -288,6 +289,8 @@ "utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="], + "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.740.0", "", {}, "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw=="], + "windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="], "windmill-parser-wasm-go": ["windmill-parser-wasm-go@1.510.1", "", {}, "sha512-HOkk6LXK0wrwvkn+zjm3Gxo90HmyL6TYqmLo2yp8fZuppy7GOngT27zwYeBtwONiPyvDKskzoqPQoEfd8VuUsQ=="], diff --git a/cli/package-lock.json b/cli/package-lock.json index c502b3b25e..949b44e948 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -20,6 +20,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -1409,6 +1410,11 @@ "node": ">= 4" } }, + "node_modules/windmill-parser-wasm-asset": { + "version": "1.740.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.740.0.tgz", + "integrity": "sha512-Dgn5sQ93vJpqTQkv9iAy25+bqH2bUnIMdCfERb2Tia0Ql9T6iHbQhi4yzH9FbGW4Drv9GP46Qyetphqvp8vFOw==" + }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", "resolved": "https://registry.npmjs.org/windmill-parser-wasm-csharp/-/windmill-parser-wasm-csharp-1.510.1.tgz", diff --git a/cli/package.json b/cli/package.json index 2f46e3829b..1cf937d95e 100644 --- a/cli/package.json +++ b/cli/package.json @@ -28,6 +28,7 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.45.2", "tar-stream": "^3.1.7", + "windmill-parser-wasm-asset": "1.740.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", diff --git a/cli/src/commands/pipeline/boundedCascade.ts b/cli/src/commands/pipeline/boundedCascade.ts index 27380d799a..9c7edb9cea 100644 --- a/cli/src/commands/pipeline/boundedCascade.ts +++ b/cli/src/commands/pipeline/boundedCascade.ts @@ -13,7 +13,13 @@ // pipeline.ts. export type BCGraph = { - runnables: { path: string; usage_kind: "script" | "flow" | "job" }[]; + runnables: { + path: string; + usage_kind: "script" | "flow" | "job"; + // `// partitioned ` on the script (daily/hourly/weekly/monthly/dynamic). + // Emitted by both the deployed graph endpoint and the local builder. + partition_kind?: string; + }[]; assets: { kind: string; path: string }[]; edges: { runnable_kind: string; @@ -40,10 +46,14 @@ export const isScriptNode = (id: string): boolean => id.startsWith(SCRIPT_PREFIX export const scriptPathOf = (id: string): string => id.slice(SCRIPT_PREFIX.length); const assetNodeId = (kind: string, path: string): string => `${kind}:${path}`; -// Native trigger kinds that fan out per-event — never bounded-run starts. -// `webhook` / `data_upload` have no trigger row in the graph payload, so a root -// whose only entry is one of those reads as a manual root. -const EVENT_TRIGGER_KINDS = new Set([ +// Native trigger kinds whose scripts must NOT be auto-run by the CLI cascade: +// event triggers fan out per external event, and `webhook`/`data_upload` are UI +// entrypoints that need caller-supplied input (a request body / an uploaded +// S3Object) — previewing any of them with empty args runs the wrong thing. +// The deployed graph omits `webhook`/`data_upload` rows, but the local graph +// emits them, so a `// on data_upload` script would otherwise read as a manual +// root and get auto-run without its upload argument. +const NON_AUTORUN_TRIGGER_KINDS = new Set([ "kafka", "mqtt", "nats", @@ -51,6 +61,8 @@ const EVENT_TRIGGER_KINDS = new Set([ "sqs", "gcp", "email", + "webhook", + "data_upload", ]); /** Resolve an asset URI (`datatable://x`, `s3://b/k`, …) to its node id. */ @@ -121,6 +133,38 @@ function closure(adj: Map>, start: string): Set { export const descendants = (dag: LineageDag, n: string): Set => closure(dag.down, n); export const ancestors = (dag: LineageDag, n: string): Set => closure(dag.up, n); +/** + * Nodes reachable from `starts` over the lineage DAG, treating `barriers` as cut + * points: a barrier node is neither included NOR traversed through. So a node + * reachable ONLY via a barrier is excluded, while one also reachable via another + * path stays. Used for whole-pipeline runs to keep event handlers AND their + * event-only downstream closure out (subtracting only the handler would leave a + * consumer whose producer was skipped, which topoOrder would then run with + * missing/stale inputs). + */ +export function reachableCutting( + dag: LineageDag, + starts: Iterable, + barriers: Set, +): Set { + const seen = new Set(); + const queue: string[] = []; + for (const s of starts) { + if (barriers.has(s) || seen.has(s)) continue; + seen.add(s); + queue.push(s); + } + while (queue.length > 0) { + const n = queue.shift()!; + for (const next of dag.down.get(n) ?? []) { + if (barriers.has(next) || seen.has(next)) continue; + seen.add(next); + queue.push(next); + } + } + return seen; +} + export type BoundedResult = { nodes: Set; reachableEnds: string[]; @@ -151,19 +195,35 @@ export function boundedSet(dag: LineageDag, start: string, ends: string[]): Boun export function validStarts(g: BCGraph): Set { const subscribers = new Set(); const scheduleScripts = new Set(); - const eventScripts = new Set(); + const nonAutorunScripts = new Set(); for (const t of g.triggers ?? []) { if (t.runnable_kind !== "script") continue; if (t.trigger_kind === "asset") subscribers.add(t.runnable_path); else if (t.trigger_kind === "schedule") scheduleScripts.add(t.runnable_path); - else if (EVENT_TRIGGER_KINDS.has(t.trigger_kind)) eventScripts.add(t.runnable_path); + else if (NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) nonAutorunScripts.add(t.runnable_path); } const out = new Set(); for (const r of g.runnables ?? []) { if (r.usage_kind !== "script") continue; const p = r.path; if (scheduleScripts.has(p)) out.add(scriptNodeId(p)); - else if (!subscribers.has(p) && !eventScripts.has(p)) out.add(scriptNodeId(p)); + else if (!subscribers.has(p) && !nonAutorunScripts.has(p)) out.add(scriptNodeId(p)); + } + return out; +} + +/** + * Script node ids that carry a trigger requiring caller-supplied input or + * per-event fanout (kafka/mqtt/nats/postgres/sqs/gcp/email/webhook/data_upload). + * These can't be run with empty args, so a whole-pipeline run must exclude them + * even when they're a lineage descendant of a valid start (not just a root). + */ +export function nonAutorunTriggerScripts(g: BCGraph): Set { + const out = new Set(); + for (const t of g.triggers ?? []) { + if (t.runnable_kind === "script" && NON_AUTORUN_TRIGGER_KINDS.has(t.trigger_kind)) { + out.add(scriptNodeId(t.runnable_path)); + } } return out; } diff --git a/cli/src/commands/pipeline/dev.ts b/cli/src/commands/pipeline/dev.ts new file mode 100644 index 0000000000..3d11dfc2bd --- /dev/null +++ b/cli/src/commands/pipeline/dev.ts @@ -0,0 +1,290 @@ +// `wmill pipeline dev [folder]` — live-preview a data pipeline from local files. +// +// The pipeline analog of `wmill dev` / `wmill app dev`: watch a folder of +// `// pipeline` scripts, rebuild the asset graph from the working tree on every +// save, and push it over a WebSocket to the `/pipeline_dev` page, which renders +// the same graph editor the UI uses and runs the cascade via preview (no deploy). +// Editing stays in the user's own editor; the page live-reloads. + +import { Command } from "@cliffy/command"; +import { colors } from "@cliffy/ansi/colors"; +import * as http from "node:http"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { createHash, randomBytes } from "node:crypto"; +import process from "node:process"; +import * as open from "open"; +import { WebSocket, WebSocketServer } from "ws"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { + mergeConfigWithConfigFile, + type SyncOptions, +} from "../../core/conf.ts"; +import { listSyncCodebases } from "../../utils/codebase.ts"; +import { resolveBindPort } from "../../utils/port-probe.ts"; +import { getConfigDirPath } from "../../../windmill-utils-internal/src/config/config.ts"; +import { buildLocalPipelineGraph, workspaceRoot } from "./localGraph.ts"; + +const PORT = 3201; +// Bind loopback only: each WS frame ships the folder's full script source +// (`scripts[].content` + `temp_script_refs`) with no auth, so it must not be +// reachable from the LAN. The webview connects via `ws://localhost`, and an SSH +// `-L` / devbox port-forward targets 127.0.0.1 on the host, so both still work. +const LISTEN_HOST = "127.0.0.1"; + +interface PipelineDevOpts extends GlobalOptions, SyncOptions { + port?: number; + open?: boolean; + defaultTs?: "bun" | "deno"; + frontend?: string; +} + +// The WS token gates access to local source. Persist it under the user-private +// config dir (0600) so a `pipeline dev` restart reuses it — an already-open +// `/pipeline_dev` page auto-reconnecting with the token from its URL then +// survives the restart, instead of every upgrade being rejected by +// `verifyClient` until the freshly printed URL is reopened. Scoped by +// remote+workspace+root+folder+port (NOT port alone): a same-session restart +// reconnects, but any *different* session — another folder, workspace, remote, or +// local checkout on the same port — gets a different token, so a stale tab can't +// reconnect and receive another session's source. Falls back to an ephemeral +// token if the config dir can't be read/written. +async function stableWsToken( + remote: string, + workspaceId: string, + root: string, + folder: string, + port: number, +): Promise { + // Hash the tuple (NUL-delimited so no value can spoof the boundary) → a + // collision-resistant, filesystem-safe key. A plain sanitized join would let + // different values collide (`a/b` and `a_b` → same name), which would reuse a + // token across sessions and reintroduce the cross-session leak. + const key = createHash("sha256") + .update(`${remote}\0${workspaceId}\0${root}\0${folder}\0${port}`) + .digest("hex") + .slice(0, 32); + let tokenFile: string | undefined; + try { + tokenFile = path.join(await getConfigDirPath(), `pipeline-dev-${key}.token`); + const existing = fs.readFileSync(tokenFile, "utf-8").trim(); + if (existing) return existing; + } catch { + // no reusable token file yet (or config dir unavailable) — mint a fresh one + } + const token = randomBytes(24).toString("base64url"); + if (tokenFile) { + try { + fs.writeFileSync(tokenFile, token, { mode: 0o600 }); + } catch { + // best-effort persistence; fall back to the in-memory token + } + } + return token; +} + +// Resolve the target folder: explicit arg, else auto-detect when cwd sits inside +// `f//…` (the supported `cd f/my_pipeline && wmill pipeline dev` form). +function resolveFolder(root: string, folderArg?: string): string | undefined { + if (folderArg) return folderArg.replace(/^f\//, "").replace(/\/$/, ""); + const rel = path.relative(root, process.cwd()).replaceAll("\\", "/"); + if (rel === "" || rel.startsWith("..")) return undefined; + const segs = rel.split("/"); + if (segs[0] === "f" && segs[1]) return segs[1]; + return undefined; +} + +async function dev(opts: PipelineDevOpts, folderArg?: string) { + const root = workspaceRoot(); + const folder = resolveFolder(root, folderArg); + if (!folder) { + log.error( + colors.red( + "Could not determine the pipeline folder. Pass it explicitly " + + "(`wmill pipeline dev `) or run from inside an `f/` directory.", + ), + ); + process.exit(1); + } + const folderDir = path.join(root, "f", folder); + if (!fs.existsSync(folderDir)) { + log.error(colors.red(`Folder not found on disk: ${folderDir}`)); + process.exit(1); + } + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const merged = await mergeConfigWithConfigFile(opts); + const codebases = await listSyncCodebases(merged); + + // Resolve relative imports from local (not-yet-deployed) content for previews. + // Snapshot at startup like `wmill dev`; restart to refresh. Degrades to + // undefined on older backends. + let tempScriptRefs: Record | undefined; + try { + const { buildPreviewTempScriptRefs } = await import( + "../generate-metadata/generate-metadata.ts" + ); + tempScriptRefs = await buildPreviewTempScriptRefs( + workspace, + merged, + codebases, + { kind: "all" }, + ); + } catch { + // best-effort + } + + const EMPTY_GRAPH = { runnables: [], assets: [], edges: [], triggers: [] }; + async function buildBundle() { + const { graph, scripts } = await buildLocalPipelineGraph({ + root, + folder: folder!, + defaultTs: merged.defaultTs, + }); + return { + type: "pipeline" as const, + folder, + graph, + scripts, + temp_script_refs: tempScriptRefs, + }; + } + + // Don't let a transient build error (e.g. a half-written file) abort startup — + // serve an empty graph and recover on the next save. + let current: Awaited>; + try { + current = await buildBundle(); + } catch (e: any) { + log.error(colors.red(`Initial graph build failed: ${e.message}`)); + current = { type: "pipeline", folder, graph: EMPTY_GRAPH, scripts: [], temp_script_refs: tempScriptRefs }; + } + log.info( + colors.blue( + `Watching f/${folder} — ${current.scripts.length} pipeline script(s)`, + ), + ); + + const clients = new Set(); + function broadcast() { + const msg = JSON.stringify(current); + for (const ws of clients) { + if (ws.readyState === WebSocket.OPEN) ws.send(msg); + } + } + + // Debounced rebuild on any change under the folder. + let timer: ReturnType | undefined; + const watcher = fs.watch(folderDir, { recursive: true }); + watcher.on("change", (_ev, filename) => { + if (filename && filename.toString().endsWith(".lock")) return; + if (timer) clearTimeout(timer); + timer = setTimeout(async () => { + timer = undefined; + try { + current = await buildBundle(); + log.info(colors.cyan(`↻ rebuilt graph (${current.scripts.length} scripts)`)); + broadcast(); + } catch (e: any) { + log.error(colors.red(`Failed to rebuild pipeline graph: ${e.message}`)); + } + }, 150); + }); + watcher.on("error", (e) => log.error(colors.red(`Watcher error: ${e.message}`))); + + const port = await resolveBindPort(opts.port ?? PORT, "wmill pipeline dev", { + info: (m) => log.info(m), + warn: (m) => log.warn(m), + }); + + const server = http.createServer((_req, res) => { + res.writeHead(200); + res.end(); + }); + // Loopback bind keeps the LAN out, but any browser tab can still open a + // `ws://localhost:` connection — and each frame ships the folder's full + // script source. Gate the upgrade on an unguessable per-session token (carried + // in the dev-page URL) so a stray page on the predictable dev port can't + // exfiltrate the source. base64url → safe as a query value. + // Stable across restarts (scoped to remote+workspace+root+folder+port) so an + // already-open page reconnects after a CLI restart (see stableWsToken), not + // just after a transient WS drop. + const wsToken = await stableWsToken( + workspace.remote, + workspace.workspaceId, + root, + folder, + port, + ); + const wss = new WebSocketServer({ + server, + verifyClient: (info) => { + try { + const u = new URL(info.req.url ?? "", "http://localhost"); + return u.searchParams.get("token") === wsToken; + } catch { + return false; + } + }, + }); + wss.on("connection", (ws: WebSocket) => { + clients.add(ws); + // Push the current bundle immediately so the page renders without waiting + // for the first file change. + try { + ws.send(JSON.stringify(current)); + } catch { + // ignore + } + ws.on("close", () => clients.delete(ws)); + ws.on("error", () => clients.delete(ws)); + }); + + // The `/pipeline_dev` page is served by the frontend, not the backend. By + // default we open it on the workspace remote, but that 404s on a remote whose + // deployed frontend predates this route — `--frontend` points the page at a + // locally-run frontend (`REMOTE= npm run dev`) while the API/token + // still target the remote. Normalize to a single trailing slash either way. + const pageBase = (opts.frontend ?? workspace.remote).replace(/\/?$/, "/"); + const url = + `${pageBase}pipeline_dev?workspace=${workspace.workspaceId}` + + `&wm_token=${workspace.token}&folder=${encodeURIComponent(folder)}&port=${port}` + + `&ws_token=${encodeURIComponent(wsToken)}`; + + server.listen(port, LISTEN_HOST, () => { + log.info(colors.green.bold(`🚀 Pipeline dev server on ws://localhost:${port}/ws`)); + log.info(colors.gray(`Open: ${url}`)); + if (opts.open !== false) { + open.default(url).catch((e: any) => + log.warn(colors.yellow(`Failed to open browser: ${e.message}`)), + ); + } + }); + + process.on("SIGINT", () => { + log.info(colors.yellow("\n🛑 Shutting down…")); + watcher.close(); + for (const ws of clients) ws.close(); + server.close(); + process.exit(0); + }); +} + +const command = new Command() + .description( + "Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy).", + ) + .arguments("[folder:string]") + .option("--port ", "Port for the dev WebSocket server.") + .option("--no-open", "Do not open the browser automatically.") + .option( + "--frontend ", + "Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page.", + ) + .action(dev as any); + +export default command; diff --git a/cli/src/commands/pipeline/docs.ts b/cli/src/commands/pipeline/docs.ts new file mode 100644 index 0000000000..b360c8b802 --- /dev/null +++ b/cli/src/commands/pipeline/docs.ts @@ -0,0 +1,253 @@ +// `wmill pipeline docs ` — generate a PIPELINE.md (+ AGENTS.md / CLAUDE.md +// pointer) describing a folder's pipeline so an editor or agentic loop has the +// same context the UI surfaces: the asset DAG, per-script triggers/IO, and the +// schemas of the datatables the pipeline touches. Mirrors the app docs pattern +// (`app/generate_agents.ts:regenerateAgentDocs`) but scoped to a pipeline folder. + +import { writeFile } from "node:fs/promises"; +import { existsSync, readFileSync } from "node:fs"; +import * as path from "node:path"; +import { OpenAPI } from "../../../gen/index.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; +import * as log from "../../core/log.ts"; +import { colors } from "@cliffy/ansi/colors"; +import { GlobalOptions } from "../../types.ts"; +import { + type AssetGraph, + buildLocalPipelineGraph, + workspaceRoot, +} from "./localGraph.ts"; + +const ASSET_KINDS = "s3object,ducklake,datatable,volume"; + +function assetUri(kind: string, p: string): string { + const prefix = kind === "s3object" ? "s3" : kind; + return `${prefix}://${p}`; +} + +async function fetchDeployedGraph( + workspaceId: string, + folder: string, +): Promise { + const res = await fetch( + `${OpenAPI.BASE}/w/${workspaceId}/assets/graph?folder=${encodeURIComponent(folder)}&asset_kinds=${ASSET_KINDS}`, + { headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` } }, + ); + if (!res.ok) { + throw new Error(`GET assets/graph -> ${res.status}: ${await res.text()}`); + } + return (await res.json()) as AssetGraph; +} + +// Render the pipeline graph as a markdown document. +function generatePipelineMarkdown( + folder: string, + graph: AssetGraph, + datatableSchemas: any[], + local: boolean, +): string { + const writesByScript = new Map(); + const readsByScript = new Map(); + for (const e of graph.edges) { + if (e.runnable_kind !== "script") continue; + const uri = assetUri(e.asset_kind, e.asset_path); + if (e.access_type === "w" || e.access_type === "rw") { + (writesByScript.get(e.runnable_path) ?? writesByScript.set(e.runnable_path, []).get(e.runnable_path)!).push(uri); + } + if (e.access_type === "r" || e.access_type === "rw" || e.access_type === undefined) { + (readsByScript.get(e.runnable_path) ?? readsByScript.set(e.runnable_path, []).get(e.runnable_path)!).push(uri); + } + } + const onByScript = new Map(); + const nativeByScript = new Map(); + for (const t of graph.triggers) { + if (t.runnable_kind !== "script") continue; + if (t.trigger_kind === "asset") { + const at = t as Extract; + (onByScript.get(at.runnable_path) ?? onByScript.set(at.runnable_path, []).get(at.runnable_path)!).push(assetUri(at.asset_kind, at.asset_path)); + } else { + (nativeByScript.get(t.runnable_path) ?? nativeByScript.set(t.runnable_path, []).get(t.runnable_path)!).push(t.trigger_kind); + } + } + + const scripts = graph.runnables.filter((r) => r.usage_kind === "script").map((r) => r.path).sort(); + + let md = `# Pipeline \`f/${folder}\` + +${local ? "_Built from local working-tree files (`// pipeline` scripts)._" : "_Built from the deployed workspace asset graph._"} + +A data pipeline is a folder of scripts marked \`// pipeline\` and wired by asset +annotations. A script subscribes to upstream data with \`// on \` and +produces data by reading/writing assets in its body (\`datatable://\`, +\`ducklake://\`, \`s3://\`, \`volume://\`). The cascade runs a producer, then every +downstream subscriber, in topological order. + +- **${scripts.length}** script${scripts.length === 1 ? "" : "s"} · **${graph.assets.length}** asset${graph.assets.length === 1 ? "" : "s"} + +## Scripts + +`; + + for (const s of scripts) { + const on = onByScript.get(s) ?? []; + const native = nativeByScript.get(s) ?? []; + const writes = [...new Set(writesByScript.get(s) ?? [])].sort(); + const reads = [...new Set(readsByScript.get(s) ?? [])].sort(); + md += `### \`${s}\`\n\n`; + if (native.length) md += `- **Triggers:** ${native.map((n) => `\`${n}\``).join(", ")}\n`; + if (on.length) md += `- **On (subscribes to):** ${on.map((u) => `\`${u}\``).join(", ")}\n`; + if (reads.length) md += `- **Reads:** ${reads.map((u) => `\`${u}\``).join(", ")}\n`; + if (writes.length) md += `- **Writes:** ${writes.map((u) => `\`${u}\``).join(", ")}\n`; + if (!native.length && !on.length && !reads.length && !writes.length) { + md += `- _No declared triggers or asset IO._\n`; + } + md += `\n`; + } + + // Datatable schemas, restricted to datatables this pipeline references. + const referencedDatatables = new Set( + graph.assets.filter((a) => a.kind === "datatable").map((a) => a.path.split("/")[0]), + ); + const relevant = (datatableSchemas ?? []).filter((dt: any) => referencedDatatables.has(dt.datatable_name)); + if (relevant.length > 0) { + md += `## Datatable schemas\n\n`; + for (const dt of relevant) { + md += `### Datatable \`${dt.datatable_name}\`\n\n`; + if (dt.error) { + md += `> ⚠️ ${dt.error}\n\n`; + continue; + } + for (const [schemaName, tables] of Object.entries(dt.schemas ?? {})) { + for (const [tableName, columns] of Object.entries(tables as Record)) { + const ref = schemaName === "public" + ? `${dt.datatable_name}/${tableName}` + : `${dt.datatable_name}/${schemaName}:${tableName}`; + md += `- \`datatable://${ref}\`\n`; + for (const [col, type] of Object.entries(columns as Record)) { + md += ` - \`${col}\`: ${type}\n`; + } + } + } + md += `\n`; + } + } + + md += `## Working with this pipeline + +- **Inspect the graph:** \`wmill pipeline show ${folder} --local\` +- **Run the cascade locally (no deploy):** \`wmill pipeline run ${folder} --local\` + (optionally \`--from {#if $workspaceStore != currentWorkspaceId} @@ -819,6 +1091,20 @@ {/snippet} + {#snippet groupActions(groupItems)} + + {@const groupConflicts = groupItems.filter((i) => { + const d = i.diff as WorkspaceItemDiff + return d.ahead > 0 && d.behind > 0 + }).length} + {#if groupConflicts > 0} + + + {groupConflicts} conflict{groupConflicts !== 1 ? 's' : ''} + + {/if} + {/snippet} {#if allCiTests.length > 0}
@@ -909,22 +1195,31 @@ {/if} - {#if !comparison.all_ahead_items_visible || !comparison.all_behind_items_visible} - - {#if !comparison.all_ahead_items_visible && !comparison.all_behind_items_visible} - This fork is ahead and behind its parent - {:else if !comparison.all_behind_items_visible} - This fork is behind of its parent - {:else if !comparison.all_ahead_items_visible} - This fork is ahead of its parent - {/if} - and some of the changes are not visible by you. Only a user with access to the whole context - may deploy or update this fork. You can share the link to this page to someone with proper - permissions to get it deployed. + {@const hiddenDir = mergeIntoParent ? comparison.hidden_ahead : comparison.hidden_behind} + {#if hiddenDir.items.length > 0} + + + {hiddenDir.items.length} + {mergeIntoParent ? 'ahead' : 'behind'} item{hiddenDir.items.length !== 1 ? 's' : ''} + {hiddenDir.items.length !== 1 ? 'are' : 'is'} excluded from the list below — they are not + resolvable as live items and are most likely stale/phantom diff rows: +
    + {#each hiddenDir.items as it} +
  • {hiddenKindLabel(it.kind)} · {it.path}
    • + {/each} +
+ + {:else if mergeIntoParent ? !comparison.all_ahead_items_visible : !comparison.all_behind_items_visible} + + {hiddenDir.total} + {mergeIntoParent ? 'ahead' : 'behind'} item{hiddenDir.total !== 1 ? 's' : ''} + ({formatHiddenByKind(hiddenDir.by_kind)}) + {hiddenDir.total !== 1 ? 'are' : 'is'} not visible to your user and + {hiddenDir.total !== 1 ? 'are' : 'is'} excluded from the list below. You can still + {mergeIntoParent ? 'deploy' : 'update'} the items you can see — share this page with someone + who has full access to include the rest. {/if} {/snippet} @@ -1079,6 +1374,27 @@ Show diff
+ {#if diff.kind === 'resource' || diff.kind === 'variable'} + {#if isPinned(diff.kind, diff.path)} + + workspace-specific + + {:else} + + {/if} + {/if} {/if} {/snippet} @@ -1088,44 +1404,46 @@
- {#if comparison.all_behind_items_visible && comparison.all_ahead_items_visible} -
- {#if mergeIntoParent && !hasOpenDeploymentRequest && !deploymentRequestPanel?.isDialogOpen()} - - {/if} + +
+ {#if mergeIntoParent && !hasOpenDeploymentRequest && !deploymentRequestPanel?.isDialogOpen()} -
- {#if !(mergeIntoParent && !canDeployToParent) && hasUnselectedOnBehalfOf} - - You must set the "on behalf of" user for all items before deploying - - The "run on behalf of" field defines which user's permissions will be - applied during execution. Make sure this is set to an appropriate user - before deploying. - - {/if} + +
+ {#if !(mergeIntoParent && !canDeployToParent) && hasUnselectedOnBehalfOf} + + You must set the "on behalf of" user for all items before deploying + + The "run on behalf of" field defines which user's permissions will be applied + during execution. Make sure this is set to an appropriate user before + deploying. + + {/if} {#if deploymentErrorMessage != ''} @@ -1158,6 +1476,53 @@
+ + {#if pinnedItems.length > 0} +
+
+ Workspace-specific items +
+

+ These resources and variables keep their own value in each environment and are excluded + from the diff. An item that exists on only one side can be seeded onto the other with + "Create in …" (copies the current value, including secrets); it will never overwrite an + existing value. +

+
+ {#each pinnedItems as it (`${it.item_kind}:${it.path}`)} + {@const missingSide = + it.onCurrent && !it.onParent + ? parentWorkspaceId + : !it.onCurrent && it.onParent + ? currentWorkspaceId + : undefined} +
+ {it.item_kind} + {it.path} + {#if missingSide} + + {/if} + +
+ {/each} +
+
+ {/if}
@@ -1210,6 +1575,27 @@ + + { + const it = createConfirm + createConfirm = undefined + if (it) createOnRemote(it) + }} + onCanceled={() => (createConfirm = undefined)} + > +

+ This copies the current value of {createConfirm?.path} + (including any secret value) from + {createConfirm?.onCurrent ? currentWorkspaceId : parentWorkspaceId} + into {createConfirm?.onCurrent ? parentWorkspaceId : currentWorkspaceId}. It stays + workspace-specific afterward, so later promotes won't overwrite it. If it already exists + there, it's left untouched and just marked workspace-specific. +

+ {:else}
No comparison data available
diff --git a/frontend/src/lib/components/CronInput.svelte b/frontend/src/lib/components/CronInput.svelte index b7febcdaca..d2b629d33e 100644 --- a/frontend/src/lib/components/CronInput.svelte +++ b/frontend/src/lib/components/CronInput.svelte @@ -204,18 +204,20 @@
-
- +
+
+ +
{#if !disabled}
{@render cronBuilder()} diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte new file mode 100644 index 0000000000..bdba23c57f --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -0,0 +1,191 @@ + + +{#if isDev && parentId} +
+

+ This is a dev workspace paired with root workspace {parentId}. Promote changes + from the home page banner or the Compare & Deploy page. +

+
+ +
+
+{:else if pairedDev} +
+

+ This workspace's dev workspace is {pairedDev.name} ({pairedDev.id}). Edits to this + workspace are redirected there. +

+
+ {#if pairedDev.isMember} + + {/if} + +
+
+{:else if parentId} +

+ Dev workspace pairing is only available for root workspaces. This workspace is a fork of + {parentId}. +

+{:else} +
+

+ Pair this workspace with a dev workspace: the same code with a different environment (resource + and variable values). Edits are made in the dev workspace and promoted here. +

+
+ Attach an existing workspace as dev + days, (v) => onDaysChange(v ?? 30)} - /> -
+ {#if !liveOnly} +
+ - -
- - {#snippet actions()} - {/snippet} - +
+
diff --git a/frontend/src/lib/components/settings/CloudQuotas.svelte b/frontend/src/lib/components/settings/CloudQuotas.svelte index 61158f13ec..ec597101ee 100644 --- a/frontend/src/lib/components/settings/CloudQuotas.svelte +++ b/frontend/src/lib/components/settings/CloudQuotas.svelte @@ -18,6 +18,7 @@ apps: QuotaInfo variables: QuotaInfo resources: QuotaInfo + forks: QuotaInfo } | undefined = $state(undefined) @@ -87,7 +88,8 @@ { label: 'Flows', key: 'flows', prunable: true }, { label: 'Apps', key: 'apps', prunable: true }, { label: 'Variables', key: 'variables', prunable: false }, - { label: 'Resources', key: 'resources', prunable: false } + { label: 'Resources', key: 'resources', prunable: false }, + { label: 'Forks', key: 'forks', prunable: false } ] diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index 1882ed33dc..395ce00fa2 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -12,7 +12,7 @@ tutorialsToDo, skippedAll } from '$lib/stores' - import { findWorkspaceDescendants } from '$lib/utils/workspaceHierarchy' + import { findWorkspaceDescendants, workspaceIsFork } from '$lib/utils/workspaceHierarchy' import { syncTutorialsTodos } from '$lib/tutorialUtils' import { SIDEBAR_SHOW_SCHEDULES } from '$lib/consts' import { @@ -187,6 +187,8 @@ const forkedDescendants = $derived( $workspaceStore ? findWorkspaceDescendants($workspaceStore, $userWorkspaces ?? []) : [] ) + // Fork/dev workspaces are detected by their parent link, not the `wm-fork-` id prefix. + const currentWsIsFork = $derived(workspaceIsFork($workspaceStore, $userWorkspaces ?? [])) let hasNewChangelogs = $state(false) let recentChangelogs: Changelog[] = $state([]) @@ -550,7 +552,7 @@ } ] : []), - ...($workspaceStore?.startsWith('wm-fork-') + ...(currentWsIsFork ? [ { label: 'Delete Forked Workspace', @@ -884,7 +886,7 @@
-{#if $workspaceStore?.startsWith('wm-fork-')} +{#if currentWsIsFork} - {#if $userStore?.is_admin} + {#if $userStore?.non_member} + Superadmin, not a member of this workspace + {:else if $userStore?.is_admin} Admin of this workspace {:else if $userStore?.operator} Operator in this workspace {/if} + {#if $userStore?.non_member} + + You are not a member, but as a superadmin you can access this workspace. You act here + under the username + {$userStore?.username} + with admin permissions. + + {/if}
diff --git a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte index a834d00317..53ac410cc8 100644 --- a/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte +++ b/frontend/src/lib/components/sidebar/WorkspaceMenu.svelte @@ -11,7 +11,8 @@ clearWorkspaceFromStorage, globalForkModal } from '$lib/stores' - import { isRuleActive } from '$lib/workspaceProtectionRules.svelte' + import { canCreateFork } from '$lib/utils/editInFork' + import { Badge } from '$lib/components/common' import { Building, Plus, Settings, GitFork } from 'lucide-svelte' import MenuButton from '$lib/components/sidebar/MenuButton.svelte' import { Menu, MenuItem } from '$lib/components/meltComponents' @@ -131,7 +132,9 @@ icon={GitFork} iconProps={iconColor ? { style: `color: ${iconColor}` } : undefined} label={removePrefix($workspaceStore ?? '', 'wm-fork-')} - sublabel={parentWorkspace?.name ? `Fork of ${parentWorkspace.name}` : undefined} + sublabel={parentWorkspace?.name + ? `${forkedWorkspace.is_dev_workspace ? 'Dev workspace of' : 'Fork of'} ${parentWorkspace.name}` + : undefined} {isCollapsed} color={$workspaceColor} {trigger} @@ -170,16 +173,26 @@ >
- +
-
- {workspace.name}{workspace.disabled ? ' (user disabled)' : ''} +
+
+ {workspace.name}{workspace.disabled ? ' (user disabled)' : ''} +
+ {#if workspace.is_dev_workspace} + dev + {/if}
{/if} - {#if !strictWorkspaceSelect && !isCloudHosted() && !isRuleActive('DisableWorkspaceForking') && $workspaceStore !== 'admins'} + {#if !strictWorkspaceSelect && (!isCloudHosted() || $isPremiumStore) && canCreateFork($userStore) && $workspaceStore !== 'admins'}
draftSync.deployed} + reserveSpace={draftSync.hasBaseline} getCurrent={() => draftSync.current} onDiscard={() => draftSync.resetToDeployed(initialPath)} disabled={!can_write} diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index 34cf4e92c3..1af45e9f2f 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -368,7 +368,7 @@ {#if mode === 'suspended'} {/if} -
+
\`). Deploy **rejects** \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`) or a non-DuckLake target. For a non-DuckDB node, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, a postgresql \`CREATE TABLE\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. + +\`// materialize \` tells the runtime to write the node's output table **for you**: write the body as a single \`SELECT\` and the runtime wraps it in the create/replace — do **not** also write your own \`CREATE TABLE\` / \`INSERT\`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- \`// materialize append\` → INSERT-append rows (incremental); +- \`// materialize key=\` → merge/upsert on \`\`. + +\`// materialize manual \` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +\`materialize\` pairs with partitioning for incremental pipelines: a \`// partitioned \` node runs **once per partition** (append/merge into a fixed-schema table), and the \`{partition}\` token inside any asset URI is substituted with the current partition value at run time. + +\`materialize\` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: \`f//\`. The folder is the pipeline. +2. Author each node as a **script draft** with \`write_script\` (or \`edit_script\`), language chosen for the work: \`duckdb\` or \`postgresql\` for SQL-shaped data work, \`bun\`/\`python3\` for general transforms. SQL-heavy lakehouse steps usually use \`duckdb\`. +3. Start each body with \`// pipeline\`, then the \`// on\` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then \`// on \` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the \`/pipeline/\` editor open, prefer the dedicated \`build_pipeline_node\` / \`edit_pipeline_node\` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node \`f/sales/orders_ingest\` (runs on a schedule, materializes a DuckLake table): + +\`\`\`sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +\`\`\` + +Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup): + +\`\`\`sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +\`\`\` +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope @@ -2602,7 +2664,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \\"yellow\\", \\"#ffff00\\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands @@ -3032,11 +3094,22 @@ inspect asset-driven pipelines (scripts marked \`// pipeline\`, wired by \`// on - \`--json\` - Output as JSON (for piping to jq) - \`pipeline show \` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal - \`--json\` - Output the raw asset graph as JSON + - \`--local\` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed. - \`pipeline run \` - run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s) - \`--from \` - Start script (short name or path). Defaults to the folder's sole schedule/manual root. - \`--to \` - End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream. - \`--dry-run\` - Print the topological run plan without executing. - \`--json\` - Output the plan as JSON (for piping to jq). + - \`--local\` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files. + - \`--upload \` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable. + - \`--arg \` - Pass a plain run arg to a script in the cascade, as SCRIPT:PARAM=VALUE (VALUE is parsed as JSON when possible, else taken as a string — e.g. daily_report:partition=2026-07-02). Repeatable. + - \`--partition \` - Partition value for \`// partitioned\` scripts in the run (e.g. 2026-06-30) — use it to backfill a past slice. With --local, time kinds (daily/hourly/weekly/monthly) default to the current UTC period when omitted; \`dynamic\` always needs it. Deployed runs without it defer to backend run-start resolution. +- \`pipeline docs \` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop + - \`--local\` - Build the graph from local working-tree files instead of the deployed workspace. +- \`pipeline dev [folder:string]\` - Live-preview a data pipeline from local files: watch an \`f/\` of \`// pipeline\` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy). + - \`--port \` - Port for the dev WebSocket server. + - \`--no-open\` - Do not open the browser automatically. + - \`--frontend \` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page. ### protection-rules @@ -3368,7 +3441,7 @@ workspace related commands - \`--datatable-behavior \` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt) - \`--from-branch \` - Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on (its bound workspace is the parent) and rename the current branch onto wm-fork//. Usually unneeded — from a working branch \`wmill workspace fork\` offers this interactively; from a base branch it creates a fresh fork branch. - \`-y --yes\` - Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for. -- \`workspace delete-fork \` - Delete a forked workspace and git branch +- \`workspace delete-fork \` - Delete a forked workspace - \`-y --yes\` - Skip confirmation prompt - \`workspace merge\` - Compare and deploy changes between a fork and its parent workspace - \`--direction \` - Deploy direction: to-parent or to-fork diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 8f7486d8e9..e7833dd4cf 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -431,11 +431,22 @@ inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal - `--json` - Output the raw asset graph as JSON + - `--local` - Build the graph from local working-tree files (// pipeline scripts) instead of the deployed workspace — no deploy needed. - `pipeline run ` - run a bounded cascade: from a schedule/manual root, fan downstream up to the --to end node(s) - `--from ` - Start script (short name or path). Defaults to the folder's sole schedule/manual root. - `--to ` - End node(s) to stop at — script names/paths or asset URIs (e.g. datatable://main/staged). Repeatable or comma-separated. Omit to run the full downstream. - `--dry-run` - Print the topological run plan without executing. - `--json` - Output the plan as JSON (for piping to jq). + - `--local` - Run the local working-tree scripts via preview (no deploy) instead of the deployed versions; the graph is built from local files. + - `--upload ` - Bind an object to a data_upload/webhook entry point so it runs in the cascade, as SCRIPT[:PARAM]=SOURCE (SOURCE is a local file or an s3://key). Local files are uploaded to the workspace store; the S3Object param is inferred when the script has exactly one. Repeatable. + - `--arg ` - Pass a plain run arg to a script in the cascade, as SCRIPT:PARAM=VALUE (VALUE is parsed as JSON when possible, else taken as a string — e.g. daily_report:partition=2026-07-02). Repeatable. + - `--partition ` - Partition value for `// partitioned` scripts in the run (e.g. 2026-06-30) — use it to backfill a past slice. With --local, time kinds (daily/hourly/weekly/monthly) default to the current UTC period when omitted; `dynamic` always needs it. Deployed runs without it defer to backend run-start resolution. +- `pipeline docs ` - generate PIPELINE.md (+ AGENTS.md pointer) describing a folder's pipeline graph and datatable schemas, for an editor / agentic loop + - `--local` - Build the graph from local working-tree files instead of the deployed workspace. +- `pipeline dev [folder:string]` - Live-preview a data pipeline from local files: watch an `f/` of `// pipeline` scripts, push the working-tree graph to the dev page, and run the cascade via preview (no deploy). + - `--port ` - Port for the dev WebSocket server. + - `--no-open` - Do not open the browser automatically. + - `--frontend ` - Origin serving the /pipeline_dev page (e.g. http://localhost:3000 for a locally-run frontend). Defaults to the workspace remote; use it when the remote's deployed frontend predates the dev page. ### protection-rules @@ -767,7 +778,7 @@ workspace related commands - `--datatable-behavior ` - How to handle datatables: skip, schema_only, or schema_and_data (default: interactive prompt) - `--from-branch ` - Non-interactive override for the 'turn my current working branch into the fork' workflow: base the fork on (its bound workspace is the parent) and rename the current branch onto wm-fork//. Usually unneeded — from a working branch `wmill workspace fork` offers this interactively; from a base branch it creates a fresh fork branch. - `-y --yes` - Skip interactive prompts (defaults datatable behavior to 'skip'). On a non-base branch, requires --from-branch since the base branch can't be prompted for. -- `workspace delete-fork ` - Delete a forked workspace and git branch +- `workspace delete-fork ` - Delete a forked workspace - `-y --yes` - Skip confirmation prompt - `workspace merge` - Compare and deploy changes between a fork and its parent workspace - `--direction ` - Deploy direction: to-parent or to-fork diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 8c8057d48f..26f06fb667 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -381,4 +381,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"The flow will be run with the permissions of the user with this email."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note (e.g., \"yellow\", \"#ffff00\")"},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes, 'group' for notes that group other nodes"},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor"}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside loops, use 'flow_input.iter.value' for the current iteration value","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryAuto":{"type":"object","description":"Automatic context management","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","description":"Explicit message history","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Conversation memory configuration","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly while a condition is true. The loop checks the condition after each iteration. Use stop_after_if on modules to control loop termination","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration. Use stop_after_if to control when the loop ends","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"Short description of what this tool does (shown to the AI)"},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with flow.input syntax."},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}},"required":["provider","user_message","output_type"]},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["tools","type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md new file mode 100644 index 0000000000..a0f2e2492a --- /dev/null +++ b/system_prompts/base/pipeline-base.md @@ -0,0 +1,60 @@ +# Data pipeline authoring + +A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates steps internally. A data pipeline is a set of **independent scripts**, each deployed on its own, that form a DAG by reading and writing shared **storage assets** (DuckLake tables, data tables, S3 objects, volumes, resources) and by declaring execution **triggers**. The pipeline is visualized and edited at `/pipeline/`; every node is a normal workspace script that happens to carry pipeline annotations. When the user asks for a "data pipeline" (or to "ingest / transform / materialize" data across steps), build pipeline-annotated scripts — do NOT build a flow. + +## What makes a script a pipeline node + +A script joins the pipeline when its source begins with the `pipeline` annotation as a top-of-file comment, **written in the script's own comment syntax** — `//` for TS/JS (bun), `--` for SQL (DuckDB/Postgres), `#` for Python/Bash. So it's `-- pipeline` in a DuckDB node, `# pipeline` in a Python node, `// pipeline` in a bun node. Every annotation below uses that same prefix (the `//` shown is the TS form). All other wiring is expressed as annotation comments near the top of the file: + +- `// on ` — declares an execution-DAG **input** (what triggers/feeds this node). `` is either: + - an **asset URI** (the node runs when that asset is produced upstream): `ducklake://main/orders`, `datatable://main/users`, `s3://`, `$res:f/folder/my_resource`, `volume://name/path`. + - a **native trigger kind**: `schedule`, `webhook`, `email`, `kafka`, `mqtt`, `nats`, `postgres`, `sqs`, `gcp`, or `data_upload` (a user-uploaded S3 file). For these the actual trigger row (cron, topic, …) is created separately; the annotation only declares the binding. +- **Outputs** are inferred from what the body writes — a `CREATE TABLE`, a `wmill.writeS3File(...)`, a DuckLake/datatable write. To declare a managed output explicitly, use `// materialize `. +- Optional badges: `// partitioned `, `// freshness ` (e.g. `1h`), `// tag `, `// retry [delay]`, `// data_test ...`. + +## Materialize (the managed output) + +> **`// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** `// materialize` on any other language (`python3`, `bun`, `postgresql`) or a non-DuckLake target. For a non-DuckDB node, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, a postgresql `CREATE TABLE`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. + +`// materialize ` tells the runtime to write the node's output table **for you**: write the body as a single `SELECT` and the runtime wraps it in the create/replace — do **not** also write your own `CREATE TABLE` / `INSERT`. Write strategy: + +- no option → **replace** the whole table each run (full refresh; the only mode whose output columns may change); +- `// materialize append` → INSERT-append rows (incremental); +- `// materialize key=` → merge/upsert on ``. + +`// materialize manual ` opts **out** of managed writes — the script writes its own DDL and the annotation only records the output asset for lineage. + +`materialize` pairs with partitioning for incremental pipelines: a `// partitioned ` node runs **once per partition** (append/merge into a fixed-schema table), and the `{partition}` token inside any asset URI is substituted with the current partition value at run time. + +`materialize` is an output **declaration** on a node — not a command. There is no "materialize run". + +## How to build one in chat + +1. Put every node in the **same folder**: `f//`. The folder is the pipeline. +2. Author each node as a **script draft** with `write_script` (or `edit_script`), language chosen for the work: `duckdb` or `postgresql` for SQL-shaped data work, `bun`/`python3` for general transforms. SQL-heavy lakehouse steps usually use `duckdb`. +3. Start each body with `// pipeline`, then the `// on` input declarations, then the transform that writes the output. +4. **Chain nodes by asset URI**: read an upstream node's output asset, then `// on ` in the downstream node so the edge forms. Reuse exact asset paths from existing nodes rather than inventing parallel ones. +5. Leave nodes as drafts unless the user asks to deploy. A pipeline only "runs" once its scripts are deployed and their triggers exist. + +When the user already has the `/pipeline/` editor open, prefer the dedicated `build_pipeline_node` / `edit_pipeline_node` tools (they stage reviewable, canvas-highlighted proposals). Outside the editor, use the standard script-draft tools with the annotations above. + +## Example (DuckDB → DuckLake, scheduled ingest + downstream transform) + +Node `f/sales/orders_ingest` (runs on a schedule, materializes a DuckLake table): + +```sql +-- pipeline +-- on schedule +-- materialize ducklake://main/orders +SELECT * FROM read_csv('s3://raw/orders/*.csv') +``` + +Node `f/sales/orders_daily` (runs when `orders` is produced, writes a rollup): + +```sql +-- pipeline +-- on ducklake://main/orders +-- materialize ducklake://main/orders_daily +SELECT date_trunc('day', ts) AS day, count(*) AS n +FROM ducklake.main.orders GROUP BY 1 +``` diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 6ee1971f43..98c03c4367 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -2369,6 +2369,7 @@ def main(): flow_base = read_markdown_file(base_dir / "flow-base.md") resources_base = read_markdown_file(base_dir / "resources.md") raw_app_base = read_markdown_file(base_dir / "raw-app.md") + pipeline_base = read_markdown_file(base_dir / "pipeline-base.md") workflow_as_code_base = read_markdown_file(base_dir / "workflow-as-code.md") flow_cli = read_markdown_file(base_dir / "flow-cli.md") flow_chat_special_modules = read_markdown_file(base_dir / "flow-chat-special-modules.md") @@ -2435,6 +2436,7 @@ def main(): 'FLOW_BASE': flow_base, 'RESOURCES_BASE': resources_base, 'RAW_APP_BASE': raw_app_base, + 'PIPELINE_BASE': pipeline_base, 'WORKFLOW_AS_CODE_BASE': workflow_as_code_base, 'FLOW_CHAT_SPECIAL_MODULES': flow_chat_special_modules, @@ -2534,6 +2536,11 @@ export function getRawAppPrompt(): string { return prompts.RAW_APP_BASE; } +// Helper for data pipeline authoring (chat consumers) +export function getPipelinePrompt(): string { + return prompts.PIPELINE_BASE; +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2586,6 +2593,7 @@ export declare function getScriptPrompt(language: string): string; export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(): string; +export declare function getPipelinePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7ae2023a7c..f11853e2bc 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.742.0", + "version": "1.746.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 8edc248c45..e0b19d396f 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.742.0", + "version": "1.746.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index bee795f5f0..c8670bfad7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.742.0 +1.746.0