From e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Thu, 8 Oct 2026 17:31:35 +0200 Subject: [PATCH 01/60] fix: name the open workspace settings tab in the page header breadcrumb (#11617) * feat: name the open workspace settings tab in the page header breadcrumb Co-Authored-By: Claude Opus 5.5 (1M context) * fix: let the page header breadcrumb shrink when nothing else in the bar can Co-Authored-By: Claude Opus 5.5 (1M context) * fix: keep the breadcrumb's slower shrink when a summary shares the bar Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../src/lib/components/NavBreadcrumb.svelte | 9 +++++++++ .../src/lib/components/PageHeaderBar.svelte | 17 ++++++++++++++--- .../lib/components/pageHeaderRegistry.svelte.ts | 3 +++ .../(logged)/workspace_settings/+page.svelte | 12 +++++++++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/NavBreadcrumb.svelte b/frontend/src/lib/components/NavBreadcrumb.svelte index 889fe17377..3081bcca4c 100644 --- a/frontend/src/lib/components/NavBreadcrumb.svelte +++ b/frontend/src/lib/components/NavBreadcrumb.svelte @@ -396,6 +396,15 @@ not this component) — so the lead can run the whole line. onPick={() => {}} /> {:else if section} + {#if section.parent} + + + {section.parent} + + {@render slash()} + {/if} -
+ page the controls beside it belong to. + The 0.3 only holds against a summary or an actions box that shrinks too: alone, a factor + sum below 1 makes flex hand out only that share of the overflow, and the bar overflows + instead. --> +
@@ -161,7 +172,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
- {#if item && (item.summaryContent || item.summary)} + {#if item && hasSummary}
diff --git a/frontend/src/lib/components/pageHeaderRegistry.svelte.ts b/frontend/src/lib/components/pageHeaderRegistry.svelte.ts index 3167d636e9..bd30855e15 100644 --- a/frontend/src/lib/components/pageHeaderRegistry.svelte.ts +++ b/frontend/src/lib/components/pageHeaderRegistry.svelte.ts @@ -24,6 +24,9 @@ export type PageHeaderItem = { /** A page with no item of its own, such as a list: the breadcrumb is the section alone. */ export type PageHeaderSection = { label: string + /** The page the section belongs to, when the section is one pane of it — a settings page's + * active tab. Drawn as a lead segment before `label`. */ + parent?: string /** Rendered in place of `label` when the name is itself a control, such as an editable title. */ content?: Snippet /** Scopes the segment's picker, when the section maps to a kind of item. */ diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 022048bbf7..1eda558a4a 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -1457,6 +1457,12 @@ } ] ) + + // The label of the entry the sidebar highlights, so the breadcrumb and the sidebar name the + // same pane: both read `tab`, which resolves the URL's aliases (`teams`, `deploy_to`, …). + const tabLabel = $derived( + navigationGroups.flatMap((g) => g.items).find((i) => i.id === tab)?.label + ) @@ -1464,7 +1470,11 @@ - +
From 8f28a5860595dbe558e6777a06bb97b6ec6a3e67 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 8 Oct 2026 18:05:37 +0200 Subject: [PATCH 02/60] fix: store a WAC parent's first start and wall duration (#11625) * fix: store a workflow-as-code parent's first start and wall duration Co-Authored-By: Claude Opus 5.5 * fix: keep a parked child's first start in its parent's timeline Co-Authored-By: Claude Opus 5.5 * fix: record a parked step with its whole span in the flow, cover inline checkpoints Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to ab8476e5feedc98b727f7d83acffa1807b3b9c5b This commit updates the EE repository reference after PR #861 was merged in windmill-ee-private. Previous ee-repo-ref: d5d95f51bb175944874e585e5931acb1eddd637a New ee-repo-ref: ab8476e5feedc98b727f7d83acffa1807b3b9c5b Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] --- ...adad56b50619036fab3f2c1bdbbf280a80217.json | 22 +++ ...2f2a64dc559e3228d3b763f68b1ad3491ae6e.json | 45 ++++++ ...a85c53c49e4ef0f42c146324d5c71fbd1e21d.json | 31 ---- ...23bf1c925feebe746299cd1f0eda3f8b949d5.json | 33 ----- ...cd9182a4812b14735e502958eaafeb4c6b33d.json | 17 +++ ...571de793b377dd6058065f51b12c974e5b665.json | 28 ---- ...40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json | 43 ++++++ ...05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json | 22 --- ...db8a21a2eaacf971bf046cef1ee14cf04e323.json | 28 ++++ ...49bf8b3523d8dde1ef6b8f1950423830cc6ed.json | 17 --- backend/ee-repo-ref.txt | 2 +- backend/tests/wac_suspend_started_at.rs | 88 ++++++++++++ backend/windmill-api-users/src/users.rs | 2 +- backend/windmill-queue/src/jobs.rs | 132 +++++++++++++----- backend/windmill-worker/src/bun_executor.rs | 5 +- .../windmill-worker/src/result_processor.rs | 40 ++++-- backend/windmill-worker/src/wac_executor.rs | 10 +- backend/windmill-worker/src/worker_flow.rs | 8 +- 18 files changed, 389 insertions(+), 184 deletions(-) create mode 100644 backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json create mode 100644 backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json delete mode 100644 backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json delete mode 100644 backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json create mode 100644 backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json delete mode 100644 backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json create mode 100644 backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json delete mode 100644 backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json create mode 100644 backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json delete mode 100644 backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json diff --git a/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json new file mode 100644 index 0000000000..1946f55366 --- /dev/null +++ b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217" +} diff --git a/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json new file mode 100644 index 0000000000..ba1cc13fac --- /dev/null +++ b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e" +} diff --git a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json b/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json deleted file mode 100644 index f105dd7825..0000000000 --- a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d" -} diff --git a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json b/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json deleted file mode 100644 index 0b166b32a4..0000000000 --- a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5" -} diff --git a/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json new file mode 100644 index 0000000000..59ed0e1767 --- /dev/null +++ b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d" +} diff --git a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json b/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json deleted file mode 100644 index 2358af807f..0000000000 --- a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665" -} diff --git a/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json new file mode 100644 index 0000000000..ba7a89ef73 --- /dev/null +++ b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json deleted file mode 100644 index ae893b6791..0000000000 --- a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int8", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" -} diff --git a/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json new file mode 100644 index 0000000000..e22953610d --- /dev/null +++ b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323" +} diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json deleted file mode 100644 index 5f86fb97d8..0000000000 --- a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 1acd2de3f0..2de431e127 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a37413ccdd52e65bc240bad3e87084bf4100e119 +ab8476e5feedc98b727f7d83acffa1807b3b9c5b diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs index 5a83730d24..ca364e5993 100644 --- a/backend/tests/wac_suspend_started_at.rs +++ b/backend/tests/wac_suspend_started_at.rs @@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Res Ok(()) } + +/// Each park clears `started_at`, so the completion would otherwise store only the last +/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that +/// started an hour late. The completed row spans the workflow from its first start, and +/// keeps the last segment aside: that is what is metered and what worker-time readers sum. +#[sqlx::test] +async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment( + db: Pool, +) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \ + 'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \ + VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Two parks, each followed by the pull re-stamping `started_at`: the second park must + // not replace the first start with its own segment's. + for resumed_ago in ["10 minutes", "3 seconds"] { + let mut tx = db.begin().await?; + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?; + tx.commit().await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \ + WHERE id = $1", + ) + .bind(job_id) + .bind(resumed_ago) + .execute(&db) + .await?; + } + + let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db) + .await? + .unwrap(); + let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?; + let (_, span) = windmill_queue::add_completed_job( + &db, + &job, + true, + false, + sqlx::types::Json(&result), + None, + 0, + None, + false, + Some(3000), + false, + ) + .await?; + + let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option) = sqlx::query_as( + "SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \ + (extras->>'wac_last_segment_ms')::bigint \ + FROM v2_job_completed WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + assert!( + (started_ago_s - 3600.0).abs() < 60.0, + "started_at must be the first start, got {started_ago_s}s ago" + ); + assert!( + (duration_ms - 3_600_000).abs() < 60_000, + "duration_ms must be the wall time since the first start, got {duration_ms}" + ); + assert_eq!(last_segment_ms, Some(3000)); + assert_eq!( + span.duration_ms, duration_ms, + "a flow records the step with the span the row holds" + ); + assert!(span.outlasts_run(job.started_at)); + assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter"); + + Ok(()) +} diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ac9defcd53..46bd6825e8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -490,7 +490,7 @@ async fn list_user_usage( " SELECT usr.email, usage.executions FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index e8e3716d2b..389ae1d112 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1221,7 +1221,7 @@ pub async fn add_completed_job_pre_shaped_failure, -) -> Result<(), Error> { +) -> Result { record_failure_metrics(completed_job, worker_name).await; tracing::error!( @@ -1229,7 +1229,7 @@ pub async fn add_completed_job_pre_shaped_failure, ) -> Result { + add_completed_job_error_with_span( + db, + completed_job, + mem_peak, + canceled_by, + e, + worker_name, + flow_is_done, + duration, + ) + .await + .map(|(wrapped, _)| wrapped) +} + +/// `add_completed_job_error`, also handing back what the completed row holds. +pub async fn add_completed_job_error_with_span( + db: &Pool, + completed_job: &MiniCompletedJob, + mem_peak: i32, + canceled_by: Option, + e: serde_json::Value, + worker_name: &str, + flow_is_done: bool, + duration: Option, +) -> Result<(WrappedError, CompletedSpan), Error> { record_failure_metrics(completed_job, worker_name).await; let result = WrappedError { error: e }; @@ -1266,7 +1291,7 @@ pub async fn add_completed_job_error( completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); - let _ = add_completed_job( + let (_, span) = add_completed_job( db, &completed_job, false, @@ -1281,7 +1306,7 @@ pub async fn add_completed_job_error( ) .warn_after_seconds(10) .await?; - Ok(result) + Ok((result, span)) } lazy_static::lazy_static! { @@ -1320,7 +1345,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, CompletedSpan), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1354,7 +1379,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { + let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1395,7 +1420,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, span)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1436,7 +1461,7 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, completed_job, duration); + apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( @@ -1453,7 +1478,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, span)) } async fn commit_completed_job( @@ -1471,7 +1496,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, bool)> { +) -> windmill_common::error::Result<(Option, CompletedSpan, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1538,11 +1563,11 @@ async fn commit_completed_job( && monitor_parent.is_none() && !is_scheduled { - let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else { + let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else { return Err(not_in_queue_error(db, job_id).await); }; - log_completed_job(completed_job, duration, success); - return Ok((None, duration, false, false)); + log_completed_job(completed_job, completed.duration_ms, success); + return Ok((None, completed, false, false)); } let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1550,7 +1575,8 @@ async fn commit_completed_job( // The parent's rows are locked ahead of the child's own queue row (see // `record_child_completion` for the order this must keep), so the duration it stamps is read // before the completion: the one the completed row will hold. `now()` is fixed for the - // transaction, and a completed row already there keeps its own duration. + // transaction, and a completed row already there keeps its own duration. For a child that + // itself parked it is the last run's, matching the last start its timeline entry carries. let mut wac_parent_ready = false; if let Some(parent_job) = wac_parent { let Some(duration) = sqlx::query_scalar!( @@ -1577,7 +1603,7 @@ async fn commit_completed_job( .await?; } - let Some(duration) = completion.execute(&mut *tx).await? else { + let Some(completed) = completion.execute(&mut *tx).await? else { return Err(not_in_queue_error(&mut *tx, job_id).await); }; @@ -1787,16 +1813,35 @@ async fn commit_completed_job( tx.commit().warn_after_seconds(10).await?; - log_completed_job(completed_job, duration, success); + log_completed_job(completed_job, completed.duration_ms, success); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); Ok(( None, - duration, + completed, _skip_downstream_error_handlers, wac_parent_ready, )) } +/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's. +/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered; +/// it differs from `duration_ms` only for a job that parked. +#[derive(Debug, Clone, Copy)] +pub struct CompletedSpan { + pub started_at: Option>, + pub duration_ms: i64, + pub last_run_ms: i64, +} + +impl CompletedSpan { + /// Whether the row spans more than the run that started at `last_started_at`, i.e. the + /// job parked and resumed. A flow records such a step with the row's span, so the step + /// reads like the job does. + pub fn outlasts_run(&self, last_started_at: Option>) -> bool { + matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last) + } +} + /// What a job's completion writes, for `Completion::execute`. struct Completion<'a> { completed_job: &'a MiniCompletedJob, @@ -1813,6 +1858,11 @@ impl Completion<'_> { /// Moves the job from the queue to the completed jobs and refreshes a flow step's parent /// ping, as one statement. Returns `None` when the job was no longer in the queue. /// + /// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's + /// `started_at` and `duration_ms` span the whole workflow from its first start, and the + /// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from + /// completed rows. + /// /// The completion takes its cancellation from the queue row it deletes, not only from /// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still /// being written, so the deleted row is the final word on whether the job was canceled. @@ -1820,7 +1870,7 @@ impl Completion<'_> { /// It locks the queue row before the completed row's key. Any other writer completing a job /// (the monitor's zombie fallback, debounce) must take them in the same order, or the two /// deadlock. - async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { + async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { let Completion { completed_job, success, @@ -1851,16 +1901,20 @@ impl Completion<'_> { .then_some(completed_job.parent_job) .flatten() else { - return sqlx::query_scalar!( + return sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1871,8 +1925,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1888,9 +1945,11 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1910,16 +1969,20 @@ impl Completion<'_> { // A canceled flow is pinged too: it is completed by its next transition like any other // flow, and the zombie flow monitor needs the ping to finish the cancel if that // transition is lost. - sqlx::query_scalar!( + sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1930,8 +1993,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1947,14 +2013,16 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ), parent_ping AS ( UPDATE v2_job_runtime r SET ping = now() FROM v2_job_queue q WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12 AND EXISTS (SELECT 1 FROM completed) ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -2021,7 +2089,7 @@ async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, bool), Error>> { +) -> Option, CompletedSpan, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index bb47d3e4be..1ae588ef11 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output( // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) - UPDATE v2_job_queue q SET running = false, started_at = null + UPDATE v2_job_queue q SET running = false, started_at = null, + extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras + ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at)) + END FROM prev WHERE q.id = $1 RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", job.id, diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0490ed1754..12edd7823e 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1776,7 +1776,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &job, true, @@ -1835,7 +1835,10 @@ pub async fn process_completed_job( true, canceled_by, result, - started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }), + span.started_at.or(started_at).map(|x| FlowJobDuration { + started_at: x, + duration_ms: span.duration_ms, + }), StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -1867,9 +1870,10 @@ pub async fn process_completed_job( // `wm_failure` field) so a real runtime failure whose raw // result happens to contain a `wm_failure` field still goes // through the standard `WrappedError { error: ... }` wrap path. - let downstream_result: Arc> = if is_pre_shaped_wm_failure_result(result.get()) - { - windmill_queue::add_completed_job_pre_shaped_failure( + let (downstream_result, span): (Arc>, _) = if is_pre_shaped_wm_failure_result( + result.get(), + ) { + let span = windmill_queue::add_completed_job_pre_shaped_failure( db, &job, mem_peak.to_owned(), @@ -1880,9 +1884,9 @@ pub async fn process_completed_job( None, ) .await?; - result.clone() + (result.clone(), span) } else { - let wrapped = add_completed_job_error( + let (wrapped, span) = windmill_queue::add_completed_job_error_with_span( db, &job, mem_peak.to_owned(), @@ -1895,7 +1899,10 @@ pub async fn process_completed_job( None, ) .await?; - Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()) + ( + Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()), + span, + ) }; #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { @@ -1927,12 +1934,19 @@ pub async fn process_completed_job( false, canceled_by, downstream_result, - duration.and_then(|d| { - job.started_at.map(|started_at| FlowJobDuration { - started_at: started_at, - duration_ms: d, + if span.outlasts_run(job.started_at) { + span.started_at.map(|started_at| FlowJobDuration { + started_at, + duration_ms: span.duration_ms, }) - }), + } else { + duration.and_then(|d| { + job.started_at.map(|started_at| FlowJobDuration { + started_at: started_at, + duration_ms: d, + }) + }) + }, StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 45d76c74e8..3cc9c1e23a 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -113,6 +113,11 @@ pub enum WacPark { /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. /// +/// The first park keeps that first start in `extras.wac_first_started_at`, so the +/// completed row can still say when the workflow began and how long it took overall +/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline +/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way. +/// /// Call it before any write to the parent's `v2_job_status` row in the same /// transaction: a child's completion locks the queue row and then the status row /// (`record_child_completion`), and taking them the other way round here can @@ -156,7 +161,10 @@ pub async fn suspend_wac_parent( sqlx::query!( "UPDATE v2_job_queue - SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null, + extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras + ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at)) + END WHERE id = $1 AND workspace_id = $2", job_id, w_id, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6d76d0d771..4f67780f18 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; let duration = if success { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, true, @@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms } else { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, false, @@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms }; flow_job_duration = flow_job .started_at From 5d90f96863b64e9bc88ed35429f8068a6a788d1e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 8 Oct 2026 22:17:44 +0200 Subject: [PATCH 03/60] feat: export worker cpu, memory and occupancy metrics (#11629) * feat: export worker cpu, memory and occupancy metrics Co-Authored-By: Claude Opus 5.5 * fix: refresh the exported memory limit with the cgroup readings Co-Authored-By: Claude Opus 5.5 * chore: update ee-repo-ref to e128a9b9bedd0b6a46d355b69db13bf8f650c4e6 This commit updates the EE repository reference after PR #864 was merged in windmill-ee-private. Previous ee-repo-ref: b286a486c69e74ec0742a2aac9d4cb1161eb0c6b New ee-repo-ref: e128a9b9bedd0b6a46d355b69db13bf8f650c4e6 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/tests/otel.rs | 27 +- backend/windmill-common/src/otel_oss.rs | 33 +++ backend/windmill-common/src/worker.rs | 93 +++++++ backend/windmill-worker/src/handle_child.rs | 1 + backend/windmill-worker/src/lib.rs | 1 + .../windmill-worker/src/resource_metrics.rs | 235 ++++++++++++++++++ backend/windmill-worker/src/worker.rs | 1 + backend/windmill-worker/src/worker_utils.rs | 13 +- 9 files changed, 402 insertions(+), 4 deletions(-) create mode 100644 backend/windmill-worker/src/resource_metrics.rs diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 2de431e127..d292c22c2e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ab8476e5feedc98b727f7d83acffa1807b3b9c5b +e128a9b9bedd0b6a46d355b69db13bf8f650c4e6 diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index f1e042037f..30585c08a1 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() { otel_set_worker_uptime("w-uptime", 3600.0); otel_set_health_status_phase("healthy"); otel_set_health_db_unresponsive(true); + otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024)); + otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35); + otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25); + otel_record_job_memory_peak("python3", 100 * 1024 * 1024); // ── Histograms ────────────────────────────────────────────────── @@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 21 metric names are present ────────────────────── + // ── Verify all metric names are present ───────────────────────── let expected = [ "windmill.queue.push_count", @@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() { "windmill.worker.uptime", "windmill.health.status", "windmill.health.db_unresponsive", + "windmill.worker.memory_usage", + "windmill.worker.windmill_memory_usage", + "windmill.worker.memory_limit", + "windmill.worker.cpu_usage", + "windmill.worker.cpu_periods", + "windmill.worker.cpu_throttled_periods", + "windmill.worker.cpu_throttled_time", + "windmill.worker.occupancy_rate", + "windmill.worker.job_memory_peak", ]; for name in expected { assert!( @@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() { .expect("worker.busy data point with worker=worker-test-1 not found"); assert_eq!(dp.1, 1); + let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu") + }) + .expect("worker.memory_limit data point with worker_group=gpu not found"); + assert_eq!(dp.1, 1024); + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); assert_eq!(gauge_i64_values(m)[0].1, 5); let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index bd0a5202df..95e635dd8f 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -132,3 +132,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {} #[cfg(not(feature = "private"))] pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_memory( + _worker: &str, + _worker_group: &str, + _usage: Option, + _windmill_usage: Option, + _limit: Option, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_add_worker_cpu( + _worker: &str, + _worker_group: &str, + _usage_secs: f64, + _periods: u64, + _throttled_periods: u64, + _throttled_secs: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_occupancy_rate( + _worker: &str, + _worker_group: &str, + _window: &'static str, + _rate: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 52fa2da905..017c244670 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1783,6 +1783,73 @@ pub fn get_cpu_period() -> Option { Some(100000) } +/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job +/// peak-memory histograms so the two exports stay comparable. +pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = { + let mut buckets = [0.0; 12]; + let mut i = 0; + while i < 12 { + buckets[i] = (16u64 << (20 + i)) as f64; + i += 1; + } + buckets +}; + +/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0 +/// when the cgroup has no CPU controller, which is also what an unthrottled one reports. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct CpuStat { + pub usage_usec: u64, + pub nr_periods: u64, + pub nr_throttled: u64, + pub throttled_usec: u64, +} + +fn cpu_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let (k, v) = line.split_once(' ')?; + (k == key).then(|| v.trim().parse().ok()).flatten() + }) +} + +fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpu_stat_field(stat, "usage_usec")?, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0), + }) +} + +/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds. +fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpuacct_usage.trim().parse::().ok()? / 1000, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000, + }) +} + +#[cfg(not(windows))] +pub fn get_cpu_stat() -> Option { + const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage"; + if Path::new(V1_USAGE).exists() { + parse_cgroup_v1_cpu_stat( + &std::fs::read_to_string(V1_USAGE).ok()?, + &std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(), + ) + } else { + let cgroup_path = get_cgroupv2_path()?; + parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?) + } +} + +#[cfg(windows)] +pub fn get_cpu_stat() -> Option { + None +} + /// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count /// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds /// the quota in, so a fraction of a CPU makes it report a single-core machine. @@ -2919,6 +2986,32 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn cgroup_cpu_stat_is_normalized_to_microseconds() { + let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\ + nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n"; + let expected = CpuStat { + usage_usec: 2_500_000, + nr_periods: 40, + nr_throttled: 7, + throttled_usec: 350_000, + }; + assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected)); + + let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n"; + assert_eq!( + parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat), + Some(expected) + ); + + // No CPU controller: usage is still reported, throttling reads as zero. + assert_eq!( + parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"), + Some(CpuStat { usage_usec: 12, ..Default::default() }) + ); + assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None); + } + #[test] fn datatable_role_is_read_from_the_leading_comment_block() { let role = |code| SqlAnnotations::datatable_role(code); diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index c25ad82a83..84fd547641 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -1049,6 +1049,7 @@ where let wm_memory_usage = get_windmill_memory_usage(); tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); + crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref()); if job_id != Uuid::nil() { if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await { tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 6059936af6..aa1d9ab209 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -81,6 +81,7 @@ mod prepare_deps; mod python_executor; #[cfg(feature = "python")] mod python_versions; +mod resource_metrics; pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; diff --git a/backend/windmill-worker/src/resource_metrics.rs b/backend/windmill-worker/src/resource_metrics.rs new file mode 100644 index 0000000000..53c7dc5b4d --- /dev/null +++ b/backend/windmill-worker/src/resource_metrics.rs @@ -0,0 +1,235 @@ +use std::sync::{atomic::Ordering, Mutex, OnceLock}; + +use windmill_common::{ + otel_oss::{ + otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory, + otel_set_worker_occupancy_rate, + }, + worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP}, + OTEL_METRICS_ENABLED, +}; + +use crate::common::OccupancyResult; + +#[cfg(feature = "prometheus")] +struct PromMetrics { + memory_usage: prometheus::IntGaugeVec, + windmill_memory_usage: prometheus::IntGaugeVec, + memory_limit: prometheus::IntGaugeVec, + cpu_usage: prometheus::CounterVec, + cpu_periods: prometheus::IntCounterVec, + cpu_throttled_periods: prometheus::IntCounterVec, + cpu_throttled_seconds: prometheus::CounterVec, + occupancy_rate: prometheus::GaugeVec, + job_memory_peak: prometheus::HistogramVec, +} + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + static ref PROM_METRICS: Option = + if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) { + const WORKER: &[&str] = &["name", "worker_group"]; + Some(PromMetrics { + memory_usage: prometheus::register_int_gauge_vec!( + "worker_memory_usage_bytes", + "Memory used by the worker's container, page cache excluded", + WORKER + ) + .unwrap(), + windmill_memory_usage: prometheus::register_int_gauge_vec!( + "worker_windmill_memory_usage_bytes", + "Resident memory of the worker process itself", + WORKER + ) + .unwrap(), + memory_limit: prometheus::register_int_gauge_vec!( + "worker_memory_limit_bytes", + "Memory limit of the worker's container", + WORKER + ) + .unwrap(), + cpu_usage: prometheus::register_counter_vec!( + "worker_cpu_usage_seconds_total", + "CPU time consumed by the worker's container", + WORKER + ) + .unwrap(), + cpu_periods: prometheus::register_int_counter_vec!( + "worker_cpu_periods_total", + "CPU enforcement periods elapsed for the worker's container", + WORKER + ) + .unwrap(), + cpu_throttled_periods: prometheus::register_int_counter_vec!( + "worker_cpu_throttled_periods_total", + "CPU enforcement periods in which the worker's container was throttled", + WORKER + ) + .unwrap(), + cpu_throttled_seconds: prometheus::register_counter_vec!( + "worker_cpu_throttled_seconds_total", + "Time the worker's container spent throttled by its CPU limit", + WORKER + ) + .unwrap(), + occupancy_rate: prometheus::register_gauge_vec!( + "worker_occupancy_rate", + "Share of time the worker spent executing jobs, over the window", + &["name", "worker_group", "window"] + ) + .unwrap(), + job_memory_peak: prometheus::register_histogram_vec!( + "worker_job_memory_peak_bytes", + "Peak memory of a job's main process", + &["tag"], + windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec() + ) + .unwrap(), + }) + } else { + None + }; +} + +fn metrics_enabled() -> bool { + #[cfg(feature = "prometheus")] + if PROM_METRICS.is_some() { + return true; + } + OTEL_METRICS_ENABLED.load(Ordering::Relaxed) +} + +// Memory and CPU are read from the cgroup, which every worker of a process shares +// (native mode runs several). Only the first worker to report exports them, so that +// summing the series over workers does not count one container several times. +static CONTAINER_SERIES_OWNER: OnceLock = OnceLock::new(); +static MEMORY_LIMIT: Mutex>> = Mutex::new(None); + +/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits. +pub(crate) fn set_memory_limit(limit: Option) { + *MEMORY_LIMIT.lock().unwrap() = Some(limit); +} +static LAST_CPU_STAT: Mutex = + Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 }); + +/// Exports the worker's resource readings. Meant for the ping ticks, which have already +/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read. +pub(crate) fn record_worker_resources( + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy: Option<&OccupancyResult>, +) { + if !metrics_enabled() { + return; + } + let worker_group = WORKER_GROUP.as_str(); + + if let Some(occupancy) = occupancy { + let windows = [ + ("total", Some(occupancy.occupancy_rate)), + ("15s", occupancy.occupancy_rate_15s), + ("5m", occupancy.occupancy_rate_5m), + ("30m", occupancy.occupancy_rate_30m), + ]; + for (window, rate) in windows { + let Some(rate) = rate.filter(|r| r.is_finite()) else { + continue; + }; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.occupancy_rate + .with_label_values(&[worker_name, worker_group, window]) + .set(rate as f64); + } + otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64); + } + } + + if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name { + return; + } + + let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory); + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + if let Some(v) = memory_usage { + m.memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = wm_memory_usage { + m.windmill_memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = memory_limit { + m.memory_limit.with_label_values(labels).set(v); + } + } + otel_set_worker_memory( + worker_name, + worker_group, + memory_usage, + wm_memory_usage, + memory_limit, + ); + + let Some(stat) = get_cpu_stat() else { + return; + }; + let delta = { + let mut last = LAST_CPU_STAT.lock().unwrap(); + // A reading below the previous one means the cgroup changed under the process: + // restart from it rather than exporting a negative increase. + let delta = if stat.usage_usec < last.usage_usec { + CpuStat::default() + } else { + CpuStat { + usage_usec: stat.usage_usec - last.usage_usec, + nr_periods: stat.nr_periods.saturating_sub(last.nr_periods), + nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled), + throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec), + } + }; + *last = stat; + delta + }; + let usage_secs = delta.usage_usec as f64 / 1e6; + let throttled_secs = delta.throttled_usec as f64 / 1e6; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + m.cpu_usage.with_label_values(labels).inc_by(usage_secs); + m.cpu_periods + .with_label_values(labels) + .inc_by(delta.nr_periods); + m.cpu_throttled_periods + .with_label_values(labels) + .inc_by(delta.nr_throttled); + m.cpu_throttled_seconds + .with_label_values(labels) + .inc_by(throttled_secs); + } + otel_add_worker_cpu( + worker_name, + worker_group, + usage_secs, + delta.nr_periods, + delta.nr_throttled, + throttled_secs, + ); +} + +/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process +/// to sample or the reading failed, and those are not observations. +pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) { + if mem_peak_kb <= 0 { + return; + } + let bytes = mem_peak_kb as u64 * 1024; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.job_memory_peak + .with_label_values(&[tag]) + .observe(bytes as f64); + } + otel_record_job_memory_peak(tag, bytes); +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d0d5d8b4e2..5b5df51b30 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -5130,6 +5130,7 @@ pub async fn handle_queued_job( // WAC v2 job suspended while waiting for child jobs — don't complete it return Ok(JobOutcome::Completed); } + crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak); process_result( cjob, result.map(|x| Arc::new(x)), diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index cabf004f4d..c634c13b6a 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full( let wm_memory_usage = get_windmill_memory_usage(); let (vcpus, memory) = if read_cgroups { - (get_vcpus(), get_memory()) + let memory = get_memory(); + crate::resource_metrics::set_memory_limit(memory); + (get_vcpus(), memory) } else { (None, None) }; + let occupancy = occupancy_metrics.update_occupancy_metrics(); + crate::resource_metrics::record_worker_resources( + worker_name, + memory_usage, + wm_memory_usage, + Some(&occupancy), + ); let OccupancyResult { occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, - } = occupancy_metrics.update_occupancy_metrics(); + } = occupancy; let ping_start = std::time::Instant::now(); if let Err(e) = (|| { From 987c321ebcd305df79fa945e941abc2b2e1d3e95 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Thu, 8 Oct 2026 16:32:55 -0400 Subject: [PATCH 04/60] fix: uniform decrypt error and clear stored credentials on logout (#11632) * fix: uniform decrypt error and clear stored credentials on logout Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Kv72vdCDggCZEjJSmNCwnX * fix: also clear mcp-oauth-callback on logout, cover the new keys in the logout test Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Kv72vdCDggCZEjJSmNCwnX --------- Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-common/src/variables.rs | 42 +++++++++++++++++++----- frontend/src/lib/logout.test.ts | 16 +++++++++ frontend/src/lib/logout.ts | 16 +++++---- 3 files changed, 60 insertions(+), 14 deletions(-) diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1feb4182ab..b3774cdda4 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -14,7 +14,7 @@ use crate::utils::WarnAfterExt; use crate::worker::Connection; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; -use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; @@ -375,13 +375,15 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { mc.encrypt_str_to_base64(value) } +const DECRYPT_FAILED: &str = + "Could not decrypt value. The value may have been encrypted with a different key."; + +/// Every failure returns the same message so responses carry no signal about why decryption +/// failed (padding, base64, UTF-8); the cause is only logged. pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { - mc.decrypt_base64_to_string(value).map_err(|e| match e { - MagicCryptError::DecryptError(_) => error::Error::internal_err( - "Could not decrypt value. The value may have been encrypted with a different key." - .to_string(), - ), - _ => error::Error::internal_err(e.to_string()), + mc.decrypt_base64_to_string(value).map_err(|e| { + tracing::warn!("decryption failed: {e}"); + error::Error::internal_err(DECRYPT_FAILED.to_string()) }) } @@ -746,7 +748,31 @@ pub async fn get_variable_or_self_as( #[cfg(test)] mod tests { - use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier}; + use super::{ + can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier, + DECRYPT_FAILED, + }; + use crate::error::Error; + use magic_crypt::MagicCryptTrait; + + #[test] + fn decrypt_failures_all_return_the_same_message() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let failures = [ + "not base64 at all!".to_string(), + mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), + other.encrypt_str_to_base64("secret"), + ]; + for value in failures { + match decrypt(&mc, value.clone()) { + Err(Error::InternalErrLoc { message, .. }) => { + assert_eq!(message, DECRYPT_FAILED, "for {value:?}") + } + other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"), + } + } + } #[test] fn is_valid_js_identifier_gates_prologue_injection() { diff --git a/frontend/src/lib/logout.test.ts b/frontend/src/lib/logout.test.ts index 160bce9934..5bdc15bac3 100644 --- a/frontend/src/lib/logout.test.ts +++ b/frontend/src/lib/logout.test.ts @@ -12,9 +12,25 @@ import { clearUser } from './logout' describe('clearUser', () => { beforeEach(() => { sessionStorage.clear() + localStorage.clear() vi.clearAllMocks() }) + it('drops credential-bearing localStorage entries and keeps preferences', async () => { + const credentialKeys = ['oauth-callback', 'mcp-oauth-callback', 'test_dev_token'] + for (const key of credentialKeys) { + localStorage.setItem(key, 'a-token') + } + localStorage.setItem('dark-mode', 'true') + + await clearUser() + + for (const key of credentialKeys) { + expect(localStorage.getItem(key)).toBeNull() + } + expect(localStorage.getItem('dark-mode')).toBe('true') + }) + // Logging out while impersonating must not leave the impersonator's own token behind. it('drops the impersonation token backup', async () => { sessionStorage.setItem('pre_impersonation_token', 'a-token') diff --git a/frontend/src/lib/logout.ts b/frontend/src/lib/logout.ts index 58fce16fe3..b4782badd3 100644 --- a/frontend/src/lib/logout.ts +++ b/frontend/src/lib/logout.ts @@ -6,15 +6,19 @@ import { clearStores } from './storeUtils' // Note: logout and logoutWithRedirect have been moved to logoutKit.ts // as they depend on SvelteKit navigation -// The impersonation backup holds the impersonator's own auth token, so it must go on -// logout too, not only when impersonation is ended from the banner — otherwise a usable -// token stays readable for the lifetime of the tab. -function clearImpersonationFromStorage(): void { +// Every browser-storage entry that can hold a credential must go on logout. The impersonation +// backup is the impersonator's own auth token; `oauth-callback` and `mcp-oauth-callback` carry a +// connected account's token response when the OAuth popup had no opener to post it to; +// `test_dev_token` is the session token of the /test_dev pages. +function clearCredentialsFromStorage(): void { try { sessionStorage.removeItem('pre_impersonation_token') sessionStorage.removeItem('pre_impersonation_email') + localStorage.removeItem('oauth-callback') + localStorage.removeItem('mcp-oauth-callback') + localStorage.removeItem('test_dev_token') } catch (e) { - console.error('error interacting with session storage', e) + console.error('error interacting with browser storage', e) } } @@ -23,7 +27,7 @@ export async function clearUser() { noteSessionEmail(undefined) accountSetup.reset() clearStores() - clearImpersonationFromStorage() + clearCredentialsFromStorage() await UserService.logout() } catch (error) {} } From 6acf1858540e83b65b5218c50c98af692092d22e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 8 Oct 2026 22:35:07 +0200 Subject: [PATCH 05/60] feat: add an api base url instance setting for displayed urls (#11628) * feat: add an api base url instance setting for displayed urls * feat: show the api base url as cli remote, keep mcp url on origin * fix: keep push captures on origin and load the api base url on svix page * fix: reject shell metacharacters in webhook and api base urls * test: drop the trivial api base url resolver test * feat: hide the api base url setting behind a toggle * fix: let the quick setup toggle reveal the api base url field * chore: update ee-repo-ref to 017710bdd0a10de857f0f40800dde238cdb1ad24 This commit updates the EE repository reference after PR #863 was merged in windmill-ee-private. Previous ee-repo-ref: f67cd621bd45840e6389e0469a67378864f269a0 New ee-repo-ref: 017710bdd0a10de857f0f40800dde238cdb1ad24 Automated by sync-ee-ref workflow. --------- Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-api-settings/src/lib.rs | 34 +++++------ backend/windmill-api/openapi.yaml | 5 +- .../windmill-common/src/global_settings.rs | 41 ++++++++++--- .../windmill-common/src/instance_config.rs | 58 +++++++++++-------- frontend/src/lib/apiBaseUrl.svelte.ts | 22 +++++++ frontend/src/lib/components/CliHelpBox.svelte | 3 +- frontend/src/lib/components/JobArgs.svelte | 3 +- .../src/lib/components/WorkerGroup.svelte | 3 +- .../components/git_sync/GitPushPreview.svelte | 3 +- .../components/home/HomeConnectDrawer.svelte | 15 +++-- .../src/lib/components/instanceSettings.ts | 31 ++++++++-- .../instanceSettingsWebhookBaseUrl.test.ts | 16 +++-- .../components/settings/CreateToken.svelte | 2 + .../azure/AzureTriggerEditorInner.svelte | 7 ++- .../gcp/GcpTriggerEditorConfigSection.svelte | 22 +------ .../triggers/gcp/GcpTriggerEditorInner.svelte | 7 ++- .../triggers/http/OpenAPISpecGenerator.svelte | 8 +-- .../src/lib/components/triggers/http/utils.ts | 10 ++-- .../webhook/WebhooksConfigSection.svelte | 10 ++-- frontend/src/lib/instanceUi.ts | 3 +- frontend/src/lib/utils_deployable.ts | 8 +-- frontend/src/lib/utils_workspace_deploy.ts | 2 +- .../svix/create-webhook/+page@(root).svelte | 8 ++- 24 files changed, 205 insertions(+), 118 deletions(-) create mode 100644 frontend/src/lib/apiBaseUrl.svelte.ts diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d292c22c2e..cc3c1a826c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e128a9b9bedd0b6a46d355b69db13bf8f650c4e6 +017710bdd0a10de857f0f40800dde238cdb1ad24 diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index c79b596617..f9f0a993fa 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,10 +58,10 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, - AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXTERNAL_INSTANCE_PG_SETTING, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -1238,26 +1238,22 @@ async fn run_setting_pre_write_hook( } } } - GITHUB_APP_WEBHOOK_BASE_URL_SETTING => { - // A bad value here yields a webhook GitHub can never deliver to, and the + GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => { + // A bad webhook base yields a webhook GitHub can never deliver to, and the // failure only shows up much later as "falling back to polling" on a - // repository — so reject it at the boundary instead. + // repository; a bad API base puts an unreachable URL in front of every user. + // So reject both at the boundary instead. match value { // Clearing (delete row) is handled by the caller; allow it through. serde_json::Value::Null => {} serde_json::Value::String(s) if s.trim().is_empty() => {} serde_json::Value::String(s) => { - windmill_common::global_settings::validate_webhook_base_url(s).map_err( - |e| { - error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}" - )) - }, - )?; + windmill_common::global_settings::validate_webhook_base_url(s) + .map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?; } _ => { return Err(error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string" + "{key} must be a URL string" ))); } } @@ -1394,8 +1390,12 @@ async fn set_instance_config( // Applied and audited one setting at a time: the writes are not transactional, so a // batch that fails midway must not leave the settings it did write unrecorded. - let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } = - settings_diff; + let instance_config::SettingsDiff { + upserts, + deletes, + mut previous_values, + unchanged_count, + } = settings_diff; let steps = upserts .into_iter() .map(|(k, v)| (k, Some(v))) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5e635e39bd..0d21402a78 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1919,7 +1919,7 @@ paths: /settings/instance_ui: get: - summary: get the instance-wide announcement banner and accent color + summary: get the instance-wide announcement banner, accent color and API base url description: | Readable by any authenticated user; every page load reads it. Each value is the raw stored setting, or null when unset. @@ -1928,7 +1928,7 @@ paths: - setting responses: "200": - description: the instance_banner and accent_color settings + description: the instance_banner, accent_color and api_base_url settings content: application/json: schema: @@ -1936,6 +1936,7 @@ paths: properties: instance_banner: {} accent_color: {} + api_base_url: {} /settings_u/ruff_config: get: diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index e1a731f3e6..1cd3609820 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -181,6 +181,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP +/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on. +/// For deployments that serve the API on a separate domain or behind a gateway. +/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]: +/// every user is shown URLs built from it. +pub const API_BASE_URL_SETTING: &str = "api_base_url"; /// Instance-wide announcement rendered above every page of the app (maintenance /// windows, incidents). Readable by any authenticated user, unlike most settings: /// the banner exists to be shown to everyone, so it must never hold anything the @@ -313,12 +319,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { pub struct InstanceUi { pub instance_banner: Option, pub accent_color: Option, + pub api_base_url: Option, } pub async fn get_instance_ui(db: &Pool) -> error::Result { let rows = sqlx::query!( "SELECT name, value FROM global_settings WHERE name = ANY($1)", - &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + &[ + INSTANCE_BANNER_SETTING, + ACCENT_COLOR_SETTING, + API_BASE_URL_SETTING + ] as &[&str] ) .fetch_all(db) .await?; @@ -327,17 +338,22 @@ pub async fn get_instance_ui(db: &Pool) -> error::Result { match row.name.as_str() { INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + API_BASE_URL_SETTING => ui.api_base_url = Some(row.value), _ => {} } } Ok(ui) } -/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. +const SHELL_ACTIVE_CHARS: [char; 16] = [ + '$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!', +]; + +/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value. /// -/// The receiver path is appended to it verbatim, so anything that doesn't -/// concatenate into a URL GitHub can POST to must be rejected at write time -/// rather than silently producing an unreachable hook: a wrong scheme +/// A path is appended to it verbatim, so anything that doesn't concatenate into a +/// URL an external caller can reach must be rejected at write time rather than +/// silently producing an unreachable hook or endpoint: a wrong scheme /// (`httpss://`), a missing host, embedded whitespace, or a query/fragment /// (appending a path after `?`/`#` keeps it inside the query/fragment). /// @@ -353,7 +369,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?; if !url.username().is_empty() || url.password().is_some() { return Err( - "must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(), + "must not embed a username or password: the URL is readable by users who are not instance admins".to_string(), ); } if !matches!(url.scheme(), "http" | "https") { @@ -366,13 +382,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { } if url.query().is_some() || url.fragment().is_some() { return Err( - "must not include a query string or fragment, since the webhook path is appended to it" + "must not include a query string or fragment, since a path is appended to it" .to_string(), ); } if value.chars().any(char::is_whitespace) { return Err("must not contain whitespace".to_string()); } + // The UI pastes the URL into shell commands users copy, where double quotes + // do not stop `$(...)` or backticks from running. + if value.contains(SHELL_ACTIVE_CHARS) { + return Err( + "must not contain shell metacharacters, since it is shown inside commands users copy" + .to_string(), + ); + } // Matches the sibling `base_url` / `hub_base_url` convention. The receiver // builder trims it anyway, so this is about keeping the stored value canonical // rather than about reachability. @@ -1101,6 +1125,9 @@ mod tests { "https://hooks.example.com#", "https://user:password@hooks.example.com", "https://user@hooks.example.com", + "https://hooks.example.com/$(uname)", + "https://hooks.example.com/`uname`", + "https://hooks.example.com/a;b", ]; for v in accept { assert!( diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 700ba71df5..bd7c28628c 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -281,6 +281,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub api_base_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub email_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub hub_base_url: Option, @@ -1375,8 +1377,8 @@ pub fn diff_worker_configs( } /// Declaratively replace the global settings, rejecting a -/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the -/// API would reject. +/// `github_app_webhook_base_url`, `api_base_url` or +/// `http_route_default_allowed_origins` the API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1399,29 +1401,35 @@ pub async fn sync_global_settings_declarative( current: &BTreeMap, desired: &BTreeMap, ) -> anyhow::Result<()> { - let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING; - // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a - // bool/number/object through as if the key were absent, and the diff below would - // then persist it — where the HTTP path answers "must be a URL string". - match desired.get(webhook_key) { - None | Some(serde_json::Value::Null) => {} - Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} - Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s) - .map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?, - // Names the JSON kind rather than printing it: this is the last message on - // this path that could report submitted content, and an object or array could - // carry a secret into `sync-config` output and operator logs. - Some(other) => { - let kind = match other { - serde_json::Value::Bool(_) => "a boolean", - serde_json::Value::Number(_) => "a number", - serde_json::Value::Array(_) => "an array", - serde_json::Value::Object(_) => "an object", - _ => "a non-string value", - }; - return Err(anyhow::anyhow!( - "{webhook_key} must be a URL string, got {kind}" - )); + for url_key in [ + crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, + crate::global_settings::API_BASE_URL_SETTING, + ] { + // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a + // bool/number/object through as if the key were absent, and the diff below would + // then persist it — where the HTTP path answers "must be a URL string". + match desired.get(url_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(serde_json::Value::String(s)) => { + crate::global_settings::validate_webhook_base_url(s) + .map_err(|e| anyhow::anyhow!("{url_key}: {e}"))? + } + // Names the JSON kind rather than printing it: this is the last message on + // this path that could report submitted content, and an object or array could + // carry a secret into `sync-config` output and operator logs. + Some(other) => { + let kind = match other { + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + _ => "a non-string value", + }; + return Err(anyhow::anyhow!( + "{url_key} must be a URL string, got {kind}" + )); + } } } diff --git a/frontend/src/lib/apiBaseUrl.svelte.ts b/frontend/src/lib/apiBaseUrl.svelte.ts new file mode 100644 index 0000000000..f87a33f0d8 --- /dev/null +++ b/frontend/src/lib/apiBaseUrl.svelte.ts @@ -0,0 +1,22 @@ +import { fromStore } from 'svelte/store' +import { base } from '$lib/base' +import { instanceUi } from '$lib/instanceUi' + +const ui = fromStore(instanceUi) + +/** + * Base of every URL shown for an external client to call (webhooks, HTTP routes, push + * trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the + * origin being browsed. Append `/api/...` to it. + * + * Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links + * to pages): those must stay on the browsing origin, where the session cookie lives. + * + * Reactive: the setting lands after page load, so read it inside a `$derived` or the + * template rather than caching the result. + */ +export function apiBaseUrl(): string { + const setting = ui.current?.api_base_url + const configured = typeof setting === 'string' ? setting.trim() : '' + return configured || `${window.location.origin}${base}` +} diff --git a/frontend/src/lib/components/CliHelpBox.svelte b/frontend/src/lib/components/CliHelpBox.svelte index 0bee33825c..e91b039328 100644 --- a/frontend/src/lib/components/CliHelpBox.svelte +++ b/frontend/src/lib/components/CliHelpBox.svelte @@ -2,10 +2,11 @@ import ClipboardPanel from './details/ClipboardPanel.svelte' import Section from './Section.svelte' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' const operatingWorkspace = useOperatingWorkspace() - let url = $derived(`${window.location.protocol}//${window.location.hostname}/`) + let url = $derived(`${apiBaseUrl()}/`)
diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index 64ca912608..e0712d66c7 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -15,6 +15,7 @@ import { isWindmillTooBigObject } from './job_args' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' interface Props { id?: string | undefined @@ -246,7 +247,7 @@ ${Object.entries(displayArgs) If using the wmill client in your code, set the following env variables: 
BASE_URL="{window.location.origin}"
+				>BASE_URL="{apiBaseUrl()}"
 WM_TOKEN="{''}"
diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 3b399a5a1e..5e8957c31a 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -55,6 +55,7 @@ import TagList from './TagList.svelte' import DedicatedWorkersSelector from './DedicatedWorkersSelector.svelte' import { computeHashedTag } from './dedicated_worker' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import ObjectStoreConfigSettings, { type ObjectStoreConfig } from './ObjectStoreConfigSettings.svelte' @@ -1238,7 +1239,7 @@

Requires a superadmin token.

curl -X POST '{window.location.origin}/api/configs/update/worker__{name}' \
+					>curl -X POST '{apiBaseUrl()}/api/configs/update/worker__{name}' \
   -H 'Content-Type: application/json' \
   -H 'Authorization: Bearer <token>' \
   -d '{JSON.stringify(cleanWorkerGroupConfig(nconfig ?? {}), null, 2)}'
') const cliCommands = $derived(`npm install -g windmill-cli wmill workspace add ${workspaceId} ${workspaceId} ${origin} @@ -55,7 +56,8 @@ wmill sync pull`)

Local setup

Run this in your local repo to bind the current workspace, create - wmill.yaml, and pull the latest files.

@@ -80,15 +82,18 @@ wmill sync pull`) />

- wmill workspace add will handle authentication, - wmill init bootstraps the local config, and - wmill sync pull fetches the workspace content. diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 9701b74e30..75dada92ab 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -175,8 +175,7 @@ export function isValidWebhookBaseUrl(value: unknown): boolean { return ( (url.protocol === 'http:' || url.protocol === 'https:') && url.host !== '' && - // Userinfo would end up in the per-repository receiver stored in workspace - // settings, which workspace admins can read. + // Userinfo would end up in URLs that users who are not instance admins can read. url.username === '' && url.password === '' && // Tested on the raw string, not `url.search`/`url.hash`: those are `''` for a @@ -187,6 +186,8 @@ export function isValidWebhookBaseUrl(value: unknown): boolean { // `new URL` silently percent-encodes a space in the path, where the server // rejects it outright. !/\s/.test(trimmed) && + // Shell-active characters: the URL is pasted into commands users copy. + !/[$`'"\\;&|<>(){}*!]/.test(trimmed) && !trimmed.endsWith('/') ) } @@ -207,7 +208,29 @@ export const settings: Record = { (value?.startsWith('http') && value.includes('://') && !value?.endsWith('/') && - !value?.endsWith(' ')) + !value?.endsWith(' ')), + advancedToggle: { + label: 'The API is served on a different url than the UI', + onChange(values) { + values['api_base_url'] = values['api_base_url'] == null ? '' : null + return values + }, + checked: (values) => values['api_base_url'] != null + } + }, + { + label: 'API base url', + description: + 'Base url shown to users for the endpoints external clients call (webhooks, HTTP routes, push trigger endpoints, the CLI remote), without trailing slash. Leave empty to show the url the UI is browsed on. Set it when the API is served on a separate domain or behind an API gateway. If that url does not also serve the UI, the CLI must authenticate with a token rather than the browser login.', + key: 'api_base_url', + fieldType: 'text', + placeholder: 'https://api.windmill.company.com', + storage: 'setting', + ee_only: '', + hiddenIfNull: true, + error: + 'API base url must be an http:// or https:// url with a host, no embedded username or password, no query string or fragment, no shell metacharacters, and no trailing slash', + isValid: isValidWebhookBaseUrl }, { label: 'Email domain', @@ -1223,7 +1246,7 @@ export const settings: Record = { storage: 'setting', ee_only: '', error: - 'Webhook base url must be an http:// or https:// url with a host, no embedded username or password, no query string or fragment, and no trailing slash', + 'Webhook base url must be an http:// or https:// url with a host, no embedded username or password, no query string or fragment, no shell metacharacters, and no trailing slash', isValid: isValidWebhookBaseUrl } ], diff --git a/frontend/src/lib/components/instanceSettingsWebhookBaseUrl.test.ts b/frontend/src/lib/components/instanceSettingsWebhookBaseUrl.test.ts index c8545b073e..175a251315 100644 --- a/frontend/src/lib/components/instanceSettingsWebhookBaseUrl.test.ts +++ b/frontend/src/lib/components/instanceSettingsWebhookBaseUrl.test.ts @@ -29,7 +29,10 @@ const REJECTED = [ 'https://hooks.example.com?', 'https://hooks.example.com#', 'https://user:password@hooks.example.com', - 'https://user@hooks.example.com' + 'https://user@hooks.example.com', + 'https://hooks.example.com/$(uname)', + 'https://hooks.example.com/`uname`', + 'https://hooks.example.com/a;b' ] describe('isValidWebhookBaseUrl', () => { @@ -41,10 +44,13 @@ describe('isValidWebhookBaseUrl', () => { expect(isValidWebhookBaseUrl(value)).toBe(false) }) - it.each([true, 42, {}, [], null] as unknown[])('rejects the non-string %j without throwing', (value) => { - // `Setting.isValid` is typed `any` and YAML mode can supply any JSON shape. - expect(isValidWebhookBaseUrl(value as never)).toBe(value === null) - }) + it.each([true, 42, {}, [], null] as unknown[])( + 'rejects the non-string %j without throwing', + (value) => { + // `Setting.isValid` is typed `any` and YAML mode can supply any JSON shape. + expect(isValidWebhookBaseUrl(value as never)).toBe(value === null) + } + ) it('treats unset and blank as valid, since the setting is optional', () => { expect(isValidWebhookBaseUrl(undefined)).toBe(true) diff --git a/frontend/src/lib/components/settings/CreateToken.svelte b/frontend/src/lib/components/settings/CreateToken.svelte index 8ee5391443..56952eb584 100644 --- a/frontend/src/lib/components/settings/CreateToken.svelte +++ b/frontend/src/lib/components/settings/CreateToken.svelte @@ -245,6 +245,8 @@ // Undefined wherever the workspace is: `/api/mcp/w/undefined/mcp` reads like a real URL // and is copyable, so the OAuth panel withholds it rather than showing a broken one. The // token branch guards the same case by disabling its generate button. + // On the browsing origin, not the API base url: the OAuth metadata the server + // publishes for this URL is built from `base_url`, and a client rejects a mismatch. const mcpUrl = $derived( isAllWorkspaces ? `${window.location.origin}/api/mcp/gateway` diff --git a/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte index 4394a5878f..ed0049168e 100644 --- a/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/azure/AzureTriggerEditorInner.svelte @@ -35,6 +35,7 @@ import LocalDraftBanner from '$lib/components/LocalDraftBanner.svelte' import TriggerSuspendedJobsAlert from '../TriggerSuspendedJobsAlert.svelte' import TriggerSuspendedJobsModal from '../TriggerSuspendedJobsModal.svelte' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import { base } from '$lib/base' import Tabs from '$lib/components/common/tabs/Tabs.svelte' import Tab from '$lib/components/common/tabs/Tab.svelte' @@ -83,7 +84,7 @@ let permissionedAs = $state(undefined) let selectedPermissionedAs = $state(undefined) let preservePermissionedAs = $state(false) - let base_endpoint = $derived(`${window.location.origin}${base}`) + let base_endpoint = $derived(apiBaseUrl()) let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') @@ -320,7 +321,9 @@ topic_name, subscription_name, event_type_filters, - base_endpoint, + // The capture URL shown to the user stays on the browsing origin, so the + // endpoint registered for it must too. + base_endpoint: `${window.location.origin}${base}`, path } } diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte index d7868a650a..a33573a1d1 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorConfigSection.svelte @@ -13,7 +13,7 @@ } from '$lib/gen' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' - import { base } from '$lib/base' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import Toggle from '$lib/components/Toggle.svelte' import { Button, Url } from '$lib/components/common' @@ -28,23 +28,17 @@ useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' - // Declared before `DEFAULT_PUSH_CONFIG` / the `base_endpoint` prop default, - // which call `getBaseUrl()` (a `wsId` reader) during component init. const operatingWorkspace = useOperatingWorkspace() const operatingUser = useOperatingUser() const actingUser = $derived(operatingUser.current) const wsId = $derived($operatingWorkspace) + const base_endpoint = $derived(`${apiBaseUrl()}/api/gcp/w/${wsId!}`) let topic_items: string[] = $state([]) let subscription_items: string[] = $state([]) let loadingTopic = $state(false) let loadingSubscription = $state(false) - const DEFAULT_PUSH_CONFIG: PushConfig = { - audience: getBaseUrl(), - authenticate: false - } - async function loadAllPubSubTopicsFromProject() { // Listing is admin-only under application default credentials, and a non-admin viewing an // inherited trigger cannot change the topic anyway, so asking would only raise a 403 toast. @@ -117,7 +111,6 @@ delivery_type?: DeliveryType | undefined delivery_config: PushConfig | undefined subscription_mode?: SubscriptionMode - base_endpoint?: string path?: string showTestingBadge?: boolean cloud_subscription_id?: string @@ -139,7 +132,6 @@ delivery_type = $bindable('pull'), delivery_config = $bindable(), subscription_mode = $bindable('create_update'), - base_endpoint = $bindable(getBaseUrl()), auto_acknowledge_msg = $bindable(true), ack_deadline = $bindable(), path = '', @@ -199,17 +191,9 @@ if (!delivery_type) { delivery_type = 'pull' } else if (delivery_type === 'push' && !delivery_config) { - delivery_config = DEFAULT_PUSH_CONFIG + delivery_config = { audience: base_endpoint, authenticate: false } } }) - function getBaseUrl() { - return `${window.location.origin}${base}/api/gcp/w/${wsId!}` - } - - $effect(() => { - !base_endpoint && (base_endpoint = getBaseUrl()) - }) - $effect(() => { if (emptyStringTrimmed(subscription_id) && !emptyStringTrimmed(path)) { subscription_id = `windmill-${wsId!}-${path.replaceAll('/', '_')}` diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index cfee255553..49ba66448c 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -36,6 +36,7 @@ import LocalDraftBanner from '$lib/components/LocalDraftBanner.svelte' import TriggerSuspendedJobsAlert from '../TriggerSuspendedJobsAlert.svelte' import TriggerSuspendedJobsModal from '../TriggerSuspendedJobsModal.svelte' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import { base } from '$lib/base' import Tabs from '$lib/components/common/tabs/Tabs.svelte' import Tab from '$lib/components/common/tabs/Tab.svelte' @@ -88,7 +89,7 @@ let permissionedAs = $state(undefined) let selectedPermissionedAs = $state(undefined) let preservePermissionedAs = $state(false) - let base_endpoint = $derived(`${window.location.origin}${base}`) + let base_endpoint = $derived(apiBaseUrl()) let auto_acknowledge_msg = $state(true) let ack_deadline: number | undefined = $state() let optionTabSelected: 'settings' | 'error_handler' | 'retries' = $state('error_handler') @@ -376,7 +377,9 @@ subscription_id, delivery_type, delivery_config, - base_endpoint, + // The capture URL shown to the user stays on the browsing origin, so the + // endpoint registered for it must too. + base_endpoint: `${window.location.origin}${base}`, auto_acknowledge_msg, ack_deadline, topic_id, diff --git a/frontend/src/lib/components/triggers/http/OpenAPISpecGenerator.svelte b/frontend/src/lib/components/triggers/http/OpenAPISpecGenerator.svelte index b0cc491393..024086552c 100644 --- a/frontend/src/lib/components/triggers/http/OpenAPISpecGenerator.svelte +++ b/frontend/src/lib/components/triggers/http/OpenAPISpecGenerator.svelte @@ -15,7 +15,7 @@ import SimpleEditor from '$lib/components/SimpleEditor.svelte' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' - import { base } from '$lib/base' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import Label from '$lib/components/Label.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import { ClipboardCopy, Download, Trash } from 'lucide-svelte' @@ -136,7 +136,7 @@ obj = { openapi_spec_format: lang, info, - url: `${window.location.origin}${base}`, + url: apiBaseUrl(), http_route_filters: getHttpRouteFilters(), webhook_filters: getWebhookFilters() } @@ -155,7 +155,7 @@ requestBody: { openapi_spec_format, info, - url: `${window.location.origin}${base}`, + url: apiBaseUrl(), http_route_filters: getHttpRouteFilters(), webhook_filters: getWebhookFilters() } @@ -225,7 +225,7 @@ {/snippet} MAX_ALLOWED_ORIGIN_LEN) return `'${origin.slice(0, 40)}…' is longer than any origin a browser sends` - if (origin.includes(',')) - return `'${origin}' must not contain a comma, which separates entries` + if (origin.includes(',')) return `'${origin}' must not contain a comma, which separates entries` if (origin.toLowerCase() === 'null') return `'null' is what a sandboxed iframe sends, so it would allow any page that can open one` if (!/^[\x21-\x7e]+$/.test(origin)) @@ -85,8 +84,7 @@ export function allowedOriginWarning(origin: string): string | undefined { if (rest.startsWith(':')) return `'${origin}' has no host` // An unclosed bracket would otherwise leave `portStart` at zero, which reads // as "no port" and lets the entry through unremarked. - if (rest.startsWith('[') && !rest.includes(']')) - return `'${origin}' has an unclosed IPv6 host` + if (rest.startsWith('[') && !rest.includes(']')) return `'${origin}' has an unclosed IPv6 host` const portStart = rest.startsWith('[') ? rest.indexOf(']') + 1 : rest.indexOf(':') // A trailing colon is a port, an empty one — distinct from having none. const port = portStart > 0 && rest[portStart] === ':' ? rest.slice(portStart + 1) : undefined @@ -160,7 +158,7 @@ export function getHttpRoute( workspaced_route: boolean, workspace_id: string ) { - return `${location.origin}${base}/api/${route_prefix}/${ + return `${apiBaseUrl()}/api/${route_prefix}/${ isCloudHosted() || workspaced_route ? workspace_id + '/' : '' }${route_path ?? ''}` } diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte index 9033aab3cf..218430316a 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte @@ -16,7 +16,7 @@ import ClipboardPanel from '../../details/ClipboardPanel.svelte' import { copyToClipboard, isObject, readFieldsRecursively } from '$lib/utils' // import { page } from '$app/state' - import { base } from '$lib/base' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import TriggerTokens from '../TriggerTokens.svelte' import { userStore } from '$lib/stores' import UserSettings from '../../UserSettings.svelte' @@ -46,7 +46,7 @@ const operatingWorkspace = useOperatingWorkspace() const wsId = $derived($operatingWorkspace) - const WEBHOOK_BASE_URL = $derived(`${location.origin}${base}/api/w/${wsId}/jobs`) + const WEBHOOK_BASE_URL = $derived(`${apiBaseUrl()}/api/w/${wsId}/jobs`) let baseWebhookUrl = $derived.by(() => { let webhookUrlPath: string @@ -223,9 +223,7 @@ async function triggerJob() { function waitForJobCompletion(UUID) { return new Promise(async (resolve, reject) => { try { - const endpoint = \`${ - location.origin - }/api/w/${wsId}/jobs_u/completed/get_result_maybe/\${UUID}\`; + const endpoint = \`${apiBaseUrl()}/api/w/${wsId}/jobs_u/completed/get_result_maybe/\${UUID}\`; const checkResponse = await fetch(endpoint, { method: 'GET', headers: ${JSON.stringify(headers(), null, 2).replaceAll('\n', '\n\t\t\t\t')} @@ -268,7 +266,7 @@ ${ ? 'echo -E $RESULT | jq' : requestType === 'async' ? ` -URL="${location.origin}/api/w/${wsId}/jobs_u/completed/get_result_maybe/$UUID" +URL="${apiBaseUrl()}/api/w/${wsId}/jobs_u/completed/get_result_maybe/$UUID" while true; do curl -s -H "Authorization: Bearer $TOKEN" $URL -o res.json COMPLETED=$(cat res.json | jq .completed) diff --git a/frontend/src/lib/instanceUi.ts b/frontend/src/lib/instanceUi.ts index b9b87d139c..3a7bb1b99c 100644 --- a/frontend/src/lib/instanceUi.ts +++ b/frontend/src/lib/instanceUi.ts @@ -2,7 +2,8 @@ import { writable } from 'svelte/store' import type { GetInstanceUiResponse } from '$lib/gen' /** - * The last `/settings/instance_ui` response: the announcement banner and accent color. + * The last `/settings/instance_ui` response: the announcement banner, accent color + * and API base url. * `undefined` until one lands, and left as is when a fetch fails, so a transient error * never retracts an announcement still in force. Written only by `InstanceUiSync`. */ diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 63ea88adcb..e3f7fa83ed 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -16,7 +16,7 @@ import { type WorkspaceDeployUISettings } from './gen' import type { TriggerKind } from './components/triggers' -import { base } from './base' +import { apiBaseUrl } from './apiBaseUrl.svelte' type DeployUIType = 'script' | 'flow' | 'app' | 'resource' | 'variable' | 'secret' | 'trigger' @@ -250,9 +250,7 @@ export async function getTriggersDeployData( ...gcpTrigger, delivery_config: gcpTrigger.delivery_config ?? undefined, base_endpoint: - gcpTrigger.delivery_type === 'push' - ? (targetBaseUrl ?? `${window.location.origin}${base}`) - : undefined, + gcpTrigger.delivery_type === 'push' ? (targetBaseUrl ?? apiBaseUrl()) : undefined, permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs } @@ -320,7 +318,7 @@ export async function getTriggersDeployData( base_endpoint: azureTrigger.azure_mode === 'namespace_pull' ? undefined - : (targetBaseUrl ?? `${window.location.origin}${base}`), + : (targetBaseUrl ?? apiBaseUrl()), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 280d0daa00..fa50805e3a 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -314,7 +314,7 @@ function makeProvider( existsTriggerByKind: (kind, p) => triggerServiceFor(kind).exists(p), getTriggerForDeploy: async (kind, p) => { // Reuses the existing per-kind transform map (e.g. GCP wipes - // subscription_id and computes base_endpoint from window.location). + // subscription_id and computes base_endpoint for this instance). // Operational-state strip is applied by the shared `deployItem` // after this returns. const { data } = await getTriggersDeployData( diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index d4ac558cb8..212696782b 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -4,6 +4,8 @@ import ScriptPicker from '$lib/components/ScriptPicker.svelte' import { Button } from '$lib/components/common' import { page } from '$app/stores' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' + import InstanceUiSync from '$lib/components/InstanceUiSync.svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import UserSettings from '$lib/components/UserSettings.svelte' @@ -26,11 +28,11 @@ let webhook: string switch (kind) { case 'script': - webhook = `${$page.url.origin}/api/w/${$workspaceStore}/jobs/run/p/${path}` + webhook = `${apiBaseUrl()}/api/w/${$workspaceStore}/jobs/run/p/${path}` redirectSuccess(webhook, token) break case 'flow': - webhook = `${$page.url.origin}/api/w/${$workspaceStore}/jobs/run/f/${path}` + webhook = `${apiBaseUrl()}/api/w/${$workspaceStore}/jobs/run/f/${path}` redirectSuccess(webhook, token) break default: @@ -72,6 +74,8 @@ }) + + { From 0a95ea47facaede2545cc4741d4a70c1c847e11a Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Thu, 8 Oct 2026 23:23:31 +0200 Subject: [PATCH 06/60] fix: keep raw app deploy panel settings shared with the editor (#11616) * fix: keep raw app deploy panel settings shared with the editor Co-Authored-By: Claude Opus 5.5 * fix: take the first deploy's policy copy after bundling Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../raw_apps/RawAppEditorHeader.svelte | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte index c8e9106943..bea1d1ea33 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditorHeader.svelte @@ -207,7 +207,7 @@ let { summary = $bindable(), - policy = $bindable(), + policy, diffDrawer = undefined, savedApp = $bindable(undefined), version = $bindable(undefined), @@ -352,8 +352,12 @@ saveDrawerOpen = false } - async function computeTriggerables() { - policy = await updateRawAppPolicy(runnables, policy) + // Returns a copy: `policy` is the editor's own object, which the Deploy panel's + // instant toggles mutate in place. Reassigning it would detach the panel from the + // editor, so later toggles would never reach the preview. After a successful save, + // copy only `triggerables_v2` back so the editor's value matches what was deployed. + function policyWithTriggerables() { + return updateRawAppPolicy(runnables, policy) } async function createApp(path: string) { @@ -364,9 +368,9 @@ if (!policy.execution_mode) { policy.execution_mode = 'publisher' } - await computeTriggerables() try { const { js, css } = await getBundle() + const deployedPolicy = await policyWithTriggerables() await AppService.createAppRaw({ workspace: opWorkspace!, formData: { @@ -374,7 +378,7 @@ value: app, path, summary: summary, - policy, + policy: deployedPolicy, deployment_message: deploymentMsg, custom_path: customPath, preserve_on_behalf_of: preserveOnBehalfOf || undefined, @@ -384,6 +388,7 @@ css } }) + policy.triggerables_v2 = deployedPolicy.triggerables_v2 // New path now exists server-side — drop the autocomplete cache so // it shows up immediately instead of after the 60s TTL. invalidateWorkspacePaths(opWorkspace!) @@ -391,7 +396,7 @@ summary: summary, value: structuredClone(stateSnapshot(app)), path: path, - policy: policy, + policy: deployedPolicy, custom_path: customPath, labels: $state.snapshot(labels) } @@ -589,10 +594,10 @@ return } const { js, css } = await getBundle() - await computeTriggerables() if (!policy.execution_mode) { policy.execution_mode = 'publisher' } + const deployedPolicy = await policyWithTriggerables() const deployed = await AppService.updateAppRaw({ workspace: opWorkspace!, path: appPath!, @@ -600,7 +605,7 @@ app: { value: app!, summary: summary, - policy, + policy: deployedPolicy, path: npath, deployment_message: deploymentMsg, preserve_on_behalf_of: preserveOnBehalfOf || undefined, @@ -614,12 +619,13 @@ css } }) + policy.triggerables_v2 = deployedPolicy.triggerables_v2 invalidateWorkspacePaths(opWorkspace!) savedApp = { summary: summary, value: structuredClone(stateSnapshot(app)), path: npath, - policy, + policy: deployedPolicy, custom_path: customPath, labels: $state.snapshot(labels) } @@ -655,12 +661,13 @@ } async function setPublishState(message?: string) { - await computeTriggerables() + const nextPolicy = await policyWithTriggerables() await AppService.updateApp({ workspace: opWorkspace!, path: appPath, - requestBody: { policy } + requestBody: { policy: nextPolicy } }) + policy.triggerables_v2 = nextPolicy.triggerables_v2 if (message) { sendUserToast(message) } else if (policy.execution_mode == 'anonymous') { From c52ba2fd30e19986522420ff02fae224a075b4f6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 8 Oct 2026 23:24:00 +0200 Subject: [PATCH 07/60] feat: agent nits: multi-select, memory-off notice, agent step label (#11622) * feat: agent nits: multi-select, memory-off notice, agent step label Co-Authored-By: Claude Opus 5.5 * fix: say saved, not renamed, on a new agent's first deploy Co-Authored-By: Claude Opus 5.5 * fix: run the agent page form on cmd+enter, select agents by keyboard Co-Authored-By: Claude Opus 5.5 * test: pin the agent step label Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../components/FlowStatusViewerInner.svelte | 6 +++- .../src/lib/components/FlowTimeline.svelte | 4 +-- .../components/common/table/AgentRow.svelte | 14 ++++++--- .../lib/components/flows/agentDraft.svelte.ts | 7 ++++- .../flows/content/AgentEditorHost.svelte | 5 ++-- .../conversations/agentEditorChat.test.ts | 10 ++++++- .../flows/conversations/agentEditorChat.ts | 9 ++++++ .../map/FlowModuleSchemaItemViewer.svelte | 4 ++- frontend/src/lib/components/home/Item.svelte | 6 ++-- .../src/lib/components/home/ItemsList.svelte | 3 +- .../lib/components/home/bulkActions.test.ts | 9 ++++++ .../src/lib/components/home/bulkActions.ts | 20 +++++++++---- .../components/home/homeSelection.svelte.ts | 2 +- .../agents/get/[...path]/+page.svelte | 29 +++++++++++++++++++ .../(root)/(logged)/run/[...run]/+page.svelte | 4 ++- 15 files changed, 107 insertions(+), 25 deletions(-) diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 1d56c20b38..638956add2 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -69,6 +69,7 @@ retainLinkedToolsScope } from './flows/linkedAgentToolsStore.svelte' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { AGENT_STEP_ID, agentStepLabel } from './flows/conversations/agentEditorChat' const operatingWorkspace = useOperatingWorkspace() @@ -1238,6 +1239,8 @@ id: string type: FlowModuleValue['type'] suspend?: boolean + /** Shown in place of the id. */ + label?: string } function allModulesForTimeline( @@ -1250,7 +1253,8 @@ ({ id: x.id, type: x.value.type, - suspend: x.suspend != undefined + suspend: x.suspend != undefined, + label: x.id === AGENT_STEP_ID ? agentStepLabel(job?.script_path) : undefined }) as FlowModuleForTimeline, { skipToolNodes: true diff --git a/frontend/src/lib/components/FlowTimeline.svelte b/frontend/src/lib/components/FlowTimeline.svelte index 4ed13c1df1..640c8728ce 100644 --- a/frontend/src/lib/components/FlowTimeline.svelte +++ b/frontend/src/lib/components/FlowTimeline.svelte @@ -179,12 +179,12 @@ />

{/if} - {#each flowModules as { id: k, type: typ, suspend: isSuspend } (k)} + {#each flowModules as { id: k, type: typ, suspend: isSuspend, label } (k)} {@const subItems = items?.[k]?.filter((x) => x.created_at && x.started_at)}
{k.startsWith('subflow:') ? k.substring(8) : k} + >{label ?? (k.startsWith('subflow:') ? k.substring(8) : k)} {#if localModuleStates[k]?.selectedForloop && (typ == 'forloopflow' || typ == 'whileloopflow')}
{/if} {/if} {:else} -
+
{#snippet header()}{/snippet} {#snippet footer()}{/snippet} @@ -1154,199 +1268,249 @@ {@const load_more_prefix = is_load_more ? item_key.slice(0, -1) : ''} {@const file_info = allFilesByKey[item_key]} -
+
{#if is_load_more} {@const loadMoreNesting = nestingLevelOf(load_more_prefix + 'x') - 2 * rootPathNestingLevel} {@const loadingMore = folderState[load_more_prefix]?.loading === true} - - -
!loadingMore && loadMore(load_more_prefix)} - class={twMerge( - 'flex flex-row h-full text-xs items-center justify-start text-secondary', - loadingMore ? 'cursor-default' : 'cursor-pointer' - )} + disabled={loadingMore} + class="w-full text-xs font-normal flex gap-2 items-center h-8 pr-1 text-secondary hover:bg-surface-hover cursor-pointer disabled:cursor-default disabled:hover:bg-transparent" + style={treeRowIndent(loadMoreNesting)} > -
- -
- {#if loadingMore} - - {:else} - - {/if} -
-
Load more
-
-
+ + {#if loadingMore} + + {:else} + + {/if} + Load more + {:else if file_info} {@const nestingLevel = file_info.nestingLevel - 2 * rootPathNestingLevel} - - -
selectItem(index)} + title={file_info.full_key.slice(rootPath.length)} class={twMerge( - 'flex flex-row h-full font-semibold text-xs items-center justify-start', - selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key - ? 'bg-surface-hover' - : '' + 'w-full text-xs font-normal text-primary flex gap-2 items-center h-8 pr-1 cursor-pointer', + isSelected ? 'bg-surface-secondary' : 'hover:bg-surface-hover' )} + style={treeRowIndent(nestingLevel)} > -
- {#if file_info.type === 'folder'} - {#if folderState[file_info.full_key]?.loading} - - {:else if file_info.collapsed}{:else}{/if} -
- - {#if file_info.display_name === ''} - (empty name) - {:else}{file_info.display_name}{/if} - {#if !lazyMode} - ({file_info.count}{count % 1000 === 0 && - lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name - ? '+' - : ''} item{file_info.count === 1 ? '' : 's'}) - {/if} -
+ {#if isFolder} + {#if folderState[file_info.full_key]?.loading} + {:else} - -
- {file_info.display_name} -
+ {/if} -
-
+ {:else} + + {/if} + + + + {#if file_info.display_name === ''} + (empty name) + {:else}{file_info.display_name}{/if} + + + {#if isFolder && !lazyMode} + + {file_info.count}{count % 1000 === 0 && + lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name + ? '+' + : ''} + + {/if} + {/if}
{/snippet}
{#if fileListLoading === true} -
- Loading content +
+ Loading...
{:else if lazyMode}
{displayedCount} item{displayedCount === 1 ? '' : 's'} shown
{:else} -
+
{displayedCount}{flatHasMore ? '+' : ''} {displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
{#if flatHasMore} - {/if} {/if}
{/if} -
+ {/if} -
- {#if fileMetadata === undefined} -
- {#if fileInfoLoading} -
- {:else if fileListUnavailable} -
- {:else} -
- {/if} -
- {:else} -
-
-

- {((p) => (p.startsWith(rootPath) ? p.slice(rootPath.length) : p))( - fileMetadata.fileKey - )} -

- {#if filePreview !== undefined && (!hideS3SpecificDetails || !readOnlyMode || allowDelete)} -
- {#if !hideS3SpecificDetails} - {@const downloadApiPath = `/w/${ws}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}${s3ResourcePath ? `&s3_resource_path=${encodeURIComponent(s3ResourcePath)}` : ''}`} - {@const downloadName = - fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'} - {#if shouldDownloadViaClient()} -
+ {:else if !csvOptions} +
+ {#if filePreview.contentType === 'Csv'} + Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character: +
+ +
+ Header row: +
+ +
+ {:else} + Previewing a {filePreview.contentType?.toLowerCase()} file. + {/if} +
+ {/if} 
{#if !emptyString(filePreview.contentPreview)}{filePreview.contentPreview}{:else if filePreview.contentType !== undefined}Preview impossible.{/if}
diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index 21371aa08b..ae3a82e97c 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -41,6 +41,8 @@ export type ColumnDef = { children?: ColumnDef[] // DBExplorer ignored?: boolean + /** False when the column cannot be filtered on, e.g. a nested value. */ + filterable?: boolean hideInsert?: boolean editable?: boolean overrideDefaultValue?: boolean diff --git a/frontend/src/lib/components/dbTableFilters.ts b/frontend/src/lib/components/dbTableFilters.ts index 9d9c83e6b0..c3266b3ac7 100644 --- a/frontend/src/lib/components/dbTableFilters.ts +++ b/frontend/src/lib/components/dbTableFilters.ts @@ -52,7 +52,7 @@ export function buildDbTableFilterSchema(columns: ColumnDef[]): DbTableFilterSch const columnOfKey: Record = {} const keyOfColumn: Record = {} for (const col of columns) { - if (!col.field) continue + if (!col.field || col.filterable === false) continue let base = col.field.replace(/\W+/g, '_') || 'column' if (base === FREE_TEXT_KEY) base = `${base}col` let key = base diff --git a/frontend/src/lib/components/s3TableOps.test.ts b/frontend/src/lib/components/s3TableOps.test.ts new file mode 100644 index 0000000000..f32ce47c56 --- /dev/null +++ b/frontend/src/lib/components/s3TableOps.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' +import { datatypeOf, jsonTableRows, serverTableFormat } from './s3TableOps' +import { dbColumnKind } from './dbTableFilters' + +describe('serverTableFormat', () => { + it('pages delimited text and newline-delimited JSON through the csv reader', () => { + expect(serverTableFormat('a/b.parquet')).toBe('parquet') + for (const key of ['a.csv', 'a.TSV', 'a.jsonl', 'a.ndjson']) { + expect(serverTableFormat(key)).toBe('csv') + } + }) + + it('leaves plain JSON to the browser', () => { + expect(serverTableFormat('a.json')).toBeUndefined() + expect(serverTableFormat('a.txt')).toBeUndefined() + }) +}) + +describe('datatypeOf', () => { + it('maps Arrow types to the kinds the grid filters by', () => { + expect(dbColumnKind(datatypeOf('Int64'))).toBe('number') + expect(dbColumnKind(datatypeOf('UInt8'))).toBe('number') + expect(dbColumnKind(datatypeOf('Float64'))).toBe('number') + expect(dbColumnKind(datatypeOf('Decimal128(21, 1)'))).toBe('number') + expect(dbColumnKind(datatypeOf('Boolean'))).toBe('boolean') + expect(dbColumnKind(datatypeOf('Utf8View'))).toBe('text') + }) + + it('names a nested type without its field list', () => { + expect(datatypeOf('List(Field { name: "item", data_type: Utf8 })')).toBe('list') + expect(datatypeOf('Timestamp(Microsecond, None)')).toBe('timestamp') + }) +}) + +describe('jsonTableRows', () => { + it('keeps objects and wraps other elements in a value column', () => { + expect(jsonTableRows([{ a: 1 }, 2, null])).toEqual([{ a: 1 }, { value: 2 }, { value: null }]) + }) + + it('has no table for an empty array or a non-array', () => { + expect(jsonTableRows([])).toBeUndefined() + expect(jsonTableRows({ a: 1 })).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/s3TableOps.ts b/frontend/src/lib/components/s3TableOps.ts new file mode 100644 index 0000000000..f5d7bd758b --- /dev/null +++ b/frontend/src/lib/components/s3TableOps.ts @@ -0,0 +1,170 @@ +import { HelpersService, OpenAPI } from '$lib/gen' +import { getHeaders } from '$lib/gen/core/request' +import { ColumnIdentity, type ColumnDef } from './apps/components/display/dbtable/utils' +import type { IDbTableOps } from './dbOps' + +/** Files the server can page through as a table, without reading them whole. */ +export function serverTableFormat(fileKey: string): 'parquet' | 'csv' | undefined { + const key = fileKey.toLowerCase() + if (key.endsWith('.parquet')) return 'parquet' + if (/\.(csv|tsv|jsonl|ndjson)$/.test(key)) return 'csv' + return undefined +} + +export type S3TableSource = { + workspace: string + fileKey: string + storage?: string + s3ResourcePath?: string + csvSeparator?: string + csvHasHeader?: boolean +} + +type Chunk = { + columns: string[] + column_types?: string[] + rows: Record[] + csv_separator?: string +} + +/** An Arrow type as the SQL type name the grid filters and formats by. */ +export function datatypeOf(arrowType: string | undefined): string { + const t = arrowType ?? '' + if (/^U?Int\d+$/.test(t)) return 'bigint' + if (/^Float\d+$/.test(t)) return 'double' + if (/^Decimal/.test(t)) return 'decimal' + if (t === 'Boolean') return 'boolean' + if (/^(Large)?Utf8/.test(t)) return 'text' + // A nested type spells out its fields, e.g. `List(Field { name: "item", … })`. + return t.split('(')[0].toLowerCase() || 'text' +} + +/** Read-only table ops over an object storage file, paged, searched, filtered and sorted + * by the server, with the CSV separator it read the file with (guessed when none is given). + */ +export async function s3TableOps( + source: S3TableSource +): Promise<{ ops: IDbTableOps; csvSeparator?: string }> { + const format = serverTableFormat(source.fileKey) + if (!format) throw new Error(`${source.fileKey} cannot be read as a table`) + const common = { + workspace: source.workspace, + path: source.fileKey, + storage: source.storage, + s3ResourcePath: source.s3ResourcePath + } + const csvOptions: { csvSeparator?: string; csvHasHeader?: boolean } = + format === 'csv' ? { csvSeparator: source.csvSeparator, csvHasHeader: source.csvHasHeader } : {} + const filtersParam = (columnFilters: Record | undefined) => + columnFilters && Object.keys(columnFilters).length ? JSON.stringify(columnFilters) : undefined + const searchParam = (quicksearch: string) => quicksearch.trim() || undefined + + async function loadChunk(params: { + offset: number + limit: number + quicksearch?: string + columnFilters?: Record + orderBy?: string + isDesc?: boolean + }): Promise { + const data = { + ...common, + ...csvOptions, + offset: params.offset, + limit: params.limit, + searchTerm: searchParam(params.quicksearch ?? ''), + filters: filtersParam(params.columnFilters), + sortCol: params.orderBy, + sortDesc: params.isDesc + } + return (await (format === 'parquet' + ? HelpersService.loadParquetPreview(data) + : HelpersService.loadCsvPreview(data))) as Chunk + } + + const head = await loadChunk({ offset: 0, limit: 0 }) + // Later reads reuse the separator the server guessed, rather than each guessing again. + if (format === 'csv') csvOptions.csvSeparator ??= head.csv_separator + const colDefs: ColumnDef[] = head.columns.map((field, i) => ({ + field, + datatype: datatypeOf(head.column_types?.[i]), + // The server cannot read a struct, map or union as text to filter it. + filterable: !/^(Struct|Map|Union)/.test(head.column_types?.[i] ?? ""), + defaultvalue: '', + isprimarykey: false, + isidentity: ColumnIdentity.No, + isnullable: 'YES', + isenum: false + })) + + const ops: IDbTableOps = { + dbType: 'duckdb', + tableKey: source.fileKey.split('/').pop() ?? source.fileKey, + colDefs, + getCount: async ({ quicksearch, columnFilters }) => { + const res = await HelpersService.loadTableRowCount({ + ...common, + ...csvOptions, + searchTerm: searchParam(quicksearch), + filters: filtersParam(columnFilters) + }) + return res.count ?? 0 + }, + getRows: async ({ + offset, + limit, + quicksearch, + columnFilters, + order_by, + is_desc, + explicitSort + }) => + ( + await loadChunk({ + offset, + limit, + quicksearch, + columnFilters, + // Unsorted, a file keeps its own row order. + orderBy: explicitSort ? order_by : undefined, + isDesc: is_desc + }) + ).rows + } + return { ops, csvSeparator: head.csv_separator } +} + +/** Plain JSON cannot be paged by the server, so only a file this small is read whole into + * the browser to be shown as a table. */ +export const JSON_TABLE_MAX_BYTES = 10 * 1024 * 1024 + +export function isJsonFile(fileKey: string): boolean { + return fileKey.toLowerCase().endsWith('.json') +} + +/** Rows of a JSON array, an element that is not an object becoming a one-column row; + * undefined for anything else. */ +export function jsonTableRows(value: unknown): Record[] | undefined { + if (!Array.isArray(value) || value.length === 0) return undefined + return value.map((v) => + v !== null && typeof v === 'object' && !Array.isArray(v) + ? (v as Record) + : { value: v } + ) +} + +export async function fetchJsonTableRows( + source: S3TableSource +): Promise[] | undefined> { + const query = new URLSearchParams({ file_key: source.fileKey }) + if (source.storage) query.set('storage', source.storage) + if (source.s3ResourcePath) query.set('s3_resource_path', source.s3ResourcePath) + const apiPath = `/w/${source.workspace}/job_helpers/download_s3_file?${query}` + const headers = await getHeaders(OpenAPI, { method: 'GET', url: apiPath }) + const response = await fetch(`${OpenAPI.BASE}${apiPath}`, { + headers, + credentials: OpenAPI.CREDENTIALS + }) + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`) + return jsonTableRows(await response.json()) +} From 478eef735222e60de8ed6086f6b036e2346f1de4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 09:56:48 +0200 Subject: [PATCH 38/60] fix: resolve $var: references in mcp resource headers (#11679) * fix: resolve $var: references in mcp resource headers Co-Authored-By: Claude Opus 5.5 * fix: require variables:read scope to resolve mcp header variables Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/tests/mcp_header_var_refs.rs | 110 ++++++++++++++++++++++++ backend/windmill-api/src/mcp_tools.rs | 28 +++--- backend/windmill-mcp/src/client/mod.rs | 3 +- backend/windmill-worker/src/ai/utils.rs | 14 ++- 4 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 backend/tests/mcp_header_var_refs.rs diff --git a/backend/tests/mcp_header_var_refs.rs b/backend/tests/mcp_header_var_refs.rs new file mode 100644 index 0000000000..158bc71a5f --- /dev/null +++ b/backend/tests/mcp_header_var_refs.rs @@ -0,0 +1,110 @@ +//! An MCP resource header whose value is a `$var:` reference reaches the MCP +//! server as the variable's value, and only for a caller allowed to read it. +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use tokio::io::AsyncReadExt; +use windmill_common::client::AuthedClient; +use windmill_test_utils::*; +use windmill_worker::{load_mcp_tools, McpResourceConfig}; + +const SECRET_VALUE: &str = "S3CRET-MCP-HEADER-VALUE"; + +/// A secret variable and an MCP resource at `url` sending it as a header. +async fn seed(db: &Pool, url: &str) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'u/test-user/mcp_header', $1, true, '', '{}')", + ) + .bind(windmill_common::variables::encrypt(&mc, SECRET_VALUE)) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/mcp', $1, 'mcp', '{}', 'test-user')", + ) + .bind(serde_json::json!({ + "name": "mcp", + "url": url, + "headers": { "X-Api-Key": "$var:u/test-user/mcp_header", "X-Plain": "plain" } + })) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_is_resolved(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + std::env::set_var("ALLOW_PRIVATE_MCP_SERVER_URLS", "true"); + + // Not an MCP server: it captures the first request and hangs up, so loading + // fails at the connect step after the headers were sent. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let mcp_port = listener.local_addr()?.port(); + let first_request = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = vec![0u8; 16 * 1024]; + let n = socket.read(&mut buf).await?; + anyhow::Ok(String::from_utf8_lossy(&buf[..n]).to_lowercase()) + }); + seed(&db, &format!("http://127.0.0.1:{mcp_port}/mcp")).await?; + + let server = ApiServer::start(db.clone()).await?; + let client = AuthedClient::new( + format!("http://localhost:{}", server.addr.port()), + "test-workspace".to_string(), + "SECRET_TOKEN".to_string(), + None, + ); + let config = vec![McpResourceConfig { + resource_path: "$res:u/test-user/mcp".to_string(), + include_tools: None, + exclude_tools: None, + }]; + let _ = load_mcp_tools(&db, "test-workspace", config, &client).await; + + let request = first_request.await??; + assert!( + request.contains(&format!("x-api-key: {}", SECRET_VALUE.to_lowercase())), + "the header must carry the variable's value, not the reference: {request}" + ); + assert!(request.contains("x-plain: plain"), "{request}"); + + Ok(()) +} + +/// A token scoped to the resource alone must not have the API read the +/// variable a header points at. +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_requires_variable_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed(&db, "https://mcp.invalid.windmill.test").await?; + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('RES_ONLY_TOKEN'::bytea), 'hex'), 'RES_ONLY_T', 'RES_ONLY_TOKEN', + 'test@windmill.dev', 'resources only', false, '{resources:read}')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{}/api/w/test-workspace/resources/mcp_tools/u/test-user/mcp", + server.addr.port() + )) + .header("Authorization", "Bearer RES_ONLY_TOKEN") + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "{body}"); + assert!( + body.contains("variables:read:u/test-user/mcp_header"), + "{body}" + ); + + Ok(()) +} diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index e1931873a9..1e07f374de 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -23,14 +23,12 @@ async fn with_deadline( what: &str, fut: impl std::future::Future>, ) -> Result { - tokio::time::timeout(MCP_DEADLINE, fut) - .await - .map_err(|_| { - Error::ExecutionErr(format!( - "MCP server did not answer within {}s ({what})", - MCP_DEADLINE.as_secs() - )) - })? + tokio::time::timeout(MCP_DEADLINE, fut).await.map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? } /// Connect to the MCP server described by the `mcp` resource at `path`. @@ -65,8 +63,9 @@ async fn connect_mcp_client( let resource_value = not_found_if_none(resource_value_o, "Resource", path)? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - let mcp_resource = serde_json::from_str::(resource_value.0.get()) - .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; + let mut mcp_resource = + serde_json::from_str::(resource_value.0.get()) + .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; #[cfg(feature = "oauth2")] { @@ -126,6 +125,15 @@ async fn connect_mcp_client( None }; + // Header secrets go through the same permissioned path as the token. + for value in mcp_resource.headers.iter_mut().flatten().map(|(_, v)| v) { + if let Some(var_path) = value.strip_prefix("$var:") { + check_scopes(authed, || format!("variables:read:{}", var_path))?; + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + *value = get_value_internal(&db_authed, w_id, var_path, false).await?; + } + } + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index 863d6b0355..5d13a40314 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -70,7 +70,8 @@ impl McpClient { headers.insert(name, value); } _ => { - tracing::warn!("Invalid header: {}={}", key, value); + // The value may be a resolved secret: keep it out of the logs. + tracing::warn!("Invalid header: {}", key); } } } diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 7437184385..50bbcad58f 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -570,7 +570,7 @@ pub async fn load_mcp_tools( // is not allowed to read (resources:read:{path}). Reading through the raw // db pool here would bypass the authorization enforced by the regular MCP // tools API (get_mcp_tools) and act as a confused deputy. - let mcp_resource = client + let mut mcp_resource = client .get_resource_value::(path) .await .map_err(|e| { @@ -615,6 +615,18 @@ pub async fn load_mcp_tools( None }; + // Header secrets go through the same permissioned path as the token. + for (name, value) in mcp_resource.headers.iter_mut().flatten() { + if let Some(var_path) = value.strip_prefix("$var:") { + *value = client.get_variable_value(var_path).await.map_err(|e| { + Error::internal_err(format!( + "Failed to resolve variable {} for header {} of MCP resource {}: {}", + var_path, name, resource_name, e + )) + })?; + } + } + // Create new MCP client for this execution tracing::debug!("Creating fresh MCP client for {}", resource_name); let mcp_conn = McpClient::from_resource(mcp_resource, token) From b347e792f85a110cc99ec13ab52459def8079195 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 10:47:52 +0200 Subject: [PATCH 39/60] make the packaged components resolvable by consumers (#11682) * fix: make the packaged components resolvable by consumers The svelte-package output of @windmill-labs/components imported the chat SDK from outside the package root, pointed `$lib/utils` at a directory with no index, and shipped an `import.meta.glob` on the `$lib` alias. Bundle the chat SDK entry point into the package, repair the directory specifier, make the glob relative, and fail packaging on any import a consumer cannot resolve. Co-Authored-By: Claude Opus 5.5 * fix: pin the chat sdk output root and widen the packaging checks Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- frontend/scripts/fix-svelte-module-imports.js | 77 ++++++++++++++++++- .../src/lib/components/mcp/providerIcon.ts | 5 +- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/frontend/scripts/fix-svelte-module-imports.js b/frontend/scripts/fix-svelte-module-imports.js index 9cec6265a0..fe4fde4959 100644 --- a/frontend/scripts/fix-svelte-module-imports.js +++ b/frontend/scripts/fix-svelte-module-imports.js @@ -16,9 +16,22 @@ // tarball. Copy the registry into the package and rewrite those imports // to point at it (mirrors how package-system-prompts.js bundles // `$system_prompts`). +// +// 3. Chat SDK: the `windmill-chat` alias (svelte.config.js) points at +// `chat-sdk/src/index.ts`, which escapes the package root the same way. +// Compile that entry point into the package and rewrite those imports. +// +// 4. `$lib/utils` from inside `utils/`: svelte-package turns it into `./`, +// which is the `utils/` directory (no index) rather than `utils.js`. +// Point a directory specifier with no index at its sibling module. +// +// A relative import that still escapes the package root, or an +// `import.meta.glob` left on an alias, fails the build: either one +// publishes fine and only breaks in the consumer's bundler. +import { execFileSync } from 'child_process' import { copyFileSync, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs' import { fileURLToPath } from 'url' -import { dirname, join, relative, resolve } from 'path' +import { dirname, join, relative, resolve, sep } from 'path' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -29,8 +42,37 @@ const oauthConnectDest = join(packageDir, 'oauth_connect.json') copyFileSync(oauthConnectSrc, oauthConnectDest) -const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?\/[^'"]*\.(?:svelte|json))\2/g +const chatSdkSrc = resolve(__dirname, '..', '..', 'chat-sdk', 'src', 'index.ts') +const chatSdkDest = join(packageDir, 'chat-sdk') + +// File arguments make tsc ignore every tsconfig and emit only what the entry +// point reaches, which leaves out the React and AI SDK adapters. +execFileSync( + 'npx', + [ + 'tsc', + chatSdkSrc, + '--outDir', + chatSdkDest, + '--rootDir', + dirname(chatSdkSrc), + '--declaration', + '--target', + 'ES2021', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck' + ], + { stdio: 'inherit', cwd: resolve(__dirname, '..') } +) + +const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?(?:\/[^'"]*)?)\2/g const OAUTH_CONNECT_RE = /^(\.\.\/)+backend\/oauth_connect\.json$/ +const CHAT_SDK_RE = /^(\.\.\/)+chat-sdk\/src\/index\.ts$/ +const DIRECTORY_RE = /(^|\/)\.\.?$|\/$/ +const ALIASED_GLOB_RE = /import\.meta\.glob(?:<[^>]*>)?\(\s*\[?\s*['"]\$/ function* walk(dir) { for (const entry of readdirSync(dir)) { @@ -50,9 +92,13 @@ function relativeSpecifier(fromFile, toFile) { let rewrittenFiles = 0 let rewrittenImports = 0 +const unresolvable = [] for (const file of walk(packageDir)) { const content = readFileSync(file, 'utf-8') + if (ALIASED_GLOB_RE.test(content)) { + unresolvable.push(`${relative(packageDir, file)}: import.meta.glob on an alias`) + } let changed = false const updated = content.replace(IMPORT_SPECIFIER_RE, (match, prefix, quote, specifier) => { if (OAUTH_CONNECT_RE.test(specifier)) { @@ -60,10 +106,27 @@ for (const file of walk(packageDir)) { rewrittenImports++ return `${prefix}${quote}${relativeSpecifier(file, oauthConnectDest)}${quote}` } + if (CHAT_SDK_RE.test(specifier)) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, join(chatSdkDest, 'index.js'))}${quote}` + } + const target = resolve(dirname(file), specifier) + if (target !== packageDir && !target.startsWith(packageDir + sep)) { + unresolvable.push(`${relative(packageDir, file)}: ${specifier}`) + return match + } + if (DIRECTORY_RE.test(specifier)) { + if (!existsSync(join(target, 'index.js')) && existsSync(target + '.js')) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, target + '.js')}${quote}` + } + return match + } if (!specifier.endsWith('.svelte')) { return match } - const target = resolve(dirname(file), specifier) // A real `.svelte` component file: leave the import alone. if (existsSync(target)) { return match @@ -82,6 +145,14 @@ for (const file of walk(packageDir)) { } } +if (unresolvable.length > 0) { + console.error( + `fix-svelte-module-imports: ${unresolvable.length} import(s) a consumer cannot resolve:\n` + + unresolvable.join('\n') + ) + process.exit(1) +} + console.log( `fix-svelte-module-imports: rewrote ${rewrittenImports} import(s) in ${rewrittenFiles} file(s)` ) diff --git a/frontend/src/lib/components/mcp/providerIcon.ts b/frontend/src/lib/components/mcp/providerIcon.ts index ed8d059a57..a666b2b1c5 100644 --- a/frontend/src/lib/components/mcp/providerIcon.ts +++ b/frontend/src/lib/components/mcp/providerIcon.ts @@ -12,8 +12,11 @@ import { findMcpEntry, findMcpEntryByUrl } from './registry' * the chat does not otherwise reach that barrel. `import.meta.glob` gives the * filenames at build time and the module only when a match is actually used, so * a workspace with two connections downloads two icons. + * + * The pattern is relative because svelte-package rewrites `$lib` only in import + * specifiers: an aliased glob ships as-is and no consumer's bundler expands it. */ -const iconModules = import.meta.glob('$lib/components/icons/*.svelte') as Record< +const iconModules = import.meta.glob('../icons/*.svelte') as Record< string, () => Promise<{ default: Component }> > From a8e2f1f1ca28e3352aff8a450188e3ff75a9637c Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 12 Oct 2026 11:45:03 +0200 Subject: [PATCH 40/60] fix: keep ai agent schema property order stable across jobs (#11687) * fix: keep ai agent schema property order stable across jobs Co-Authored-By: Claude Opus 5.5 * fix: keep tool parameter order when filtering preset inputs Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/windmill-ai/src/types.rs | 45 +++++++++++++++++++------ backend/windmill-worker/src/ai/utils.rs | 35 +++++++++++++++++-- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index e591e85159..59733e352c 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -1,3 +1,4 @@ +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; @@ -533,7 +534,7 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none")] pub items: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub properties: Option>>, + pub properties: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub required: Option>, #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] @@ -560,9 +561,9 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none", rename = "$ref")] pub ref_path: Option, #[serde(skip_serializing_if = "Option::is_none", rename = "$defs")] - pub defs: Option>>, + pub defs: Option>>, #[serde(skip_serializing_if = "Option::is_none")] - pub definitions: Option>>, + pub definitions: Option>>, // Schema composition #[serde(skip_serializing_if = "Option::is_none", rename = "allOf")] @@ -829,7 +830,7 @@ impl OpenAPISchema { // Merge properties (other's properties are added if key doesn't exist) if let Some(ref other_props) = other.properties { - let props = self.properties.get_or_insert_with(HashMap::new); + let props = self.properties.get_or_insert_with(IndexMap::new); for (key, value) in other_props { props.entry(key.clone()).or_insert_with(|| value.clone()); } @@ -880,13 +881,13 @@ impl OpenAPISchema { // Merge $defs and definitions if let Some(ref other_defs) = other.defs { - let defs = self.defs.get_or_insert_with(HashMap::new); + let defs = self.defs.get_or_insert_with(IndexMap::new); for (key, value) in other_defs { defs.entry(key.clone()).or_insert_with(|| value.clone()); } } if let Some(ref other_definitions) = other.definitions { - let definitions = self.definitions.get_or_insert_with(HashMap::new); + let definitions = self.definitions.get_or_insert_with(IndexMap::new); for (key, value) in other_definitions { definitions .entry(key.clone()) @@ -981,7 +982,29 @@ pub struct S3ObjectWithType { #[cfg(test)] mod tests { use super::*; - use std::collections::HashMap; + + /// The schema is part of the prompt prefix the provider caches on, so its keys must + /// serialize in the same order on every request. + #[test] + fn strict_schema_keeps_the_defined_property_order() { + let keys = [ + "zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "kappa", + ]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let mut schema: OpenAPISchema = + serde_json::from_str(&format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + schema.make_strict(); + + let value = serde_json::to_value(&schema).unwrap(); + let serialized: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(serialized, keys); + assert_eq!(value["required"], serde_json::json!(keys)); + } /// Whichever provider served the request, `input_tokens` has to end up meaning the /// whole prompt. Leaving the Anthropic shape as reported under-states it by the @@ -1248,7 +1271,7 @@ mod tests { #[test] fn test_make_strict_defs() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1274,7 +1297,7 @@ mod tests { #[test] fn test_make_strict_definitions() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut definitions = HashMap::new(); + let mut definitions = IndexMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; @@ -1528,7 +1551,7 @@ mod tests { #[test] fn test_flatten_all_of_preserves_defs() { let def_schema = object_schema(vec![("field", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1642,7 +1665,7 @@ mod tests { schema_url: Some("http://json-schema.org/draft-07/schema#".to_string()), ..Default::default() }; - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 50bbcad58f..f0a0b5c20f 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -103,9 +103,9 @@ pub fn filter_schema_by_input_transforms( .get_mut("properties") .and_then(|p| p.as_object_mut()) { - for key in &keys_to_remove { - properties.remove(key); - } + // `remove` swaps the last entry into the freed slot, so removing in set order + // would leave the survivors in a different order on every run. + properties.retain(|key, _| !keys_to_remove.contains(key)); } // Also remove from required array @@ -748,3 +748,32 @@ pub fn any_tool_needs_previous_result(tools: &[Tool]) -> bool { false }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A tool's schema sits in the prompt prefix the provider caches on, so the + /// parameters left for the model must keep their order on every run. + #[test] + fn filtering_preset_inputs_keeps_the_remaining_property_order() { + let keys = ["a", "b", "c", "d", "e", "f", "g", "h"]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let schema = + RawValue::from_string(format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + let preset = ["a", "c", "f"] + .iter() + .map(|k| (k.to_string(), InputTransform::Static { value: to_raw_value(&1) })) + .collect(); + + let filtered = filter_schema_by_input_transforms(schema, &preset).unwrap(); + let value: serde_json::Value = serde_json::from_str(filtered.get()).unwrap(); + let remaining: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(remaining, ["b", "d", "e", "g", "h"]); + } +} From 54eeb451e4c083e55d5f9b35692d65304c8abc49 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:31:01 +0200 Subject: [PATCH 41/60] refactor: remove vue as a full-code app framework (#11690) Co-authored-by: Claude Opus 5.5 --- backend/windmill-api/src/apps_raw_bundler.ts | 2 +- cli/src/commands/app/bundle.ts | 22 +- cli/src/commands/app/bundle_command.ts | 4 +- cli/src/commands/app/dev.ts | 6 +- cli/src/commands/app/lint.ts | 2 +- cli/src/commands/app/new.ts | 37 +- cli/src/commands/app/raw_apps.ts | 2 +- cli/src/guidance/skills.gen.ts | 16 +- .../components/copilot/chat/global/core.ts | 9 +- .../components/raw_apps/RawAppEditor.svelte | 5 +- .../src/lib/components/raw_apps/templates.ts | 408 +----------------- .../auto-generated/cli/cli-commands.md | 4 +- system_prompts/auto-generated/prompts.ts | 12 +- .../skills/cli-commands/SKILL.md | 4 +- .../auto-generated/skills/raw-app/SKILL.md | 12 +- system_prompts/base/raw-app-cli.md | 4 +- system_prompts/base/raw-app.md | 8 +- 17 files changed, 52 insertions(+), 505 deletions(-) diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index 6663a0f83e..965a9289b5 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -6,7 +6,7 @@ * The build itself is `wmill app bundle`, the same one `wmill app push` runs, so * an app deployed through the API is compiled exactly as the CLI and the editor * compile it — entry point, virtual `wmill` module, `/ui/` shared-UI resolution - * and the Svelte/Vue plugins included. Reimplementing any of that here would be + * and the Svelte plugin included. Reimplementing any of that here would be * a third bundler to keep in step with the other two. */ declare const __wmillAppPolicy: { diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 286e878cfb..287d01ebed 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -58,10 +58,10 @@ function conditionsFor(svelte: boolean): string[] { /** * Detects which frontend frameworks are present in package.json */ -export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolean } { +export function detectFrameworks(appDir: string): { svelte: boolean } { const packageJsonPath = path.join(appDir, "package.json"); if (!fs.existsSync(packageJsonPath)) { - return { svelte: false, vue: false }; + return { svelte: false }; } try { @@ -73,10 +73,9 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea return { svelte: "svelte" in allDeps, - vue: "vue" in allDeps, }; } catch { - return { svelte: false, vue: false }; + return { svelte: false }; } } @@ -256,17 +255,6 @@ export async function createFrameworkPlugins(appDir: string): Promise { plugins.push(createSveltePlugin(appDir)); } - if (frameworks.vue) { - log.info(colors.blue("🔧 Vue detected, adding vue plugin...")); - throw new Error("Vue plugin not supported yet"); - // try { - // const esbuildPluginVue = await import("esbuild-plugin-vue3"); - // plugins.push(esbuildPluginVue.default()); - // } catch (error: any) { - // log.warn(colors.yellow(`Failed to load vue plugin: ${error.message}`)); - // } - } - return plugins; } @@ -313,7 +301,7 @@ export async function createBundle( // Use the entryPoint's directory if provided, otherwise fall back to cwd. const appDir = options.entryPoint ? path.dirname(options.entryPoint) : process.cwd(); const frameworks = detectFrameworks(appDir); - const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = options.entryPoint ?? defaultEntry; const outDir = options.outDir ?? "dist"; @@ -331,7 +319,7 @@ export async function createBundle( // Ensure node_modules exists in the app directory await ensureNodeModules(appDir); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); // Ensure output directory exists diff --git a/cli/src/commands/app/bundle_command.ts b/cli/src/commands/app/bundle_command.ts index bef13f4268..c4119aafb0 100644 --- a/cli/src/commands/app/bundle_command.ts +++ b/cli/src/commands/app/bundle_command.ts @@ -12,10 +12,10 @@ interface BundleOptions { } /** Every entry point a raw app may have, most specific first, with the - * framework's preferred one promoted — Svelte and Vue mount from a `.ts`. */ + * framework's preferred one promoted — Svelte mounts from a `.ts`. */ function entryPointFor(appDir: string): string { const frameworks = detectFrameworks(appDir); - const candidates = frameworks.svelte || frameworks.vue + const candidates = frameworks.svelte ? ["index.ts", "index.tsx", "index.js"] : ["index.tsx", "index.ts", "index.js"]; const entry = candidates.find((c) => fs.existsSync(path.join(appDir, c))); diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 2dac8639e2..fdf26fa7da 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -481,7 +481,7 @@ async function dev(opts: DevOptions, appFolder?: string) { // Detect frameworks to determine default entry point const frameworks = detectFrameworks(process.cwd()); - const defaultEntry = frameworks.svelte || frameworks.vue + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = opts.entry ?? defaultEntry; @@ -577,7 +577,7 @@ async function dev(opts: DevOptions, appFolder?: string) { const buildOptions = getDevBuildOptions(entryPoint, frameworks.svelte); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); const wmillPlugin = { @@ -1602,7 +1602,7 @@ const command = new Command() }) .option( "--entry ", - "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)", + "Entry point file (default: index.ts for Svelte, index.tsx otherwise)", ) .option("--no-open", "Don't automatically open the browser") .option( diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 9356b5dd3c..2276a77cad 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -114,7 +114,7 @@ async function validateBuild( // Try to create a bundle - this will validate that all dependencies are in place const frameworks = detectFrameworks(appDir); - const entryFile = frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = path.join(appDir, entryFile); await createBundle({ entryPoint, diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts index b011965ae8..40cc2a4416 100644 --- a/cli/src/commands/app/new.ts +++ b/cli/src/commands/app/new.ts @@ -85,21 +85,6 @@ const app = mount(App, { target: document.getElementById("root")! }); export default app; `; -const appVue = ` - -`; - -const indexVue = `import { createApp } from 'vue' -import App from './App.vue' -import "./index.css"; - -createApp(App).mount('#root')`; - const indexCss = `body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; @@ -171,21 +156,6 @@ const templates: Record = { "svelte": "^5.55.5", "windmill-client": "^1" } -}`, - }, - }, - vue: { - name: "Vue 3", - files: { - "/index.ts": indexVue, - "/App.vue": appVue, - "/index.css": indexCss, - "/package.json": `{ - "dependencies": { - "core-js": "3.26.1", - "vue": "3.5.13", - "windmill-client": "^1" - } }`, }, }, @@ -251,7 +221,7 @@ interface DataConfig { schema?: string; } -type FrameworkKey = "react19" | "react18" | "svelte5" | "vue"; +type FrameworkKey = "react19" | "react18" | "svelte5"; interface NewAppOptions extends GlobalOptions { /** App summary (short description). Skips the prompt when provided. */ @@ -384,7 +354,7 @@ async function newApp(opts: NewAppOptions) { } // Ask for framework (skipped if --framework is provided) - const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5", "vue"]; + const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5"]; let framework: string; if (opts.framework !== undefined) { if (!VALID_FRAMEWORKS.includes(opts.framework)) { @@ -403,7 +373,6 @@ async function newApp(opts: NewAppOptions) { { name: "React 19 (Recommended)", value: "react19" }, { name: "React 18", value: "react18" }, { name: "Svelte 5", value: "svelte5" }, - { name: "Vue 3", value: "vue" }, ], }); } @@ -900,7 +869,7 @@ const command = new Command() ) .option( "--framework ", - "Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode." + "Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode." ) .option( "--datatable ", diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 36c8260e25..16179d4662 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -461,7 +461,7 @@ export async function pushRawApp( log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); // Detect frameworks to determine entry point const frameworks = detectFrameworks(localPath); - const entryFile = frameworks.svelte || frameworks.vue + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = localPath + entryFile; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index eb5d46ef57..4e8cc464d5 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6351,7 +6351,7 @@ You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. \`f/folder/my_app\` or \`u/username/my_app\` -3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\`, \`vue\` +3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\` **summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. @@ -6423,7 +6423,7 @@ my_app{{RAW_APP_SUFFIX}}/ ├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) ├── raw_app.yaml # App configuration (summary, path, data settings) ├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component +├── App.tsx # Main React/Svelte component ├── index.css # Styles ├── package.json # Frontend dependencies ├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) @@ -6581,7 +6581,7 @@ For the rest, tell the user which command fits their intent and let them run it # Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -6597,7 +6597,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -6627,7 +6627,7 @@ Then build what was chosen: the data table and schema become the app's \`data.da ### Entrypoint -The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim. +The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` and keep the entrypoint as the mount shim. React (\`index.tsx\`): @@ -6639,7 +6639,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() \`\`\` -Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. Vue (\`index.ts\`): \`createApp(App).mount('#root')\`. +Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. **Never replace the entrypoint with a bare component** (\`export default function App() { ... }\` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. @@ -8102,7 +8102,7 @@ app related commands - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement - \`--port \` - Port to run the dev server on (will find next available port if occupied) - \`--host \` - Host to bind the dev server to - - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - \`--entry \` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - \`--no-open\` - Don't automatically open the browser - \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability @@ -8113,7 +8113,7 @@ app related commands - \`app new\` - create a new raw app from a template - \`--summary \` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - \`--framework \` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index 2a795f1f81..5295712626 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -1306,12 +1306,7 @@ const readDomSchema = z.object({ const takeScreenshotSchema = z.object({}) -const FRAMEWORK_KEYS = [ - 'react19', - 'react18', - 'svelte5', - 'vue' -] as const satisfies readonly FrameworkKey[] +const FRAMEWORK_KEYS = ['react19', 'react18', 'svelte5'] as const satisfies readonly FrameworkKey[] const initAppSchema = z.object({ path: z @@ -2493,7 +2488,7 @@ function getAppInstructions(language?: ScriptLang): string { - Global mode edits raw app drafts only; it does not save or deploy unless the user explicitly asks to deploy. - App drafts are addressed by workspace path. Follow the path conventions in the system prompt: default to \`u//\` for bare names; only use \`f//\` when the folder is known to exist. The first write tool snapshots the workspace app onto the draft, and subsequent writes accumulate. -- To create a new app, use \`init_app\` with a path, optional summary, a framework (\`react19\` / \`react18\` / \`svelte5\` / \`vue\`) and, for an app that stores data, its \`data\` setup. "Starting a new app" below decides the framework and when the data setup is asked. Ask each of its questions with \`askUserQuestion\`, one question per call with the suggested answer first, before \`init_app\` and before any SQL. \`init_app\` only records \`data\`: a new schema is yours to create with the datatable SQL tool. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing. +- To create a new app, use \`init_app\` with a path, optional summary, a framework (\`react19\` / \`react18\` / \`svelte5\`) and, for an app that stores data, its \`data\` setup. "Starting a new app" below decides the framework and when the data setup is asked. Ask each of its questions with \`askUserQuestion\`, one question per call with the suggested answer first, before \`init_app\` and before any SQL. \`init_app\` only records \`data\`: a new schema is yours to create with the datatable SQL tool. \`init_app\` errors if an app already exists at the path or a draft is already in flight; in that case, edit the existing one rather than re-initializing. - \`init_app\` seeds a starter inline runnable named \`a\` (bun, \`main(x: string) => string\`) so the React/Svelte demo button works on first render. Replace or remove it once you start building real backend runnables. - Frontend file paths start with \`/\` (e.g. \`/index.tsx\`, \`/App.tsx\`, \`/styles.css\`). Use \`write_app_file\` / \`patch_app_file\` / \`delete_app_file\`. - Backend inline runnables are addressed as \`backend//main.{ts|py}\` from the file tools, but you create or update them via \`write_app_runnable\` / \`delete_app_runnable\` (which take the runnable shape directly: \`{ name, type, inlineScript?, path?, staticInputs? }\`). diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index f708b97efe..46c23b79b3 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -496,7 +496,7 @@ addEventListener('message', function (e) { // entrypoint defines a component without ever mounting it. Cleared on next build. let emptyRender = $state(false) // The repair hint above has to match the app's framework: only React apps have - // an `index.tsx` and `createRoot`; Svelte and Vue mount from `index.ts`. Keyed + // an `index.tsx` and `createRoot`; Svelte mounts from `index.ts`. Keyed // off file extensions rather than exact template filenames, which users rename. let mountHint = $derived.by(() => { const paths = Object.keys(files ?? {}).map((p) => p.replace(/^\//, '')) @@ -507,9 +507,6 @@ addEventListener('message', function (e) { call: "mount(App, { target: document.getElementById('root')! })" } } - if (paths.some((p) => p.endsWith('.vue'))) { - return { entrypoint: entrypoint ?? 'index.ts', call: "createApp(App).mount('#root')" } - } return { entrypoint: entrypoint ?? 'index.tsx', call: "createRoot(document.getElementById('root')!).render()" diff --git a/frontend/src/lib/components/raw_apps/templates.ts b/frontend/src/lib/components/raw_apps/templates.ts index 50eae99e97..00d7594386 100644 --- a/frontend/src/lib/components/raw_apps/templates.ts +++ b/frontend/src/lib/components/raw_apps/templates.ts @@ -1,5 +1,5 @@ // The starter app is a tour of what a raw app can do: each card calls one of the -// STARTER_RUNNABLES below through a different `./wmill` export. The three framework +// STARTER_RUNNABLES below through a different `./wmill` export. The framework // variants share `indexCss` and must stay in step with each other and with the runnables. const reactIndex = ` @@ -13,7 +13,7 @@ root.render(); ` const STATS_SAMPLE_TEXT = - 'Windmill turns scripts into apps. Write the frontend in React, Svelte or Vue, and the backend in TypeScript, Python or any other supported language. Scripts run on workers, and the app calls them like functions.' + 'Windmill turns scripts into apps. Write the frontend in React or Svelte, and the backend in TypeScript, Python or any other supported language. Scripts run on workers, and the app calls them like functions.' const appTsx = `import React, { useState } from 'react' import { backend, backendAsync, getJob, streamJob } from './wmill' @@ -516,230 +516,6 @@ const app = mount(App, { target: document.getElementById("root")! }); export default app; ` -const appVue = ` - -` - -const indexVue = `import { createApp } from 'vue' -import App from './App.vue' -import "./index.css"; - -createApp(App).mount('#root')` - const indexCss = `/* The app renders inside an iframe whose host page has its own theme the iframe cannot read, so paint an opaque surface here: without one the host's background shows through and dark text lands on it unreadable. */ @@ -1158,23 +934,10 @@ export const svelte5Template = { }` } -export const vueTemplate = { - '/index.ts': indexVue, - '/App.vue': appVue, - '/index.css': indexCss, - '/package.json': `{ - "dependencies": { - "core-js": "3.26.1", - "vue": "3.5.13" - } -}` -} - export const FRAMEWORK_TEMPLATES = { react19: react19Template, react18: react18Template, - svelte5: svelte5Template, - vue: vueTemplate + svelte5: svelte5Template } export type FrameworkKey = keyof typeof FRAMEWORK_TEMPLATES @@ -1551,171 +1314,6 @@ export async function main() { } } -export const appVueRouter = ` - - - - -` - export const appSvelteRouter = ` + +{#if loadError} + {loadError} +{:else if status} +
+
+ Tokens are signed by: + {modeLabels[status.mode]} +
+ {#if status.signer} +
+ Last key listing: + {status.signer.last_refresh_success + ? displayDate(status.signer.last_refresh_success) + : 'never'} + {#if status.signer.breaker !== 'closed'} + Signer unreachable + {:else if !status.signer.last_refresh_error && status.signer.last_refresh_success} + Signer reachable + {/if} +
+ {#if status.signer.last_refresh_error} + + {status.signer.last_refresh_error} + + {/if} + {#if status.signer.keys_from_store} + + The keys below are those stored by the last accepted listing. Tokens cannot be signed + until the signer answers. + + {/if} + {#if status.signer.tls_files_error} + + {status.signer.tls_files_error}. The previous certificate and key are kept. + + {/if} + {/if} + + + + Key ID (kid) + Algorithm + State + Source + + + + {#each status.keys as key, i (`${key.source}:${key.kid ?? i}`)} + + + + {key.kid} + {#if key.kid}{/if} + + + {key.alg} + + {key.state} + + {key.source} + + {/each} + + + {#if status.keys.length === 0} +
The JWKS serves no key.
+ {/if} +
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte index aa7a1a14df..d863eb7848 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/instance_settings/+page.svelte @@ -11,12 +11,14 @@ tabToAuthSubTab, categoryToTabMap, buildSearchableSettingItems, + listedNavigationGroups, + usesExternalOidcSigner, type SearchableSettingItem } from '$lib/components/instanceSettings' import SettingsSearchInput from '$lib/components/instanceSettings/SettingsSearchInput.svelte' import Breadcrumb from '$lib/components/common/breadcrumb/Breadcrumb.svelte' import { ChevronRight, ArrowLeft } from 'lucide-svelte' - import { superadmin } from '$lib/stores' + import { enterpriseLicense, superadmin } from '$lib/stores' import { onDestroy, tick } from 'svelte' import { SettingService, UserService } from '$lib/gen' import { sendUserToast } from '$lib/toast' @@ -178,7 +180,14 @@ } // --- Settings search (full mode) --- - const searchableItems = buildSearchableSettingItems(setupNavigationGroups) + let externalOidcSigner = $state(false) + $effect(() => { + if ($enterpriseLicense) usesExternalOidcSigner().then((v) => (externalOidcSigner = v)) + }) + const navigationGroups = $derived( + listedNavigationGroups(setupNavigationGroups, externalOidcSigner) + ) + const searchableItems = $derived(buildSearchableSettingItems(navigationGroups)) let scrollTimeout: ReturnType | undefined let highlightTimeout: ReturnType | undefined @@ -487,7 +496,7 @@
From 72d1092de97d7449ff03c3a44835c9d23cfe5e02 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 18:47:07 +0200 Subject: [PATCH 45/60] fix: run a workflow-as-code task called from inside a task in place (#11699) * fix: run a workflow-as-code task called from inside a task in place Co-Authored-By: Claude Opus 5.5 * fix: hand an async task body an awaitable for a sync nested task Co-Authored-By: Claude Opus 5.5 * fix: mark the task-body scope per nested task body Co-Authored-By: Claude Opus 5.5 * docs: note the task-body behaviour of step and sleep Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- cli/src/guidance/skills.gen.ts | 37 ++++++-- python-client/wmill/tests/test_workflow.py | 80 +++++++++++++++- python-client/wmill/wmill/client.py | 88 ++++++++++++++++-- system_prompts/auto-generated/prompts.ts | 31 +++++-- system_prompts/auto-generated/script.md | 12 ++- system_prompts/auto-generated/sdks/python.md | 9 +- .../auto-generated/sdks/typescript.md | 3 + .../auto-generated/sdks/wac-python.md | 9 +- .../auto-generated/sdks/wac-typescript.md | 8 +- .../skills/write-script-bun/SKILL.md | 3 + .../skills/write-script-bunnative/SKILL.md | 3 + .../skills/write-script-deno/SKILL.md | 3 + .../skills/write-script-python3/SKILL.md | 9 +- .../skills/write-workflow-as-code/SKILL.md | 19 +++- system_prompts/base/workflow-as-code.md | 2 + typescript-client/client.ts | 75 ++++++++++++--- .../tests/workflow_nested_task.test.ts | 91 +++++++++++++++++++ 17 files changed, 438 insertions(+), 44 deletions(-) create mode 100644 typescript-client/tests/workflow_nested_task.test.ts diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 4e8cc464d5..fff7df1153 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -1106,6 +1106,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -1949,6 +1952,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2898,6 +2904,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -5073,7 +5082,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -5159,12 +5170,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -7436,6 +7450,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with \`@task()\` or \`@task\`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or \`step()\` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. \`sleep()\` there waits in the job, and \`taskScript()\` / \`taskFlow()\` / \`waitForApproval()\` (\`task_script()\` / \`task_flow()\` / \`wait_for_approval()\`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: \`\`\`typescript @@ -7601,8 +7617,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -7646,6 +7663,9 @@ export function workflow(fn: (...args: any[]) => Promise) * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -7744,7 +7764,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -7830,12 +7852,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/python-client/wmill/tests/test_workflow.py b/python-client/wmill/tests/test_workflow.py index ceed39bfeb..d32ae1458b 100644 --- a/python-client/wmill/tests/test_workflow.py +++ b/python-client/wmill/tests/test_workflow.py @@ -7,7 +7,7 @@ import pytest from datetime import datetime, timezone -from wmill.client import WorkflowCtx, _StepSuspend, TaskError, workflow, task, step, sleep, parallel, wait_for_approval, _run_workflow, _run_workflow_async +from wmill.client import WorkflowCtx, _StepSuspend, TaskError, workflow, task, task_script, step, sleep, parallel, wait_for_approval, _run_workflow, _run_workflow_async class _FakeItems(dict): @@ -752,6 +752,84 @@ class TestChildModeParallel: assert result["result"] == 8 +class TestSdkCallsInsideTaskBody: + """A task body runs as one child job, so the SDK calls it makes cannot be + dispatched as steps of their own.""" + + def _run_child(self, wf, key): + async def bounded(): + return await asyncio.wait_for( + _run_workflow_async(wf, {"completed_steps": {}, "_executing_key": key}, {}), + timeout=2, + ) + + return asyncio.run(bounded()) + + def test_nested_task_and_step_run_in_the_tasks_own_job(self): + @task + async def inner(x: str): + return f"inner:{x}" + + @task + def sync_leaf(x: str): + return f"sync:{x}" + + # A sync body calls its tasks without awaiting, whatever called it. + @task + def sync_inner(x: str): + return sync_leaf(x).upper().lower() + + @task + async def outer(x: str): + via_task = await inner(x) + via_sync = await sync_inner(x) + via_step = await step("s", lambda: "stepped") + return f"{via_task}+{via_sync}+{via_step}" + + @workflow + async def wf(): + return await outer("a") + + result = self._run_child(wf, "outer") + assert result == {"type": "complete", "result": "inner:a+sync:a+stepped"} + + def test_sibling_task_of_the_workflow_body_is_not_run(self): + # The workflow body keeps replaying beside the task body, and its own + # calls must stay parked: running them would execute a sibling task in + # this job. + ran = [] + + @task + async def inner(x: str): + ran.append(x) + return x + + @task + async def outer(x: str): + await asyncio.sleep(0) + return await inner("nested") + + @workflow + async def wf(): + return await asyncio.gather(outer("a"), inner("sibling")) + + result = self._run_child(wf, "outer") + assert result == {"type": "complete", "result": "nested"} + assert ran == ["nested"] + + def test_step_that_would_need_a_dispatch_fails_instead_of_hanging(self): + @task + async def dispatching(): + return await task_script("f/other/script")() + + @workflow + async def wf(): + return await dispatching() + + with pytest.raises(RuntimeError, match="cannot be called from inside a task"): + self._run_child(wf, "dispatching") + + class TestKeyDeterminism: def test_keys_consistent_across_replays(self): r1 = _run_workflow(det_wf, {}, {"n": 3}) diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 97da60aadc..824db9741f 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -2766,6 +2766,7 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]: import asyncio as _asyncio import contextvars as _contextvars +import inspect as _inspect import sys as _sys import traceback as _traceback @@ -2993,6 +2994,64 @@ _workflow_ctx: _contextvars.ContextVar["WorkflowCtx"] = _contextvars.ContextVar( "_workflow_ctx" ) +# Set while a task body runs in its child job: ``"async"`` or ``"sync"`` after +# the kind of function the body is. A flag on the ctx cannot stand in: under +# ``asyncio.gather`` the workflow body keeps replaying beside the task body, and +# its calls must still park rather than run a sibling task's body in this job. +_in_task_body: _contextvars.ContextVar[Optional[str]] = _contextvars.ContextVar( + "_in_task_body", default=None +) + + +def _dispatch_ctx() -> "WorkflowCtx | None": + """The ctx SDK calls dispatch through, or None where they run in place: + outside a workflow, and inside a task body, which is one job and has no + rounds of its own to dispatch or checkpoint a step in.""" + if _in_task_body.get(): + return None + return _workflow_ctx.get(None) + + +async def _resolved(value): + return value + + +def _call_in_task_body(func, caller: str, args, kwargs): + """Run a task called from inside a task body in place, round-tripping its + result as a dispatched one would be. + + The workflow body awaits every task, sync ones included, so an async caller + gets an awaitable back either way; a sync caller gets the value. The scope + is re-marked for the nested body, which may be of the other kind.""" + + async def run_async(coro): + token = _in_task_body.set("async") + try: + return _json_round_trip(await coro) + finally: + _in_task_body.reset(token) + + if _inspect.iscoroutinefunction(func): + return run_async(func(*args, **kwargs)) + token = _in_task_body.set("sync") + try: + result = func(*args, **kwargs) + finally: + _in_task_body.reset(token) + if _asyncio.iscoroutine(result): + return run_async(result) + result = _json_round_trip(result) + return _resolved(result) if caller == "async" else result + + +def _assert_not_inside_task_body(what: str) -> None: + if _in_task_body.get(): + raise RuntimeError( + f"{what} cannot be called from inside a task: a task runs as a single " + "job and cannot suspend or dispatch steps. Call it from the @workflow " + "body instead." + ) + class WorkflowCtx: """Internal context for workflow replay/suspension. @@ -3117,15 +3176,20 @@ class WorkflowCtx: raise _StepSuspend({"mode": "sleep", "key": key, "seconds": seconds, "steps": []}) async def _resolved(self, value): - return value + return await _resolved(value) async def _execute_directly(self, func, **kwargs): + token = _in_task_body.set( + "async" if _inspect.iscoroutinefunction(func) else "sync" + ) try: result = func(**kwargs) if _asyncio.iscoroutine(result): result = await result except Exception as exc: raise _StepFailure(exc) from exc + finally: + _in_task_body.reset(token) raise _StepSuspend({"mode": "step_complete", "steps": [], "result": result}) async def _never_resolve(self): @@ -3367,7 +3431,9 @@ def task( - **v2 (inside @workflow)**: dispatches as a checkpoint step. - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. - - **Standalone**: executes the function body directly. + - **Standalone, or called from inside another v2 task**: executes the + function body directly. A task runs as a single job, so the tasks it + calls run in that same job, without their own options. A task runs as its own job, so its result is always encoded as JSON and decoded back before the caller sees it: a ``datetime`` comes back as a @@ -3447,12 +3513,16 @@ def task( @functools.wraps(func) def wrapper(*args, **kwargs): # WAC v2: inside a @workflow context - ctx = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: script = task_path if task_path else task_name merged = _merge_args(args, kwargs) return ctx._next_step(task_name, script, func, _task_options=_task_opts, **merged) + caller = _in_task_body.get() + if caller is not None: + return _call_in_task_body(func, caller, args, kwargs) + # WAC v1: running inside a Windmill job but not in a @workflow if ( os.environ.get("WM_JOB_ID") is not None @@ -3531,6 +3601,7 @@ def task_script( _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): + _assert_not_inside_task_body(f'task_script("{path}")') ctx = _workflow_ctx.get(None) if ctx is not None: return ctx._next_step(name, path, dispatch_type="script", _task_options=_opts, **kwargs) @@ -3570,6 +3641,7 @@ def task_flow( _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): + _assert_not_inside_task_body(f'task_flow("{path}")') ctx = _workflow_ctx.get(None) if ctx is not None: return ctx._next_step(name, path, dispatch_type="flow", _task_options=_opts, **kwargs) @@ -3604,8 +3676,11 @@ async def step(name: str, fn): ``fn``'s result is encoded as JSON and decoded back before it is returned, so the round that runs the body sees the same types every replay sees: a ``datetime`` comes back as a string, a tuple as a list. + + Called from inside a task body, ``fn`` runs in place and nothing is + checkpointed: a task is a single job and is never replayed. """ - ctx: WorkflowCtx | None = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: return await ctx._run_inline_step(name, fn) result = fn() @@ -3620,9 +3695,9 @@ async def sleep(seconds: int): """Server-side sleep — suspend the workflow for the given duration without holding a worker. Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. - Outside a workflow, falls back to ``asyncio.sleep``. + Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. """ - ctx: WorkflowCtx | None = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: return await ctx._sleep(seconds) await _asyncio.sleep(seconds) @@ -3660,6 +3735,7 @@ async def wait_for_approval( await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) result = await wait_for_approval(key="manager", timeout=3600) """ + _assert_not_inside_task_body("wait_for_approval()") ctx: WorkflowCtx | None = _workflow_ctx.get(None) if ctx is not None: return await ctx._wait_for_approval( diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index cfa3415e35..393b724f6b 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1427,6 +1427,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with \`@task()\` or \`@task\`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or \`step()\` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. \`sleep()\` there waits in the job, and \`taskScript()\` / \`taskFlow()\` / \`waitForApproval()\` (\`task_script()\` / \`task_flow()\` / \`wait_for_approval()\`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: \`\`\`typescript @@ -2029,6 +2031,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2871,7 +2876,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -2957,12 +2964,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -3084,8 +3094,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -3129,6 +3140,9 @@ export function workflow(fn: (...args: any[]) => Promise) * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -3227,7 +3241,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -3313,12 +3329,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 14ca869180..a4064c3d9e 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1906,6 +1906,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2748,7 +2751,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -2834,12 +2839,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 13a93ed063..1964c88685 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -669,7 +669,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -755,12 +757,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index 55df35ccf7..da9e79ec94 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -463,6 +463,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index a8e98a9d4f..6dd307dd24 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -36,7 +36,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -122,12 +124,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index 608d75d18c..dd04a64de0 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -67,8 +67,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -112,6 +113,9 @@ export function workflow(fn: (...args: any[]) => Promise) * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 6ad44db068..055b1613b2 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -659,6 +659,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index a5f599a946..beaf7817bb 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -671,6 +671,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index ce0c610da7..afefb7e987 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -661,6 +661,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index 4946ed0f6e..ee50b6d62d 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -877,7 +877,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -963,12 +965,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index 13d305c74e..09789b388a 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -143,6 +143,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or `step()` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. `sleep()` there waits in the job, and `taskScript()` / `taskFlow()` / `waitForApproval()` (`task_script()` / `task_flow()` / `wait_for_approval()`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: ```typescript @@ -308,8 +310,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -353,6 +356,9 @@ export function workflow(fn: (...args: any[]) => Promise) * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -451,7 +457,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -537,12 +545,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/base/workflow-as-code.md b/system_prompts/base/workflow-as-code.md index 0e529f6803..efc29df360 100644 --- a/system_prompts/base/workflow-as-code.md +++ b/system_prompts/base/workflow-as-code.md @@ -95,6 +95,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or `step()` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. `sleep()` there waits in the job, and `taskScript()` / `taskFlow()` / `waitForApproval()` (`task_script()` / `task_flow()` / `wait_for_approval()`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: ```typescript diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 3f7b03117a..da957239cf 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -1832,6 +1832,52 @@ export function setWorkflowCtx(ctx: WorkflowCtx | null) { Reflect.set(globalThis, "__wmill_wf_ctx", ctx); } +type TaskBodyScope = { run(store: true, fn: () => R): R; getStore(): true | undefined }; + +/** Marks the async context of a task body running in its child job. A flag on + * the ctx cannot stand in: the workflow body keeps replaying beside the task + * body (`Promise.all([a(), b()])`), and its calls must still park rather than + * run a sibling task's body in this job. Kept on `globalThis`, like the ctx, + * so a bundle holding two copies of this module shares one scope. Resolved + * through `process.getBuiltinModule` because a `node:` import would break + * browser bundles of this package. */ +function taskBodyScope(): TaskBodyScope | null { + const existing = Reflect.get(globalThis, "__wmill_wf_task_body"); + if (existing !== undefined) return existing; + let scope: TaskBodyScope | null = null; + try { + const AsyncLocalStorage = (globalThis as any).process?.getBuiltinModule?.("node:async_hooks")?.AsyncLocalStorage; + if (AsyncLocalStorage) scope = new AsyncLocalStorage(); + } catch {} + Reflect.set(globalThis, "__wmill_wf_task_body", scope); + return scope; +} + +function runTaskBody(fn: () => R): R { + const scope = taskBodyScope(); + return scope ? scope.run(true, fn) : fn(); +} + +function insideTaskBody(): boolean { + return taskBodyScope()?.getStore() === true; +} + +/** The ctx SDK calls dispatch through, or null where they run in place: outside + * a workflow, and inside a task body, which is one job and has no rounds of its + * own to dispatch or checkpoint a step in. */ +function dispatchCtx(): WorkflowCtx | null { + if (insideTaskBody()) return null; + return _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); +} + +function assertNotInsideTaskBody(what: string): void { + if (insideTaskBody()) { + throw new Error( + `${what} cannot be called from inside a task: a task runs as a single job and cannot suspend or dispatch steps. Call it from the workflow() body instead.`, + ); + } +} + export class WorkflowCtx { private completed: Record; @@ -2304,11 +2350,11 @@ export class WorkflowCtx { } export async function sleep(seconds: number): Promise { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { return ctx._sleep(seconds) as Promise; } - // Outside workflow context, just wait locally + // Outside a workflow, or inside a task body, just wait locally await new Promise((r) => setTimeout(r, seconds * 1000)); } @@ -2319,12 +2365,15 @@ export async function sleep(seconds: number): Promise { * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step( name: string, fn: () => T | Promise, ): Promise>> { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { return ctx._runInlineStep(name, fn) as Promise>>; } @@ -2342,8 +2391,9 @@ export async function step( * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -2377,7 +2427,7 @@ export function task Promise>( // .then() which throws StepSuspend and empties pending before the caller // can flush. const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { // Inside a workflow with checkpoint/replay context — dispatch as step const script = taskPath ?? taskName; @@ -2397,7 +2447,7 @@ export function task Promise>( return (async () => { let result: any; try { - result = await fn(...args); + result = await runTaskBody(() => fn(...args)); } catch (e) { if (isSuspendSignal(e, StepSuspend)) throw e; ctx._raiseStepFailure(e); @@ -2406,7 +2456,7 @@ export function task Promise>( })(); } return stepResult; - } else if (getEnv("WM_JOB_ID") && !getEnv("WM_FLOW_JOB_ID")) { + } else if (!insideTaskBody() && getEnv("WM_JOB_ID") && !getEnv("WM_FLOW_JOB_ID")) { // Inside a Windmill root job without checkpoint context — v1 HTTP dispatch // WM_FLOW_JOB_ID is set on child jobs, so we skip dispatch for those return (async () => { @@ -2456,7 +2506,8 @@ export function taskScript(path: string, options?: TaskOptions): (...args: any[] assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody(`taskScript("${path}")`); + const ctx = dispatchCtx(); if (ctx) { const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null ? args[0] @@ -2482,7 +2533,8 @@ export function taskFlow(path: string, options?: TaskOptions): (...args: any[]) assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody(`taskFlow("${path}")`); + const ctx = dispatchCtx(); if (ctx) { const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null ? args[0] @@ -2535,7 +2587,8 @@ export function waitForApproval(options?: { skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody("waitForApproval()"); + const ctx = dispatchCtx(); if (!ctx) { throw new Error("waitForApproval can only be called inside a workflow()"); } diff --git a/typescript-client/tests/workflow_nested_task.test.ts b/typescript-client/tests/workflow_nested_task.test.ts new file mode 100644 index 0000000000..39ac2185d8 --- /dev/null +++ b/typescript-client/tests/workflow_nested_task.test.ts @@ -0,0 +1,91 @@ +/** + * A task body runs as one child job, so the SDK calls it makes cannot be + * dispatched as steps of their own. Against the real client. + * + * Run with: bun test typescript-client/tests/workflow_nested_task.test.ts + */ +import { expect, test, describe, mock, afterEach } from "bun:test"; + +mock.module("../services.gen", () => ({ + ResourceService: {}, + VariableService: {}, + JobService: {}, + HelpersService: {}, + AppService: {}, + MetricsService: {}, + OidcService: {}, + UserService: {}, + KafkaTriggerService: {}, +})); +mock.module("../core/OpenAPI", () => ({ + OpenAPI: { BASE: "http://localhost:8000/api", TOKEN: "tok" }, +})); + +const { WorkflowCtx, StepSuspend, task, taskScript, step, setWorkflowCtx } = await import( + "../client.ts" +); + +let ran: string[] = []; + +const inner = task(async function inner(x: string) { + ran.push("inner"); + return `inner:${x}`; +}); + +const outer = task(async function outer(x: string) { + ran.push("outer"); + await Promise.resolve(); + const viaTask = await inner(x); + const viaStep = await step("s", () => "stepped"); + return `${viaTask}+${viaStep}`; +}); + +const dispatching = task(async function dispatching() { + return await taskScript("f/other/script")({}); +}); + +/** Run `body` as the child round executing `key`, and hand back what it raised. */ +async function childRound(key: string, body: () => Promise): Promise { + setWorkflowCtx(new WorkflowCtx({ completed_steps: {}, _executing_key: key })); + const hung = new Promise((r) => setTimeout(() => r("hung"), 500)); + return await Promise.race([body().then(() => "returned", (e) => e), hung]); +} + +afterEach(() => { + ran = []; + setWorkflowCtx(null); +}); + +describe("SDK calls inside a task body", () => { + test("a nested task and step run in the task's own job", async () => { + const raised = await childRound("outer", async () => { + await outer("a"); + }); + + expect(raised).toBeInstanceOf(StepSuspend); + expect(raised.dispatchInfo).toMatchObject({ + mode: "step_complete", + result: "inner:a+stepped", + }); + }); + + // The workflow body keeps replaying beside the task body, and its own calls + // must stay parked: running them would execute a sibling task in this job. + test("a sibling task of the workflow body is not run", async () => { + const raised = await childRound("outer", async () => { + await Promise.all([outer("a"), inner("sibling")]); + }); + + expect(raised.dispatchInfo).toMatchObject({ mode: "step_complete" }); + expect(ran).toEqual(["outer", "inner"]); + }); + + test("a step that would need a dispatch fails instead of hanging", async () => { + const raised = await childRound("dispatching", async () => { + await dispatching(); + }); + + expect(raised).toBeInstanceOf(Error); + expect(raised.message).toContain("cannot be called from inside a task"); + }); +}); From 7627c1be8d82c298173ad4a7002483d1caee6662 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 18:47:59 +0200 Subject: [PATCH 46/60] fix: fill app ctx inputs from the viewer's session, not the request (#11698) * fix: fill raw app ctx inputs on the server, not from the request Co-Authored-By: Claude Opus 5.5 * fix: read declared ctx inputs from the deployed app at run time Co-Authored-By: Claude Opus 5.5 * test: count the revocation's cache invalidation apart from the deploy's Co-Authored-By: Claude Opus 5.5 * refactor: drop the deploy-time cache eviction trigger, rely on the cache ttl Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/tests/app_ctx_inputs.rs | 88 ++++++++++++++++ backend/windmill-api/src/apps.rs | 167 ++++++++++++++++++++++--------- 2 files changed, 209 insertions(+), 46 deletions(-) create mode 100644 backend/tests/app_ctx_inputs.rs diff --git a/backend/tests/app_ctx_inputs.rs b/backend/tests/app_ctx_inputs.rs new file mode 100644 index 0000000000..cf386adc89 --- /dev/null +++ b/backend/tests/app_ctx_inputs.rs @@ -0,0 +1,88 @@ +//! A context input a raw app runnable declares (`fields. = { type: "ctx", .. }`) is +//! filled by the server from the viewer's session. Whatever the request sends under that +//! name is discarded: a runnable filters rows on it, so a viewer who could set it would +//! read someone else's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) +//! test-user-2 (developer, token SECRET_TOKEN_2) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const VIEWER_TOKEN: &str = "SECRET_TOKEN_2"; +const APP_PATH: &str = "u/test-user/ctx_app"; +const CONTENT: &str = "export function main(who: string) { return who }"; +// sha256 of CONTENT, the key an inline runnable is pinned under. +const CONTENT_SHA: &str = "6e198199621d57e4fed834510cb0c2692274aa4cbe976e1704fab2806be9d744"; + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn a_declared_ctx_input_ignores_the_value_the_viewer_sends( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + let client = reqwest::Client::new(); + + let resp = authed(client.post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "ctx inputs", + "value": { + "runnables": { + "whoami": { + "type": "inline", + "inlineScript": { "language": "bun", "content": CONTENT }, + "fields": { "who": { "type": "ctx", "ctx": "username" } } + } + } + }, + // The policy says nothing of context inputs: the server reads the + // declaration from the deployed runnable. + "policy": { + "execution_mode": "anonymous", + "triggerables_v2": { + format!("whoami:rawscript/{CONTENT_SHA}"): { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + + // Both spellings a viewer could send: a forged literal, and no argument at all. + for args in [json!({ "who": "test-user" }), json!({})] { + let resp = authed( + client.post(format!("{ws}/apps_u/execute_component/{APP_PATH}")), + VIEWER_TOKEN, + ) + .json(&json!({ + "component": "whoami", + "args": args, + "raw_code": { "language": "bun", "content": CONTENT } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let job_id: uuid::Uuid = resp.text().await?.parse()?; + + let queued: serde_json::Value = sqlx::query_scalar("SELECT args FROM v2_job WHERE id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + assert_eq!(queued["who"], json!("test-user-2"), "sent {args}"); + } + + Ok(()) +} diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 430aa801a0..2c2179d830 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -4393,10 +4393,55 @@ lazy_static! { /// change or deletion, which don't bump the version — see /// `invalidate_app_policy_cache` and `process_notify_event`); /// - the entry is within its TTL (backstop for a missed event or a restart). - static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = + static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = cache::Cache::new(1000); } +/// What `execute_component` runs a deployed app against. +struct DeployedPolicy { + policy: Policy, + /// Runnable id -> argument -> context property (`username`, `email`, `groups`, + /// `workspace`, `author`), for every `ctx` field the app's runnables declare. Read + /// from the deployed app rather than taken from the request, which only says what + /// the browser chose to send: a runnable filters rows on these. A deploy that + /// changes them under an unchanged policy evicts nothing, so a caller still naming + /// the replaced version keeps the previous declarations until the entry's TTL. + ctx_inputs: HashMap>, +} + +async fn deployed_ctx_inputs( + db: &DB, + w_id: &str, + path: &str, +) -> Result>> { + let runnables = sqlx::query_scalar::<_, Option>>( + "SELECT app_version.value -> 'runnables' FROM app + JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.path = $1 AND app.workspace_id = $2", + ) + .bind(path) + .bind(w_id) + .fetch_optional(db) + .await? + .flatten(); + let Some(serde_json::Value::Object(runnables)) = runnables.map(|r| r.0) else { + return Ok(HashMap::new()); + }; + Ok(runnables + .into_iter() + .filter_map(|(id, runnable)| { + let ctx_inputs: HashMap = runnable + .get("fields")? + .as_object()? + .iter() + .filter(|(_, f)| f.get("type").and_then(|t| t.as_str()) == Some("ctx")) + .filter_map(|(arg, f)| Some((arg.clone(), f.get("ctx")?.as_str()?.to_string()))) + .collect(); + (!ctx_inputs.is_empty()).then_some((id, ctx_inputs)) + }) + .collect()) +} + /// Backstop time-to-live for an [`APP_POLICY_CACHE`] entry — the maximum a policy /// change can go unreflected if its invalidation event is never consumed. const APP_POLICY_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(300); @@ -4449,7 +4494,7 @@ async fn execute_component( _ => {} }; - let arc_policy: Arc; + let arc_policy: Arc; let policy_triggerables_default = Default::default(); // Preview mode means the request was issued from the editor; the editing // user is already trusted by the policy check, so client-supplied `tag` @@ -4532,7 +4577,7 @@ async fn execute_component( // - The policy and triggerables are fetched from the database. // - In case of inline script, if an entry exists in the `app_script` table, push // an `AppScript` job payload, as in (.1) otherwise. - let (policy, policy_triggerables) = match payload { + let (policy, policy_triggerables, ctx_inputs) = match payload { // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), @@ -4551,6 +4596,7 @@ async fn execute_component( sensitive_inputs: force_viewer_sensitive_inputs.unwrap_or_default(), tag: None, }, + None, ), // 2. "run" mode. _ => { @@ -4580,9 +4626,12 @@ async fn execute_component( .map(|(_, p, _)| p); let policy = if let Some(p) = fresh { arc_policy = p; - &*arc_policy + &arc_policy.policy } else { - arc_policy = Arc::new(policy_fut.await?); + arc_policy = Arc::new(DeployedPolicy { + policy: policy_fut.await?, + ctx_inputs: deployed_ctx_inputs(&db, &w_id, path).await?, + }); APP_POLICY_CACHE.insert( cache_key, ( @@ -4591,7 +4640,7 @@ async fn execute_component( std::time::Instant::now(), ), ); - &*arc_policy + &arc_policy.policy }; // Caller-supplied inline code (`raw_code`), with or without an @@ -4653,7 +4702,11 @@ async fn execute_component( }) .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; - (policy, policy_triggerables) + ( + policy, + policy_triggerables, + arc_policy.ctx_inputs.get(&payload.component), + ) } }; @@ -4731,6 +4784,7 @@ async fn execute_component( let (mut args, job_id) = build_args( policy, policy_triggerables, + ctx_inputs, payload.args, opt_authed.as_ref(), &user_db, @@ -6161,6 +6215,54 @@ fn policy_granted_viewer(authed: &ApiAuthed) -> ApiAuthed { ApiAuthed { scopes: None, ..authed.clone() } } +/// The value of an app context property for this run. Everything about the viewer comes +/// from `authed`, the session the request authenticated with; an anonymous run has none +/// and gets `null`. +async fn ctx_value( + prop: &str, + policy: &Policy, + authed: Option<&ApiAuthed>, + w_id: &str, + db: &DB, +) -> Result> { + let value = match prop { + "username" => authed.map(|a| serde_json::to_value(a.display_username())), + "email" => authed.map(|a| serde_json::to_value(&a.email)), + "workspace" => Some(serde_json::to_value(w_id)), + "groups" => authed.map(|a| serde_json::to_value(&a.groups)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } + _ => { + return Err(Error::BadRequest(format!( + "context variable {} not allowed", + prop + ))) + } + }; + Ok(to_raw_value( + &value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { + Error::internal_err(format!("failed to serialize ctx variable {}: {}", prop, e)) + })?, + )) +} + async fn build_args( policy: &Policy, PolicyTriggerableInputs { @@ -6170,6 +6272,7 @@ async fn build_args( sensitive_inputs, .. }: &PolicyTriggerableInputs, + ctx_inputs: Option<&HashMap>, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -6179,6 +6282,16 @@ async fn build_args( let mut job_id: Option = None; let mut safe_args = HashMap::>::new(); + // Before anything reads `args`: a context input the runnable declares is never taken + // from the request, or a viewer could name someone else as themselves. + for (k, prop) in ctx_inputs.into_iter().flatten() { + args.remove(k); + safe_args.insert( + k.to_string(), + ctx_value(prop, policy, authed, w_id, db).await?, + ); + } + // tracing::error!("{:?}", allow_user_resources); for k in allow_user_resources.iter() { if let Some(arg_val) = args.get(k) { @@ -6270,47 +6383,9 @@ async fn build_args( if arg_str.starts_with("\"$ctx:") { let prop = arg_str.trim_start_matches("\"$ctx:").trim_end_matches("\""); - let value = match prop { - "username" => authed - .as_ref() - .map(|a| serde_json::to_value(a.display_username())), - "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), - "workspace" => Some(serde_json::to_value(&w_id)), - "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. - "author" => { - let author = match ( - policy.on_behalf_of_email.as_deref(), - policy.on_behalf_of.as_deref(), - ) { - (Some(email), _) => Some(email.to_string()), - (None, Some(permissioned_as)) => Some( - windmill_common::users::get_email_from_permissioned_as( - permissioned_as, - w_id, - db, - ) - .await?, - ), - (None, None) => None, - }; - Some(serde_json::to_value(&author)) - } - _ => { - return Err(Error::BadRequest(format!( - "context variable {} not allowed", - prop - ))) - } - }; safe_args.insert( k.to_string(), - to_raw_value(&value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { - Error::internal_err(format!( - "failed to serialize ctx variable for {}: {}", - k, e - )) - })?), + ctx_value(prop, policy, authed, w_id, db).await?, ); } else if !arg_str.contains("\"$var:") && !arg_str.contains("\"$res:") { safe_args.insert(k.to_string(), v); From a92b29dae6950d29c2b8970e07d055fa02ff979e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 19:07:23 +0200 Subject: [PATCH 47/60] fix: load the s3 explorer table lazily to keep monaco out of the app shell (#11701) Co-authored-by: Claude Opus 5.5 --- .../lib/components/S3FilePickerInner.svelte | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 5f8dcfc3d7..89c23e84ae 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -26,7 +26,6 @@ import Toggle from './Toggle.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' - import DBTable from './DBTable.svelte' import type { IDbTableOps } from './dbOps' import { resource } from 'runed' import { staticDbTableOps } from './staticDbTableOps' @@ -1160,6 +1159,12 @@ }) +{#snippet tableLoading()} +
+ Loading table... +
+{/snippet} + {#if workspaceSettingsInitialized === false}
{#if fromWorkspaceSettings} @@ -1522,17 +1527,20 @@ {#if showTable}
{#if currentTable?.ops} - {#key currentTable.ops} - - {/key} + + {#await import('./DBTable.svelte')} + {@render tableLoading()} + {:then DBTable} + {#key currentTable.ops} + + {/key} + {/await} {:else if tableOps.error}
Could not read this file as a table: {tableOps.error.message}
{:else} -
- Loading table... -
+ {@render tableLoading()} {/if}
{:else} From 14519884e4c9aa4d0a006cbc019a776f9a038967 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 19:28:05 +0200 Subject: [PATCH 48/60] feat: support array arguments in snowflake scripts (#11702) * feat: support array arguments in snowflake scripts Co-Authored-By: Claude Opus 5.5 * docs: cast the parsed snowflake list to array in the guidance Co-Authored-By: Claude Opus 5.5 * fix: parse a json array default of a sql list argument as a list Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../parsers/windmill-parser-sql/src/lib.rs | 45 ++++++++++++++++++- .../windmill-worker/src/snowflake_executor.rs | 16 ++++++- cli/src/guidance/skills.gen.ts | 21 +++++++++ system_prompts/auto-generated/prompts.ts | 21 +++++++++ system_prompts/auto-generated/script.md | 21 +++++++++ .../skills/write-script-snowflake/SKILL.md | 21 +++++++++ system_prompts/languages/snowflake.md | 21 +++++++++ 7 files changed, 164 insertions(+), 2 deletions(-) diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 10929e409a..341906a3e3 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -235,7 +235,7 @@ lazy_static::lazy_static! { // -- $name (type) = default static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \(([A-Za-z0-9_\[\]]+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); - static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -254,6 +254,12 @@ fn parsed_default(parsed_typ: &Typ, default: String) -> Option= 2 && default.starts_with("'") && default.ends_with("'") => { Some(json!(&default[1..default.len() - 1])) } + Typ::List(_) => Some( + serde_json::from_str::(&default) + .ok() + .filter(|v| v.is_array()) + .unwrap_or_else(|| json!(default)), + ), _ => Some(json!(default)), } } @@ -1039,7 +1045,11 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ { } pub fn parse_snowflake_typ(typ: &str) -> Typ { + if let Some(base_typ) = typ.strip_suffix("[]") { + return Typ::List(Box::new(parse_snowflake_typ(base_typ))); + } match typ { + "array" => Typ::List(Box::new(Typ::Unknown)), "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None), "binary" | "varbinary" => Typ::Bytes, "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz" @@ -1591,6 +1601,39 @@ SELECT ?; Ok(()) } + #[test] + fn test_parse_snowflake_array_types() -> anyhow::Result<()> { + let code = r#" +-- ? groups (varchar[]) +-- ? ids (int[]) = [1, 2] +-- ? anything (ARRAY) +SELECT 1; +"#; + let args = parse_snowflake_sig(code)?.args; + assert_eq!(args[1].default, Some(json!([1, 2]))); + let got = args + .iter() + .map(|a| (a.name.as_str(), a.otyp.as_deref(), &a.typ)) + .collect::>(); + assert_eq!( + got, + vec![ + ( + "groups", + Some("varchar[]"), + &Typ::List(Box::new(Typ::Str(None))) + ), + ("ids", Some("int[]"), &Typ::List(Box::new(Typ::Int))), + ( + "anything", + Some("array"), + &Typ::List(Box::new(Typ::Unknown)) + ), + ] + ); + Ok(()) + } + #[test] fn test_parse_mssql_sig() -> anyhow::Result<()> { let code = r#" diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 24b07eacb9..411dd520b9 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -873,7 +873,8 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value { let mut v = arg_v; if !v.is_string() { - // if not string, convert to string for api request + // The SQL API has no ARRAY/VARIANT binding type: a list or object + // is bound as its JSON text, which the query reads with PARSE_JSON(?). v = json!(v.to_string()); } @@ -935,3 +936,16 @@ fn parse_val(value: &Value, typ: &str) -> Value { )) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn list_arg_binds_as_json_text() { + assert_eq!( + convert_typ_val("varchar[]".to_string(), json!(["sales", "g/ops"])), + json!({ "type": "TEXT", "value": r#"["sales","g/ops"]"# }) + ); + } +} diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index fff7df1153..e08db46f92 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -5625,6 +5625,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` +## Array parameters + +Declare a list with a \`[]\` suffix on the element type (\`(varchar[])\`, +\`(int[])\`), or \`(array)\` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +\`PARSE_JSON(?)\`, and cast to \`ARRAY\` where a function expects one. + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +\`\`\` + +Or join on the elements with \`FLATTEN\`: + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +\`\`\` + ## Receiving an S3Object as a script parameter Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 393b724f6b..83513de11a 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -5815,6 +5815,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` +## Array parameters + +Declare a list with a \`[]\` suffix on the element type (\`(varchar[])\`, +\`(int[])\`), or \`(array)\` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +\`PARSE_JSON(?)\`, and cast to \`ARRAY\` where a function expects one. + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +\`\`\` + +Or join on the elements with \`FLATTEN\`: + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +\`\`\` + ## Receiving an S3Object as a script parameter Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index a4064c3d9e..d178ddb6c0 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1409,6 +1409,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for diff --git a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md index 8c438e7038..cd79a1f5d4 100644 --- a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md @@ -84,6 +84,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for diff --git a/system_prompts/languages/snowflake.md b/system_prompts/languages/snowflake.md index 8ded76eb3c..b7798e3461 100644 --- a/system_prompts/languages/snowflake.md +++ b/system_prompts/languages/snowflake.md @@ -10,6 +10,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for From 26afeecbd5b0a11c084813182e92805a15d21bbb Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 19:28:38 +0200 Subject: [PATCH 49/60] feat(cli): declare app sandbox and frontend api scopes in app yaml (#11703) Co-authored-by: Claude Opus 5.5 --- cli/src/commands/app/app.ts | 130 +++++++++++++++++- cli/src/commands/app/lint.ts | 7 + cli/src/commands/app/raw_apps.ts | 21 ++- cli/src/commands/sync/sync.ts | 4 +- cli/src/guidance/skills.gen.ts | 12 ++ cli/test/app_isolation_unit.test.ts | 60 ++++++++ system_prompts/auto-generated/prompts.ts | 4 + .../auto-generated/skills/raw-app/SKILL.md | 12 ++ system_prompts/base/raw-app.md | 14 ++ 9 files changed, 254 insertions(+), 10 deletions(-) create mode 100644 cli/test/app_isolation_unit.test.ts diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0d4ad634c8..470f0f4b11 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -32,6 +32,8 @@ export interface AppFile { guests?: boolean; value: any; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; // Mirrors granular ACLs on the app path. Omitted from app.yaml when no @@ -160,6 +162,100 @@ export function executionModeForPush( } return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; } +/** Mirrors `FRONTEND_SDK_ALLOWED_SCOPES` in the backend's `apps.rs`, which stays + * the authority: this copy only fails a push before it bundles. */ +export const FRONTEND_API_SCOPES = [ + "jobs:run", + "jobs:read", + "users:read", + "resources:read", + "variables:read", + "flow_conversations:read", + "flow_conversations:write", +]; + +export type AppIsolation = { sandbox: boolean; frontendApi: string[] }; + +/** Write `sandbox` and `frontend_api` on a pulled app, only when they are on, so + * an app using neither keeps the file it had. Scopes of an unsandboxed app are + * inert and stay out: a file stating them without `sandbox` does not push. */ +export function markIsolationFromPolicy(app: any, rawApp: boolean) { + if (app?.["policy"]?.["sandbox"] !== true) { + return; + } + app.sandbox = true; + const scopes = app.policy.frontend_sdk_scopes; + if (rawApp && Array.isArray(scopes) && scopes.length > 0) { + app.frontend_api = scopes; + } +} + +/** The isolation the tracked file states, or `undefined` when it has no `sandbox` + * key and the deployed policy carries over. A file that has the key states both + * fields: `sandbox: false` turns isolation off, and a missing `frontend_api` + * means no scopes. */ +export function statedIsolation( + localApp: any, + rawApp: boolean, +): AppIsolation | undefined { + const sandbox = localApp?.["sandbox"]; + const scopes = localApp?.["frontend_api"] ?? []; + if (sandbox !== undefined && typeof sandbox !== "boolean") { + throw new Error("'sandbox' must be true or false"); + } + if ( + !Array.isArray(scopes) || + scopes.some((s: unknown) => typeof s !== "string") + ) { + throw new Error("'frontend_api' must be a list of scopes"); + } + if (scopes.length > 0 && !rawApp) { + throw new Error("'frontend_api' is only available to full-code apps"); + } + const seen = new Set(); + for (const scope of scopes) { + if (!FRONTEND_API_SCOPES.includes(scope)) { + throw new Error( + `Invalid frontend_api scope '${scope}'. Allowed scopes: ${FRONTEND_API_SCOPES.join(", ")}`, + ); + } + if (seen.has(scope)) { + throw new Error(`Duplicate frontend_api scope '${scope}'`); + } + seen.add(scope); + } + if (scopes.length > 0 && sandbox !== true) { + throw new Error( + "'frontend_api' requires 'sandbox: true': an unsandboxed app gets no frontend API token", + ); + } + return sandbox === undefined ? undefined : { sandbox, frontendApi: scopes }; +} + +/** Whether the deployed policy already has the stated isolation. The file's two + * keys are kept out of the superset comparison against the deployed app, which + * could not see a key the file dropped. */ +export function isolationUpToDate( + isolation: AppIsolation | undefined, + deployedPolicy: Policy | undefined, + rawApp: boolean, +): boolean { + if (!isolation) { + return true; + } + if ((deployedPolicy?.sandbox === true) !== isolation.sandbox) { + return false; + } + if (!rawApp) { + return true; + } + const deployed = deployedPolicy?.frontend_sdk_scopes ?? []; + return ( + deployed.length === isolation.frontendApi.length && + deployed.every((s, i) => s === isolation.frontendApi[i]) + ); +} + export async function pushApp( workspace: string, remotePath: string, @@ -198,6 +294,7 @@ export async function pushApp( } const path = localPath + "app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; + const isolation = statedIsolation(localApp, false); replaceInlineScripts(localApp.value, localPath, true); // On create the backend applies folder defaults, so there is nothing to preserve. @@ -210,17 +307,26 @@ export async function pushApp( localApp, remotePath, executionModeForPush(localApp, deployedPolicy), - basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of), + isolation ); // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). - const { extra_perms: localPerms, ...localAppBody } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppBody + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { - if (isSuperset(localAppBody, app)) { + if ( + isSuperset(localAppBody, app) && + isolationUpToDate(isolation, deployedPolicy, false) + ) { log.info(colors.green(`App ${remotePath} is up to date`)); } else { log.info(colors.bold.yellow(`Updating app ${remotePath}...`)); @@ -269,12 +375,13 @@ export async function generatingPolicy( app: any, path: string, executionMode: AppExecutionMode, - base: Policy | undefined + base: Policy | undefined, + isolation?: AppIsolation ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updatePolicy(app.value, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -332,10 +439,21 @@ export function preserveOnBehalfOfFields( * deployed app would keep being able to run runnables this push removed. */ export function finalizeDerivedPolicy( policy: Policy, - executionMode: AppExecutionMode + executionMode: AppExecutionMode, + isolation?: AppIsolation, + rawApp: boolean = false ) { policy.triggerables = undefined; policy.execution_mode = executionMode; + if (isolation) { + // `false` is sent rather than left out: the server reads an omitted value as + // "no opinion" and may sandbox the app anyway, hiding that the file was refused. + policy.sandbox = isolation.sandbox; + if (rawApp) { + policy.frontend_sdk_scopes = + isolation.frontendApi.length > 0 ? isolation.frontendApi : undefined; + } + } } async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 2276a77cad..36f3187cd3 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -9,6 +9,7 @@ import { GlobalOptions } from "../../types.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; import { loadRunnablesFromBackend } from "./raw_apps.ts"; +import { statedIsolation } from "./app.ts"; import { getFolderSuffix, hasFolderSuffix, @@ -42,6 +43,12 @@ function validateRawAppYaml(appData: any): { errors.push("Field 'summary' must be a string"); } + try { + statedIsolation(appData, true); + } catch (error: any) { + errors.push(error.message); + } + // Note: 'runnables' is no longer required in raw_app.yaml // Runnables can be stored in separate files in the backend folder diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 16179d4662..4c3d8d5215 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -20,13 +20,16 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { type AppExecutionMode, + type AppIsolation, basePolicy, executionModeForPush, finalizeDerivedPolicy, + isolationUpToDate, markAccessFromPolicy, preserveOnBehalfOfFields, replaceInlineScripts, repopulateFields, + statedIsolation, } from "./app.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; @@ -49,6 +52,8 @@ export interface AppFile { runnables?: any; custom_path?: string; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; data?: { @@ -402,6 +407,7 @@ export async function pushRawApp( } const appFilePath = localPath + "raw_app.yaml"; const localApp = (await yamlParseFile(appFilePath)) as AppFile; + const isolation = statedIsolation(localApp, true); // Load runnables from separate YAML files in the backend folder // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) @@ -454,6 +460,7 @@ export async function pushRawApp( deployedPolicy, !!preserveFields.preserve_on_behalf_of, ), + isolation, ); const files = await collectAppFiles(localPath); @@ -485,14 +492,21 @@ export async function pushRawApp( // to `.raw_app.json`, not `.app.json`. The backend granular_acls // handler routes "raw_app" to the `app` table (where v2 raw apps actually // live) while still dispatching DeployedObject::RawApp for git-sync. - const { extra_perms: localPerms, ...localAppNoPerms } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppNoPerms + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { // Check both metadata/runnables AND files for changes // Files need separate comparison because isSuperset only checks if local keys exist in remote - const metadataUpToDate = isSuperset({ ...localAppNoPerms, runnables }, app); + const metadataUpToDate = + isSuperset({ ...localAppNoPerms, runnables }, app) && + isolationUpToDate(isolation, deployedPolicy, true); const filesUpToDate = deepEqual(files, app.value?.files); if (metadataUpToDate && filesUpToDate) { log.info(colors.green(`App ${remotePath} is up to date`)); @@ -560,11 +574,12 @@ export async function generatingPolicy( path: string, executionMode: AppExecutionMode, base: Policy | undefined, + isolation?: AppIsolation, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation, true); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 54887028f0..b9e373abad 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -149,7 +149,7 @@ import { extractCurrentMapping, } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; -import { markAccessFromPolicy } from "../app/app.ts"; +import { markAccessFromPolicy, markIsolationFromPolicy } from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, @@ -1359,6 +1359,7 @@ export function ZipFSElement( } markAccessFromPolicy(app); + markIsolationFromPolicy(app, false); app.policy = undefined; yield { isDirectory: false, @@ -1377,6 +1378,7 @@ export function ZipFSElement( throw error; } markAccessFromPolicy(rawApp); + markIsolationFromPolicy(rawApp, true); // console.log("rawApp", rawApp); rawApp.policy = undefined; // custom_path is derived from the file path, don't store it diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e08db46f92..8491e837e3 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6840,6 +6840,18 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. \`public: true\` in \`raw_app.yaml\` deploys the app for anonymous users, and \`guests: true\` for guests. Remove the line and the next push closes the app again. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are \`jobs:run\`, \`jobs:read\`, \`users:read\`, \`resources:read\`, \`variables:read\`, \`flow_conversations:read\` and \`flow_conversations:write\`; grant only the ones the frontend calls. +In \`raw_app.yaml\`: + +\`\`\`yaml +sandbox: true +frontend_api: [jobs:run, users:read] +\`\`\` + +A file with a \`sandbox\` key states both settings: \`sandbox: false\` turns isolation off, and a missing \`frontend_api\` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing \`sandbox: true\` does not unsandbox it. \`frontend_api\` needs \`sandbox: true\`; the push fails otherwise. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. diff --git a/cli/test/app_isolation_unit.test.ts b/cli/test/app_isolation_unit.test.ts new file mode 100644 index 0000000000..201c4a52a4 --- /dev/null +++ b/cli/test/app_isolation_unit.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { + finalizeDerivedPolicy, + isolationUpToDate, + markIsolationFromPolicy, + statedIsolation, +} from "../src/commands/app/app.ts"; + +function pushed(file: any, deployed: any, rawApp = true) { + const policy = { ...deployed }; + finalizeDerivedPolicy(policy, "publisher", statedIsolation(file, rawApp), rawApp); + return policy; +} + +test("a pull writes isolation only when it is on, and a push deploys what was pulled", () => { + const off: any = { policy: { sandbox: false, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(off, true); + expect("sandbox" in off).toBe(false); + expect("frontend_api" in off).toBe(false); + + const on: any = { policy: { sandbox: true, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(on, true); + expect(on.sandbox).toBe(true); + expect(on.frontend_api).toEqual(["jobs:run"]); + // Into a workspace where the app does not exist yet. + const fresh = pushed(on, undefined); + expect(fresh.sandbox).toBe(true); + expect(fresh.frontend_sdk_scopes).toEqual(["jobs:run"]); +}); + +test("the file wins when it states sandbox, the deployed policy carries over when it does not", () => { + const deployed = { sandbox: true, frontend_sdk_scopes: ["jobs:run"] }; + + const silent = pushed({}, deployed); + expect(silent.sandbox).toBe(true); + expect(silent.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(isolationUpToDate(statedIsolation({}, true), deployed, true)).toBe(true); + + // A file with the key states the scopes too: none here. + const narrowed = pushed({ sandbox: true }, deployed); + expect(narrowed.sandbox).toBe(true); + expect(narrowed.frontend_sdk_scopes).toBeUndefined(); + expect(isolationUpToDate(statedIsolation({ sandbox: true }, true), deployed, true)).toBe(false); + + const opened = pushed({ sandbox: false }, deployed); + expect(opened.sandbox).toBe(false); + expect(opened.frontend_sdk_scopes).toBeUndefined(); +}); + +test("scopes are refused without sandbox, outside the allowlist, or repeated", () => { + expect(() => statedIsolation({ frontend_api: ["jobs:run"] }, true)).toThrow("requires 'sandbox: true'"); + expect(() => statedIsolation({ sandbox: false, frontend_api: ["jobs:run"] }, true)).toThrow( + "requires 'sandbox: true'", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["apps:write"] }, true)).toThrow("Invalid"); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run", "jobs:run"] }, true)).toThrow( + "Duplicate", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run"] }, false)).toThrow("full-code"); +}); diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 83513de11a..1c6541e155 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1116,6 +1116,10 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. When a deploy widens who may open the app, tell the user in plain words. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are \`jobs:run\`, \`jobs:read\`, \`users:read\`, \`resources:read\`, \`variables:read\`, \`flow_conversations:read\` and \`flow_conversations:write\`; grant only the ones the frontend calls. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 92bfe68aab..8d7788dd94 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -471,6 +471,18 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. `public: true` in `raw_app.yaml` deploys the app for anonymous users, and `guests: true` for guests. Remove the line and the next push closes the app again. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are `jobs:run`, `jobs:read`, `users:read`, `resources:read`, `variables:read`, `flow_conversations:read` and `flow_conversations:write`; grant only the ones the frontend calls. +In `raw_app.yaml`: + +```yaml +sandbox: true +frontend_api: [jobs:run, users:read] +``` + +A file with a `sandbox` key states both settings: `sandbox: false` turns isolation off, and a missing `frontend_api` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing `sandbox: true` does not unsandbox it. `frontend_api` needs `sandbox: true`; the push fails otherwise. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index 3d8e0d3e86..dea8ebd21e 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -239,6 +239,20 @@ Either one lets those people run the app's backend runnables, so never open an a When a deploy widens who may open the app, tell the user in plain words. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are `jobs:run`, `jobs:read`, `users:read`, `resources:read`, `variables:read`, `flow_conversations:read` and `flow_conversations:write`; grant only the ones the frontend calls. + +In `raw_app.yaml`: + +```yaml +sandbox: true +frontend_api: [jobs:run, users:read] +``` + +A file with a `sandbox` key states both settings: `sandbox: false` turns isolation off, and a missing `frontend_api` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing `sandbox: true` does not unsandbox it. `frontend_api` needs `sandbox: true`; the push fails otherwise. + + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. From 36ea2da0bfaa5ebd3ee2d63eae2395f6f54172c9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 19:30:29 +0200 Subject: [PATCH 50/60] warn when a raw app sql viewer-identity input is not a ctx field (#11704) Co-authored-by: Claude Opus 5.5 --- cli/src/commands/app/lint.ts | 46 +++++++++++++++++++ .../app_lint_viewer_identity_unit.test.ts | 24 ++++++++++ .../RawAppInlineScriptRunnable.svelte | 9 ++++ .../raw_apps/RawAppInputsSpecEditor.svelte | 11 ++++- frontend/src/lib/infer.ts | 2 +- 5 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 cli/test/app_lint_viewer_identity_unit.test.ts diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 36f3187cd3..c90a22105d 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -9,6 +9,7 @@ import { GlobalOptions } from "../../types.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; import { loadRunnablesFromBackend } from "./raw_apps.ts"; +import { inferSchema } from "../../utils/metadata.ts"; import { statedIsolation } from "./app.ts"; import { getFolderSuffix, @@ -55,6 +56,50 @@ function validateRawAppYaml(appData: any): { return { errors, warnings }; } +const SQL_LANGUAGES = new Set([ + "postgresql", + "mysql", + "bigquery", + "snowflake", + "mssql", + "oracledb", + "duckdb", +]); + +// The viewer properties a `ctx` field can supply. The Inputs tab of the app +// editor hints on the same names (RawAppInlineScriptRunnable.svelte). +const VIEWER_IDENTITY_ARGS = ["username", "email", "groups"]; + +/** + * A SQL runnable argument named after a viewer property is most often a + * per-viewer filter. Left as a plain input it takes whatever the browser + * sends, so it has to be a `ctx` field for the server to fill it. + */ +export async function viewerIdentityInputWarnings( + runnables: Record, +): Promise { + const warnings: string[] = []; + for (const [id, runnable] of Object.entries(runnables)) { + const { language, content } = runnable?.inlineScript ?? {}; + if (!SQL_LANGUAGES.has(language) || typeof content !== "string") { + continue; + } + const { schema } = await inferSchema(language, content, {}, id); + for (const arg of VIEWER_IDENTITY_ARGS) { + const field = runnable.fields?.[arg]; + const serverFilled = field?.ctx !== undefined || + field?.value !== undefined; + if (arg in (schema?.properties ?? {}) && !serverFilled) { + warnings.push( + `Runnable '${id}': argument '${arg}' is a frontend input, so a viewer can send any value. ` + + `If it identifies the viewer, set 'fields: { ${arg}: { ctx: ${arg} } }' in backend/${id}.yaml (next to 'type: inline') so the server fills it.`, + ); + } + } + } + return warnings; +} + /** * Validates that runnables exist either in backend/*.yaml files or in raw_app.yaml */ @@ -84,6 +129,7 @@ async function validateRunnables( " - Or a 'runnables' field in raw_app.yaml (legacy format)", ); } else if (hasBackendRunnables) { + warnings.push(...await viewerIdentityInputWarnings(runnablesFromBackend)); log.info( colors.gray( ` Found ${ diff --git a/cli/test/app_lint_viewer_identity_unit.test.ts b/cli/test/app_lint_viewer_identity_unit.test.ts new file mode 100644 index 0000000000..f328ffe4e2 --- /dev/null +++ b/cli/test/app_lint_viewer_identity_unit.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "bun:test"; +// app.ts first: it is the entry the CLI loads lint.ts through, and loading +// lint.ts ahead of it trips their import cycle. +import "../src/commands/app/app.ts"; +import { viewerIdentityInputWarnings } from "../src/commands/app/lint.ts"; + +const sql = (content: string, fields?: Record) => ({ + type: "inline", + inlineScript: { language: "duckdb", content }, + ...(fields ? { fields } : {}), +}); + +test("app lint warns on a SQL viewer-identity argument the server does not fill", async () => { + const content = + "-- $username (text)\n-- $limit (int)\nSELECT * FROM orders WHERE owner = $username LIMIT $limit"; + + const warnings = await viewerIdentityInputWarnings({ + open: sql(content), + filled: sql(content, { username: { ctx: "username" } }), + }); + + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("Runnable 'open': argument 'username'"); +}); diff --git a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte index 200e76d85b..ddbdae5bf3 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInlineScriptRunnable.svelte @@ -15,6 +15,7 @@ import { Pane, Splitpanes } from 'svelte-splitpanes' import Tabs from '../common/tabs/Tabs.svelte' import { Tab } from '../common' + import { SQL_LANGUAGES } from '$lib/infer' import RawAppInputsSpecEditor from './RawAppInputsSpecEditor.svelte' import SplitPanesWrapper from '../splitPanes/SplitPanesWrapper.svelte' import SchemaForm from '../SchemaForm.svelte' @@ -80,6 +81,13 @@ } } + // A SQL argument named after a viewer property is most often a per-viewer filter, + // which only a ctx field protects. `wmill app lint` warns on the same names. + const VIEWER_IDENTITY_ARGS = ['username', 'email', 'groups'] + let isSql = $derived( + isRunnableByName(runnable) && SQL_LANGUAGES.includes(runnable.inlineScript?.language ?? '') + ) + let selectedTab = $state('test') let args = $state({}) let hubFlowPreview: OpenFlow | undefined = $state(undefined) @@ -266,6 +274,7 @@ documentationLink={meta?.['documentationLink']} allowTypeChange={meta?.['allowTypeChange']} displayType + suggestCtx={isSql && VIEWER_IDENTITY_ARGS.includes(k)} /> {/each}
diff --git a/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte index 67ead38501..859a693ca9 100644 --- a/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppInputsSpecEditor.svelte @@ -13,6 +13,7 @@ import { fieldTypeToTsType } from '../apps/utils' import type { InputType } from '../apps/inputType' import Select from '$lib/components/select/Select.svelte' + import { Alert } from '$lib/components/common' import { userStore } from '$lib/stores' import { useOperatingUser, @@ -67,6 +68,8 @@ allowTypeChange?: boolean loading?: boolean documentationLink?: string | undefined + /** Warn that this input, left as a frontend input, is open to any value a viewer sends. */ + suggestCtx?: boolean } let { @@ -85,7 +88,8 @@ displayType = false, allowTypeChange = true, loading = false, - documentationLink = undefined + documentationLink = undefined, + suggestCtx = false }: Props = $props() run(() => { @@ -193,6 +197,11 @@
{:else if componentInput?.type === 'user' || componentInput?.type == undefined} Field's value is a frontend input + {#if suggestCtx} + + If this identifies the viewer, switch it to context so the server fills it. + + {/if} {/if} {#if componentInput?.type === 'user' && ((fieldType == 'object' && format?.startsWith('resource-') && format !== 'resource-s3_object') || fieldType == 'resource')}
diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 2eb15f3c34..9e9cddeb00 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -301,7 +301,7 @@ export async function inferAnsibleExecutionMode(code: string): Promise { } } -const SQL_LANGUAGES = [ +export const SQL_LANGUAGES = [ 'postgresql', 'mysql', 'bigquery', From 05c7b688e5e963c6c313cb5da8e777cee11cd122 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:31:40 +0200 Subject: [PATCH 51/60] fix: stop session dom tools reading a raw app preview that is not current (#11700) * fix: wait for the raw app build before reading its preview dom * fix: warn the dom and screenshot tools when the raw app build failed * fix: tell the dom and screenshot tools when the raw app awaits sdk consent * fix: tell the dom and screenshot tools while the raw app sdk token loads --- .../components/raw_apps/RawAppEditor.svelte | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 46c23b79b3..ca23505ff1 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -21,6 +21,7 @@ genWmillTs, normalizeRawAppRuntimeLogs, createRawAppBuildTracker, + RAW_APP_BUILD_WAIT_MS, type Runnable, type RawAppRuntimeLogEntry, type RawAppRuntimeLogRequester, @@ -488,6 +489,17 @@ addEventListener('message', function (e) { // Latest UI Builder error; cleared on next successful build. let buildError = $state(undefined) const buildTracker = createRawAppBuildTracker() + // The preview shell can be ready before the UI Builder has the files, so a fresh + // editor counts as building: reads made then would see an empty page. + buildTracker.start() + const BUILD_PENDING_TEXT = `The app is still building (or its build never started, e.g. no index entrypoint file): no build result arrived within ${RAW_APP_BUILD_WAIT_MS / 1000} seconds, so the preview shows an older build or nothing yet. Call get_app_runtime_logs to check the build, then retry.` + // A failed build never reaches the preview, which keeps the last good one, if any. + const buildFailedText = (error: string) => + `The latest build failed, so the preview does not include the latest edits (it shows the last build that succeeded, or nothing if none has). Fix the build first. Build error:\n${error}` + const SDK_CONSENT_PENDING_TEXT = + 'The preview is waiting for the user to approve the permissions the app declares in policy.frontend_sdk_scopes, so the app code is not running and the page is empty. This is not a bug in the app: do not change the code. Ask the user to approve the permissions in the preview, then retry.' + const SDK_STARTING_TEXT = + 'The preview is still loading the permissions the app declares in policy.frontend_sdk_scopes, so the app code has not started yet. Retry in a moment.' const PREVIEW_SETTLE_MS = 1000 let editorDestroyed = false // Latest uncaught runtime error thrown by the rendered app; cleared on next build. @@ -1976,6 +1988,18 @@ addEventListener('message', function (e) { // Live DOM inspection for the session chat. The element is re-read on every call, // so the model always sees the current render. const requestDomQuery: RawAppDomRequester = async (query: RawAppDomQuery) => { + await buildTracker.wait() + if (editorDestroyed) return undefined + if (buildTracker.pending) return { text: BUILD_PENDING_TEXT } + if (sdkPrompt) return { text: SDK_CONSENT_PENDING_TEXT } + if (previewSdkPending) return { text: SDK_STARTING_TEXT } + const res = await readPreviewDom(query) + return res && buildError !== undefined + ? { text: `${buildFailedText(buildError)}\n\n${res.text}` } + : res + } + + const readPreviewDom: RawAppDomRequester = async (query: RawAppDomQuery) => { const selector = query.selector?.trim() const res = await requestFromPreview('domRead', { selector }) if (!res) return undefined @@ -2142,6 +2166,12 @@ addEventListener('message', function (e) { // reconstructs the DOM into an SVG foreignObject, and a WebGL canvas is only // captured when its context was created with preserveDrawingBuffer. const captureScreenshot: RawAppScreenshotRequester = async () => { + await buildTracker.wait() + if (editorDestroyed) throw new Error('The app preview was closed') + if (buildTracker.pending) throw new Error(BUILD_PENDING_TEXT) + if (sdkPrompt) throw new Error(SDK_CONSENT_PENDING_TEXT) + if (previewSdkPending) throw new Error(SDK_STARTING_TEXT) + if (buildError !== undefined) throw new Error(buildFailedText(buildError)) const body = await requestFromPreview('elementRect', { selector: 'body' }) if (!body?.rect) throw new Error('App preview is not ready') const { width, height } = body.rect From c6692ace156f5295196af07015ad82aee6391116 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:32:10 +0200 Subject: [PATCH 52/60] fix: keep a custom error handler's slack argument on save (#11696) Co-authored-by: Claude Opus 5.5 (1M context) --- .../components/ErrorOrRecoveryHandler.svelte | 13 +++++++----- .../lib/components/handlerSlackArg.test.ts | 21 +++++++++++++++++++ .../src/lib/components/handlerSlackArg.ts | 17 +++++++++++++++ .../(logged)/workspace_settings/+page.svelte | 17 ++++++++++----- 4 files changed, 58 insertions(+), 10 deletions(-) create mode 100644 frontend/src/lib/components/handlerSlackArg.test.ts create mode 100644 frontend/src/lib/components/handlerSlackArg.ts diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index aab564a4cf..c624b5a148 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -69,6 +69,8 @@ } from 'lucide-svelte' import { hubPaths } from '$lib/hub' import { isCloudHosted } from '$lib/cloud' + import { untrack } from 'svelte' + import { syncSlackBotArg } from './handlerSlackArg' import SmtpConfigurationStatus from './common/smtp/SmtpConfigurationStatus.svelte' import { SettingService } from '$lib/gen' import { isSmtpSettingsValid } from './instanceSettings/SmtpSettings.svelte' @@ -316,12 +318,13 @@ loadSmtpConfiguration() }) + let builtinArgs: Record | undefined $effect(() => { - if (handlerSelected === 'slack' && isSlackHandler(handlerPath)) { - handlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - handlerExtraArgs['slack'] = undefined - } + const isSlackBuiltin = handlerSelected === 'slack' && isSlackHandler(handlerPath) + const args = handlerExtraArgs + // A replaced args object (Discard, reload) holds the stored config, not what the built-in set. + untrack(() => syncSlackBotArg(args, isSlackBuiltin, args === builtinArgs)) + builtinArgs = isSlackBuiltin ? args : undefined }) let lastHandlerSelected: ErrorHandler | undefined = $state(undefined) diff --git a/frontend/src/lib/components/handlerSlackArg.test.ts b/frontend/src/lib/components/handlerSlackArg.test.ts new file mode 100644 index 0000000000..b087a55f26 --- /dev/null +++ b/frontend/src/lib/components/handlerSlackArg.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest' +import { syncSlackBotArg } from './handlerSlackArg' + +const AUTO = '$res:f/slack_bot/bot_token' + +describe('syncSlackBotArg', () => { + it("leaves a custom handler's slack value alone", () => { + const args = { slack: '$res:u/me/my_slack' } + syncSlackBotArg(args, false, false) + syncSlackBotArg(args, false, true) + expect(args).toEqual({ slack: '$res:u/me/my_slack' }) + }) + + it('sets the bot token for the built-in and drops it when leaving', () => { + const args: Record = {} + syncSlackBotArg(args, true, false) + expect(args.slack).toBe(AUTO) + syncSlackBotArg(args, false, true) + expect(args).toStrictEqual({}) + }) +}) diff --git a/frontend/src/lib/components/handlerSlackArg.ts b/frontend/src/lib/components/handlerSlackArg.ts new file mode 100644 index 0000000000..6b8e4d6f5c --- /dev/null +++ b/frontend/src/lib/components/handlerSlackArg.ts @@ -0,0 +1,17 @@ +import { WORKSPACE_SLACK_BOT_TOKEN_PATH } from '$lib/consts' + +const AUTO_SLACK_ARG = '$res:' + WORKSPACE_SLACK_BOT_TOKEN_PATH + +// `slack` belongs to the Slack built-in only while it holds the value the picker set; any other +// value is a custom handler's own argument and must survive handler switches and reloads. +export function syncSlackBotArg( + args: Record, + isSlackBuiltin: boolean, + wasSlackBuiltin: boolean +) { + if (isSlackBuiltin) { + args.slack = AUTO_SLACK_ARG + } else if (wasSlackBuiltin && args.slack === AUTO_SLACK_ARG) { + delete args.slack + } +} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 7228cf9f48..58f9d5d897 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -1133,10 +1133,14 @@ } // Strip keys from extraArgs that are auto-managed by child components: - // - 'slack': computed by ErrorOrRecoveryHandler's $effect based on handler type + // - 'slack' while the Slack built-in is selected: set by ErrorOrRecoveryHandler's $effect // - 'channel_name': display metadata stripped by SchemaForm's removeExtraKey() - function normalizeHandlerExtraArgs(args: Record): Record { - const { slack: _, channel_name: __, ...rest } = args + function normalizeHandlerExtraArgs( + args: Record, + handlerSelected: ErrorHandler + ): Record { + const { channel_name: _, ...rest } = args + if (handlerSelected === 'slack') delete rest.slack return rest } @@ -1146,7 +1150,10 @@ errorHandlerSelected: initialErrorHandlerSelected, errorHandlerScriptPath: initialErrorHandlerScriptPath, errorHandlerItemKind: initialErrorHandlerItemKind, - errorHandlerExtraArgs: normalizeHandlerExtraArgs(initialErrorHandlerExtraArgs), + errorHandlerExtraArgs: normalizeHandlerExtraArgs( + initialErrorHandlerExtraArgs, + initialErrorHandlerSelected + ), errorHandlerMutedOnCancel: initialErrorHandlerMutedOnCancel, errorHandlerMutedOnUserPath: initialErrorHandlerMutedOnUserPath } @@ -1155,7 +1162,7 @@ errorHandlerSelected: errorHandlerSelected, errorHandlerScriptPath: errorHandlerScriptPath, errorHandlerItemKind: errorHandlerItemKind, - errorHandlerExtraArgs: normalizeHandlerExtraArgs(errorHandlerExtraArgs), + errorHandlerExtraArgs: normalizeHandlerExtraArgs(errorHandlerExtraArgs, errorHandlerSelected), errorHandlerMutedOnCancel: errorHandlerMutedOnCancel, errorHandlerMutedOnUserPath: errorHandlerMutedOnUserPath } From 8cdb42aaae764977008f8ac4598709c7ae09af3f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 20:41:26 +0200 Subject: [PATCH 53/60] perf: run shellcheck directly from the lsp launcher (#11707) --- docker/DockerfileExtra | 2 +- docker/test_windmill_extra.ts | 55 +++++++++ lsp/pyls_launcher.py | 225 +++++++++++++++++++++++++++++++++- 3 files changed, 278 insertions(+), 4 deletions(-) diff --git a/docker/DockerfileExtra b/docker/DockerfileExtra index ace382528b..1d8d0003da 100644 --- a/docker/DockerfileExtra +++ b/docker/DockerfileExtra @@ -71,7 +71,7 @@ ENV XDG_CACHE_HOME=/pyls/.cache RUN uv pip install --system --break-system-packages pipenv tornado python-lsp-jsonrpc ruff==0.16.0 Cython # Install Node-based language servers -RUN npm install -g diagnostic-languageserver pyright +RUN npm install -g pyright # Setup LSP working directory WORKDIR /pyls diff --git a/docker/test_windmill_extra.ts b/docker/test_windmill_extra.ts index 5fd1a74108..c5884c2dd4 100644 --- a/docker/test_windmill_extra.ts +++ b/docker/test_windmill_extra.ts @@ -121,6 +121,60 @@ async function testLspWebSocket(): Promise { }) } +async function testLspShellDiagnostics(): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + ws.close() + reject(new Error('no shellcheck diagnostics within 10s')) + }, 10000) + + const ws = new WebSocket(`ws://${HOST}:${LSP_PORT}/ws/diagnostic`) + const send = (msg: object) => ws.send(JSON.stringify({ jsonrpc: '2.0', ...msg })) + + ws.onopen = () => { + // Editors still send a linter definition; the server's own configuration + // applies, so the diagnostics below have to come from shellcheck. + send({ + id: 1, + method: 'initialize', + params: { + capabilities: {}, + initializationOptions: { + linters: { shellcheck: { command: 'true', sourceName: 'shellcheck' } }, + filetypes: { shell: 'shellcheck' } + } + } + }) + send({ + method: 'textDocument/didOpen', + params: { + textDocument: { + uri: 'file:///tmp/monaco/test.sh', + languageId: 'shell', + version: 1, + text: '#!/bin/bash\necho "$unassigned"\n' + } + } + }) + } + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data.toString()) + if (msg.method !== 'textDocument/publishDiagnostics') return + clearTimeout(timeout) + ws.close() + const codes = msg.params.diagnostics.map((d: { code: string }) => d.code) + if (codes.includes('SC2154')) resolve() + else reject(new Error(`expected SC2154, got ${JSON.stringify(msg.params.diagnostics)}`)) + } + + ws.onerror = () => { + clearTimeout(timeout) + reject(new Error('LSP WebSocket connection failed')) + } + }) +} + async function runLspTests(): Promise { log('\n=== LSP Service Tests (Port ' + LSP_PORT + ') ===') @@ -132,6 +186,7 @@ async function runLspTests(): Promise { let passed = true passed = (await runTest('LSP health check', testLspHealth)) && passed passed = (await runTest('LSP WebSocket connection (pyright)', testLspWebSocket)) && passed + passed = (await runTest('LSP shell diagnostics', testLspShellDiagnostics)) && passed return passed } diff --git a/lsp/pyls_launcher.py b/lsp/pyls_launcher.py index 9e522d8ce5..b3fbe7fd9a 100644 --- a/lsp/pyls_launcher.py +++ b/lsp/pyls_launcher.py @@ -1,6 +1,9 @@ +import asyncio import logging +import shutil import subprocess import threading +import time import os import urllib.request import urllib.error @@ -149,8 +152,221 @@ class LanguageServerWebSocketHandler(websocket.WebSocketHandler): class PyrightLS(LanguageServerWebSocketHandler): procargs = ["pipenv", "run", "pyright-langserver", "--stdio"] -class DiagnosticLS(LanguageServerWebSocketHandler): - procargs = ["diagnostic-languageserver", "--stdio", "--log-level", "4"] +SHELLCHECK_BIN = shutil.which("shellcheck") +# The linter is configured here rather than by the editor's initializationOptions. +SHELLCHECK_ARGS = ["--norc", "--format=json1", "-"] +SHELLCHECK_TIMEOUT_SECS = float(os.environ.get("SHELLCHECK_TIMEOUT_SECS", "10")) +SHELLCHECK_MAX_DOC_CHARS = int(os.environ.get("SHELLCHECK_MAX_DOC_CHARS", "200000")) +SHELLCHECK_MAX_DOCS_PER_CONN = 8 +# A document whose checks finish faster than this is checked on every change +# with no pause; see ShellcheckLS._changed. +SHELLCHECK_INSTANT_SECS = 0.05 +SHELLCHECK_MAX_DEBOUNCE_SECS = 1.0 +# Shared by every connection: shellcheck is CPU-bound and single-threaded, so +# more concurrent runs than cores only makes all of them slower. +_shellcheck_slots = asyncio.Semaphore(os.cpu_count() or 2) + +_SHELLCHECK_SEVERITIES = {"error": 1, "warning": 2, "info": 3, "style": 3} + + +def _utf16_col(line, col): + """shellcheck counts code points, LSP positions count UTF-16 code units.""" + col = max(col - 1, 0) + if line.isascii(): + return col + return len(line[:col].encode("utf-16-le")) // 2 + + +def shellcheck_diagnostics(text, output): + lines = text.split("\n") + + def position(line, col): + line = max(line - 1, 0) + return { + "line": line, + "character": _utf16_col(lines[line], col) if line < len(lines) else 0, + } + + diagnostics = [] + for c in json.loads(output).get("comments", []): + code = "SC%d" % c["code"] + diagnostics.append( + { + "range": { + "start": position(c["line"], c["column"]), + "end": position(c["endLine"], c["endColumn"]), + }, + "severity": _SHELLCHECK_SEVERITIES.get(c["level"], 3), + "code": code, + "codeDescription": {"href": "https://www.shellcheck.net/wiki/" + code}, + "source": "shellcheck", + "message": c["message"], + } + ) + return diagnostics + + +class ShellcheckLS(websocket.WebSocketHandler): + """Lints shell documents by running shellcheck itself, speaking only the + slice of LSP that takes: document sync in, diagnostics out.""" + + def open(self): + self.docs = {} + self.tasks = {} + self.cost = {} + self.linted = {} + + def check_origin(self, origin): + return True + + def on_message(self, message): + try: + msg = json.loads(message) + method = msg.get("method") + params = msg.get("params") or {} + if method == "initialize": + self._send( + { + "jsonrpc": "2.0", + "id": msg.get("id"), + "result": { + "capabilities": { + "textDocumentSync": {"openClose": True, "change": 1} + }, + "serverInfo": {"name": "windmill-shellcheck"}, + }, + } + ) + elif method == "textDocument/didOpen": + doc = params["textDocument"] + self._changed(doc["uri"], doc.get("version"), doc["text"]) + elif method == "textDocument/didChange": + doc = params["textDocument"] + self._changed( + doc["uri"], doc.get("version"), params["contentChanges"][-1]["text"] + ) + elif method == "textDocument/didClose": + self._forget(params["textDocument"]["uri"]) + elif method == "shutdown": + self._send({"jsonrpc": "2.0", "id": msg.get("id"), "result": None}) + elif method is not None and "id" in msg: + self._send( + { + "jsonrpc": "2.0", + "id": msg["id"], + "error": {"code": -32601, "message": "Method not found"}, + } + ) + except (ValueError, KeyError, IndexError, TypeError, AttributeError): + log.warning("shellcheck: dropping malformed message") + + def on_close(self): + for uri in list(self.docs): + self._forget(uri) + + def _send(self, msg): + try: + self.write_message(json.dumps(msg)) + except websocket.WebSocketClosedError: + pass + + def _forget(self, uri): + task = self.tasks.pop(uri, None) + if task: + task.cancel() + self.docs.pop(uri, None) + self.cost.pop(uri, None) + self.linted.pop(uri, None) + + def _changed(self, uri, version, text): + if not isinstance(uri, str) or not isinstance(text, str): + raise TypeError + if uri not in self.docs and len(self.docs) >= SHELLCHECK_MAX_DOCS_PER_CONN: + return + if len(text) > SHELLCHECK_MAX_DOC_CHARS: + # Not checked, so not kept either: reported as having no findings. + text = None + self.docs[uri] = (version, text) + if uri in self.linted and self.linted[uri] == text: + return + running = self.tasks.pop(uri, None) + # The pause before a check is what that document's checks cost: one that + # is cheap is checked at once, one that is not waits out a typing burst + # rather than burning a core on results the next keystroke discards. + delay = self.cost.get(uri, 0) + if delay < SHELLCHECK_INSTANT_SECS: + delay = 0 + if running and not running.done(): + running.cancel() + delay = max(delay, SHELLCHECK_INSTANT_SECS) + self.tasks[uri] = asyncio.ensure_future(self._lint(uri, version, text, delay)) + + async def _lint(self, uri, version, text, delay): + if delay: + await asyncio.sleep(delay) + diagnostics = [] + if SHELLCHECK_BIN and text is not None: + async with _shellcheck_slots: + started = time.monotonic() + try: + output = await self._run(text) + self.cost[uri] = 0 + finally: + # A killed check counts too, as a lower bound: it is the only + # measure a document edited faster than it can be checked gets. + if uri in self.docs: + self.cost[uri] = min( + max(time.monotonic() - started, self.cost.get(uri, 0)), + SHELLCHECK_MAX_DEBOUNCE_SECS, + ) + if output is None: + return + try: + diagnostics = shellcheck_diagnostics(text, output) + except (ValueError, KeyError, TypeError) as e: + log.warning("shellcheck: unreadable output: %s", e) + return + if self.docs.get(uri) != (version, text): + return + self.linted[uri] = text + params = {"uri": uri, "diagnostics": diagnostics} + if version is not None: + params["version"] = version + self._send( + { + "jsonrpc": "2.0", + "method": "textDocument/publishDiagnostics", + "params": params, + } + ) + + async def _run(self, text): + try: + proc = await asyncio.create_subprocess_exec( + SHELLCHECK_BIN, + *SHELLCHECK_ARGS, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + except OSError as e: + log.warning("shellcheck: could not start: %s", e) + return None + try: + output, _ = await asyncio.wait_for( + proc.communicate(text.encode("utf-8", "replace")), + SHELLCHECK_TIMEOUT_SECS, + ) + return output + except asyncio.TimeoutError: + log.warning("shellcheck: timed out on a %d char document", len(text)) + return None + finally: + # Also reached when a newer change cancels this check mid-run. + if proc.returncode is None: + proc.kill() + await proc.wait() + class RuffLS(LanguageServerWebSocketHandler): procargs = ["ruff", "server"] @@ -201,11 +417,14 @@ if __name__ == "__main__": # spawned `ruff server` picks it up. start_ruff_config_poller() + if not SHELLCHECK_BIN: + log.warning("shellcheck is not on PATH: shell scripts will not be linted") + port = int(os.environ.get("PORT", "3001")) app = web.Application( [ (r"/ws/pyright", PyrightLS), - (r"/ws/diagnostic", DiagnosticLS), + (r"/ws/diagnostic", ShellcheckLS), (r"/ws/ruff", RuffLS), (r"/ws/deno", DenoLS), (r"/ws/go", GoLS), From 4be67fe3ac09f722f31ef85e6fc757bb0a2fabe4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Mon, 12 Oct 2026 20:57:36 +0200 Subject: [PATCH 54/60] fix(groups): recompute member roles when an instance group role is cleared (#11708) * fix(groups): recompute member roles when an instance group role is cleared Co-Authored-By: Claude Opus 5.5 * fix: restore sqlx cache entry still used by the scim group delete Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/windmill-api-groups/src/groups.rs | 43 +++++------- .../tests/groups.rs | 66 +++++++++++++++++++ 2 files changed, 82 insertions(+), 27 deletions(-) diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index 0209af8585..d9550d2f5b 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -470,26 +470,16 @@ async fn update_igroup( require_super_admin(&db, &authed).await?; let mut tx: Transaction<'_, Postgres> = db.begin().await?; - let exists_opt = sqlx::query("SELECT 1 FROM instance_group WHERE name = $1") - .bind(name.clone()) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(exists_opt, "instance_group", name.clone())?; + let old_role_opt = sqlx::query_scalar!( + "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await?; + let old_role = not_found_if_none(old_role_opt, "instance_group", name.clone())?; let validated_role = validate_instance_role(&igroup_update.instance_role)?; - // Fetch old role before updating so we can detect changes - let old_role = if igroup_update.instance_role.is_some() { - sqlx::query_scalar!( - "SELECT instance_role FROM instance_group WHERE name = $1", - &name - ) - .fetch_one(&mut *tx) - .await? - } else { - None - }; - sqlx::query("UPDATE instance_group SET summary = $1, instance_role = $2 WHERE name = $3") .bind(igroup_update.new_summary) .bind(&validated_role) @@ -497,8 +487,9 @@ async fn update_igroup( .execute(&mut *tx) .await?; - // If instance_role actually changed, propagate to all group members - if igroup_update.instance_role.is_some() && old_role != validated_role { + // An omitted or null instance_role is stored as NULL above, so it revokes like any other + // change: members must be recomputed whenever the stored role differs. + if old_role != validated_role { propagate_instance_group_roles(&name, &mut tx).await?; } @@ -664,21 +655,19 @@ async fn delete_igroup( // FOR UPDATE: the group row is the group-level mutex, taken before the workspace // advisory locks (see reconcile_workspace_instance_groups). - let group_role = sqlx::query_scalar!( + sqlx::query_scalar!( "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", &name ) .fetch_optional(&mut *tx) - .await? - .flatten(); + .await?; - let affected_members: Vec = if group_role.is_some() { + // Collected whatever the group's current role: a member can still hold a role this group + // granted before it was cleared. + let affected_members: Vec = sqlx::query_scalar!("SELECT email FROM email_to_igroup WHERE igroup = $1", &name) .fetch_all(&mut *tx) - .await? - } else { - vec![] - }; + .await?; // Captured and advisory-locked before the settings update strips the group from them. #[cfg(feature = "private")] diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index 26e5caeea1..9928d1c060 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -497,6 +497,72 @@ async fn test_remove_user_from_instance_group_rederives_role( Ok(()) } +/// Clearing an instance group's role, or deleting a group whose role was cleared out from under +/// its members, must take the instance-level role back from them. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_role_revoked_on_clear_and_delete( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let email = "test2@windmill.dev"; + + let is_super_admin = || async { + sqlx::query_scalar::<_, bool>("SELECT super_admin FROM password WHERE email = $1") + .bind(email) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "admins_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": "superadmin" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "set role: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/adduser/admins_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser: {}", resp.text().await?); + assert!(is_super_admin().await?, "group should grant superadmin"); + + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "clearing the group's role must revoke superadmin" + ); + + // A member left elevated while the group's stored role is already NULL. + sqlx::query( + "UPDATE password SET super_admin = true, role_source = 'instance_group' WHERE email = $1", + ) + .bind(email) + .execute(&db) + .await?; + + let resp = authed(client().delete(format!("{global_base}/delete/admins_grp"))) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "deleting a role-cleared group must revoke superadmin" + ); + + Ok(()) +} + /// An overwrite import that moves a member from a dropped group to a retained one must keep /// their workspace data. /// From 4942149b05c9384b39402f0e3cb7771565888f45 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 13 Oct 2026 02:27:13 -0400 Subject: [PATCH 55/60] feat: encrypt workspace secrets with AES-256-GCM (#11678) * feat: encrypt workspace secrets with AES-256-GCM Co-Authored-By: Claude Opus 5.5 (1M context) * docs: note the aead gate covers workers only and is one-way Co-Authored-By: Claude Opus 5.5 (1M context) * fix: skip secrets the upgrade cannot store and clear its guard on panic Co-Authored-By: Claude Opus 5.5 (1M context) * fix: start aead secret writes one release after read support Co-Authored-By: Claude Opus 5.5 (1M context) * chore: update ee-repo-ref to 06610ecc950ed64a2709d7f1aa8c7b49a95fdbae This commit updates the EE repository reference after PR #869 was merged in windmill-ee-private. Previous ee-repo-ref: 599eae0034b34c5e0d3d42df37cf18bbe01f9b49 New ee-repo-ref: 06610ecc950ed64a2709d7f1aa8c7b49a95fdbae Automated by sync-ee-ref workflow. * feat: widen variable.value to fit AES-GCM ciphertext of the longest legacy secret Co-Authored-By: Claude Opus 5.5 (1M context) * fix: refuse legacy secrets longer than 15000 chars so every secret can be upgraded Co-Authored-By: Claude Opus 5.5 (1M context) * fix: cap legacy secrets at 15000 chars with a check constraint so every writer is covered Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/Cargo.lock | 1 + backend/Cargo.toml | 1 + ...261012213357_widen_variable_value.down.sql | 12 + ...20261012213357_widen_variable_value.up.sql | 8 + backend/src/monitor.rs | 19 ++ .../src/remote_deploy.rs | 6 +- .../windmill-api-workspaces/src/workspaces.rs | 4 +- backend/windmill-api/src/ai_sessions.rs | 6 +- backend/windmill-common/Cargo.toml | 5 +- backend/windmill-common/src/lib.rs | 1 + backend/windmill-common/src/min_version.rs | 5 + backend/windmill-common/src/utils.rs | 3 + backend/windmill-common/src/variables.rs | 191 +++++++++++-- .../windmill-common/src/workspace_crypt.rs | 255 ++++++++++++++++++ backend/windmill-store/src/variables.rs | 3 +- cli/src/commands/variable/variable.ts | 3 +- 16 files changed, 496 insertions(+), 27 deletions(-) create mode 100644 backend/migrations/20261012213357_widen_variable_value.down.sql create mode 100644 backend/migrations/20261012213357_widen_variable_value.up.sql create mode 100644 backend/windmill-common/src/workspace_crypt.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 610768134f..1bd38513a7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16333,6 +16333,7 @@ dependencies = [ "git-version", "globset", "hex", + "hkdf", "hmac 0.12.1", "hyper 1.12.0", "indexmap 2.14.2", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6f2420b8a3..aaf5a2e8df 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -599,6 +599,7 @@ p256 = "0.13" spki = { version = "0.7", features = ["pem"] } pkcs1 = "0.7" aes-gcm = "0.10.3" +hkdf = "0.12.4" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" dashmap = "6.1.0" diff --git a/backend/migrations/20261012213357_widen_variable_value.down.sql b/backend/migrations/20261012213357_widen_variable_value.down.sql new file mode 100644 index 0000000000..48b61c3f65 --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.down.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE + too_long BIGINT; +BEGIN + SELECT count(*) INTO too_long FROM variable WHERE length(value) > 15000; + IF too_long > 0 THEN + RAISE EXCEPTION 'cannot narrow variable.value to VARCHAR(15000): % row(s) are longer than 15000 characters', too_long; + END IF; +END $$; + +ALTER TABLE variable DROP CONSTRAINT IF EXISTS variable_value_length; +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000); diff --git a/backend/migrations/20261012213357_widen_variable_value.up.sql b/backend/migrations/20261012213357_widen_variable_value.up.sql new file mode 100644 index 0000000000..06dc8a191a --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.up.sql @@ -0,0 +1,8 @@ +-- AES-256-GCM ciphertext is 32 bytes (magic, nonce, tag) longer than legacy CBC before base64, +-- so the longest CBC value that fits 15000 chars is exactly 15040 chars once re-encrypted. +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15040); + +-- Only GCM secrets (base64 of the `wmA2` magic starts `d21BM`) may use the extra room: a longer +-- CBC value could never be re-encrypted to fit, and would fail every rotation of its workspace key. +ALTER TABLE variable ADD CONSTRAINT variable_value_length + CHECK (length(value) <= 15000 OR (is_secret AND value LIKE 'd21BM%')); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 8de85baa24..24f9d8221c 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -4449,6 +4449,24 @@ pub async fn monitor_db( } }; + // Re-encrypt the legacy-format secret variables once new values are written in AES-256-GCM. + // Every ~5 min until one pass has gone through every workspace; spawned so a large instance + // does not hold up the join below. + let legacy_secret_upgrade_f = async { + if server_mode + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(30) + && windmill_common::variables::legacy_secret_upgrade_pending() + { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { + windmill_common::variables::upgrade_legacy_secret_variables(&db).await + }); + } + } + }; + // Delete the AI session backups older than their workspace's retention. Every ~40 min // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention // counts in days. Spawned for the same reason as the credential maintenance above, a @@ -4521,6 +4539,7 @@ pub async fn monitor_db( git_auto_pull_f, git_credential_maintenance_f, ai_session_retention_f, + legacy_secret_upgrade_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, stranded_jobs_f, diff --git a/backend/windmill-api-workspaces/src/remote_deploy.rs b/backend/windmill-api-workspaces/src/remote_deploy.rs index 3e701cbf14..a0f922a607 100644 --- a/backend/windmill-api-workspaces/src/remote_deploy.rs +++ b/backend/windmill-api-workspaces/src/remote_deploy.rs @@ -25,11 +25,11 @@ use axum::{ }; use chrono::{DateTime, Utc}; use lazy_static::lazy_static; -use magic_crypt::MagicCrypt256; use regex::Regex; use serde::{Deserialize, Serialize}; use windmill_api_auth::{is_effectively_unscoped, ApiAuthed, Tokened}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::variables::WorkspaceCrypt; use windmill_common::{ auth::hash_token, error::{error_source_chain, Error, Result}, @@ -676,8 +676,8 @@ async fn proxy( pub(crate) async fn reencrypt_tokens( conn: &mut sqlx::PgConnection, w_id: &str, - old: &MagicCrypt256, - new: &MagicCrypt256, + old: &WorkspaceCrypt, + new: &WorkspaceCrypt, ) -> Result<()> { let rows = sqlx::query!( "SELECT email, token FROM remote_deploy_token diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 7f6eace660..3bfcef5c96 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -6317,8 +6317,8 @@ async fn set_encryption_key( async fn reencrypt_git_sync_secrets( conn: &mut sqlx::PgConnection, w_id: &str, - old: &magic_crypt::MagicCrypt256, - new: &magic_crypt::MagicCrypt256, + old: &windmill_common::variables::WorkspaceCrypt, + new: &windmill_common::variables::WorkspaceCrypt, ) -> Result<()> { let Some((mut credentials, mut git_sync)) = sqlx::query_as::<_, (serde_json::Value, Option)>( diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index 65a8137c4e..9e980a54f4 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -24,7 +24,7 @@ use axum::{ Extension, Json, Router, }; use futures::{StreamExt, TryStreamExt}; -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use magic_crypt::MagicCryptTrait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::sync::Arc; @@ -35,7 +35,7 @@ use windmill_api_workspaces::ai_session_backups::{ use windmill_api_workspaces::workspaces::sessions_retention_days; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; -use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key, WorkspaceCrypt}; use windmill_object_store::object_store_reexports::{ ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, }; @@ -135,7 +135,7 @@ enum Read { /// The user's prefix in the workspace storage, plus what reads and writes it. struct Backend { store: Arc, - mc: MagicCrypt256, + mc: WorkspaceCrypt, prefix: String, /// Name the storage and the generation the objects are under, for the browser's sync /// state: a row recorded against another storage or generation is stale, a removal is diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 97dbc7e7bf..e447baf563 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -8,7 +8,7 @@ edition.workspace = true default = [] enterprise = ["dep:aws-config"] instance_config_schema = ["dep:schemars"] -local_reports = ["dep:rsa", "dep:aes-gcm"] +local_reports = ["dep:rsa"] private = ["dep:aws-sdk-rds", "dep:aws-sdk-secretsmanager", "dep:aws-config"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] @@ -119,7 +119,8 @@ rsa = { workspace = true, optional = true } p256 = { workspace = true, optional = true } spki = { workspace = true } pkcs1 = { workspace = true } -aes-gcm = { workspace = true, optional = true } +aes-gcm.workspace = true +hkdf.workspace = true semver.workspace = true croner.workspace = true diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index f15b43182f..58a477d7b9 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -157,6 +157,7 @@ pub mod wac; pub mod webhook; pub mod worker; pub mod worker_group_job_stats; +pub mod workspace_crypt; pub mod workspaces; pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; diff --git a/backend/windmill-common/src/min_version.rs b/backend/windmill-common/src/min_version.rs index 930b718b7e..05b664b23d 100644 --- a/backend/windmill-common/src/min_version.rs +++ b/backend/windmill-common/src/min_version.rs @@ -23,6 +23,11 @@ pub const MIN_VERSION_SUPPORTS_BUN_LOCKFILE_V2: VC = vc(1, 794, 0, "Bun v2 lockf // them, so a restriction set while one is live is silently ignored on the jobs it runs. Must // name the release this ships in. pub const MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES: VC = vc(1, 822, 0, "Restricted job tokens"); +// Gates writing AES-256-GCM workspace ciphertext and the background pass that rewrites the +// stored secret variables. Reading it ships one release earlier (1.831), so a rollback to +// that release still reads every secret; below it, secrets written after the switch stay +// unreadable until rolled forward. Only worker versions are seen here, not servers. +pub const MIN_VERSION_SUPPORTS_AEAD_SECRETS: VC = vc(1, 832, 0, "AES-GCM secret encryption"); pub const MIN_VERSION_SUPPORTS_NODE_DEBOUNCING: VC = vc(1, 658, 0, "Flow node debouncing"); pub const MIN_VERSION_SUPPORTS_TOKEN_HASH: VC = vc(1, 659, 0, "Token hash storage"); pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing"); diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index af6bdfe2b3..116a15303e 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -281,6 +281,9 @@ pub fn sanitize_db_error(e: sqlx::Error) -> Error { return Error::from(e); }; match db_err.code().as_deref() { + Some("23514") if db_err.constraint() == Some("variable_value_length") => { + Error::BadRequest("A field exceeds its maximum length".to_string()) + } // check_violation — `proper_id` / `proper_name` and friends Some("23514") => Error::BadRequest( "Invalid path or name: it does not match the required format".to_string(), diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index f45d457093..58766cf25b 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -12,11 +12,13 @@ use crate::scripts::ScriptHash; use crate::secret_backend::{get_secret_value, is_external_stored_value}; use crate::utils::WarnAfterExt; use crate::worker::Connection; +use crate::workspace_crypt::CipherFormat; +pub use crate::workspace_crypt::WorkspaceCrypt; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicBool, Ordering}; lazy_static::lazy_static! { pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); @@ -211,7 +213,7 @@ pub struct CreateVariable { pub ws_specific: Option, } -pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result { +pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result { // Check cache first (300-second staleness) let cached_key_o = WORKSPACE_CRYPT_CACHE.get(w_id).and_then(|(ts, key)| { if ts > chrono::Utc::now().timestamp() - 300 { @@ -235,7 +237,7 @@ pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result crate::error::Result { +) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; Ok(crypt_from_key_with_suffix(&key, key_suffix)) } /// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. -pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> WorkspaceCrypt { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - magic_crypt::new_magic_crypt!(crypt_key, 256) + WorkspaceCrypt::new(&crypt_key) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { @@ -371,27 +373,178 @@ pub async fn get_secret_value_as_admin( Ok(r) } -pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { - mc.encrypt_str_to_base64(value) +pub fn encrypt(mc: &WorkspaceCrypt, value: &str) -> String { + mc.encrypt_str(value) } const DECRYPT_FAILED: &str = "Could not decrypt value. The value may have been encrypted with a different key."; /// Every failure returns the same message so responses carry no signal about why decryption -/// failed (padding, base64, UTF-8); the cause is only logged. -pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { - mc.decrypt_base64_to_string(value).map_err(|e| { +/// failed (authentication, padding, base64, UTF-8); the kind of failure is only logged. +pub fn decrypt(mc: &WorkspaceCrypt, value: String) -> error::Result { + mc.decrypt_str(&value).map_err(|e| { tracing::warn!("decryption failed: {e}"); error::Error::internal_err(DECRYPT_FAILED.to_string()) }) } +static LEGACY_SECRET_UPGRADE_DONE: AtomicBool = AtomicBool::new(false); +static LEGACY_SECRET_UPGRADE_RUNNING: AtomicBool = AtomicBool::new(false); +const LEGACY_SECRET_UPGRADE_BATCH: i64 = 200; + +/// Whether this process still has to run [`upgrade_legacy_secret_variables`]: once new values +/// are written in v2, until one pass has gone through every workspace. +pub fn legacy_secret_upgrade_pending() -> bool { + CipherFormat::current() == CipherFormat::V2 + && !LEGACY_SECRET_UPGRADE_DONE.load(Ordering::Relaxed) +} + +/// Re-encrypt the secret variables still in the legacy format in v2, workspace by workspace. +/// Idempotent, and emits no git-sync event: the plaintexts do not change. +pub async fn upgrade_legacy_secret_variables(db: &DB) { + struct Running; + impl Drop for Running { + fn drop(&mut self) { + LEGACY_SECRET_UPGRADE_RUNNING.store(false, Ordering::Release); + } + } + if !legacy_secret_upgrade_pending() + || LEGACY_SECRET_UPGRADE_RUNNING.swap(true, Ordering::AcqRel) + { + return; + } + // Cleared on drop, so a pass that panics does not stop every later one in this process. + let _running = Running; + upgrade_legacy_secret_variables_pass(db).await; +} + +async fn upgrade_legacy_secret_variables_pass(db: &DB) { + let workspaces: Vec = match sqlx::query_scalar("SELECT id FROM workspace ORDER BY id") + .fetch_all(db) + .await + { + Ok(w) => w, + Err(e) => { + tracing::error!("secret encryption upgrade: listing workspaces failed: {e:#}"); + return; + } + }; + tracing::info!( + "secret encryption upgrade: re-encrypting legacy secret variables of {} workspaces in AES-256-GCM", + workspaces.len() + ); + let (mut upgraded, mut skipped, mut failed_workspaces) = (0, 0, 0); + for w_id in workspaces { + match upgrade_workspace_secret_variables(db, &w_id).await { + Ok((u, s)) => { + if u > 0 || s > 0 { + tracing::info!( + "secret encryption upgrade: workspace {w_id}: {u} re-encrypted, {s} skipped" + ); + } + upgraded += u; + skipped += s; + } + Err(e) => { + failed_workspaces += 1; + tracing::error!("secret encryption upgrade: workspace {w_id} failed: {e:#}"); + } + } + } + tracing::info!( + "secret encryption upgrade: done, {upgraded} re-encrypted, {skipped} skipped, {failed_workspaces} workspaces failed" + ); + if failed_workspaces == 0 { + LEGACY_SECRET_UPGRADE_DONE.store(true, Ordering::Relaxed); + } +} + +/// The number of values re-encrypted and of values skipped as they do not decrypt. +async fn upgrade_workspace_secret_variables(db: &DB, w_id: &str) -> error::Result<(usize, usize)> { + let (mut upgraded, mut skipped) = (0, 0); + let mut after = String::new(); + loop { + let mut tx = db.begin().await?; + // Shared, so a rotation (which locks the key row for update and rewrites every secret) + // never interleaves with a batch, and the key read here is the one the values are under. + let key: Option = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR SHARE", + ) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + let Some(key) = key else { + return Ok((upgraded, skipped)); + }; + let mc = crypt_from_key_with_suffix(&key, ""); + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT path, value FROM variable WHERE workspace_id = $1 AND is_secret AND path > $2 \ + ORDER BY path LIMIT $3", + ) + .bind(w_id) + .bind(&after) + .bind(LEGACY_SECRET_UPGRADE_BATCH) + .fetch_all(&mut *tx) + .await?; + let last_page = (rows.len() as i64) < LEGACY_SECRET_UPGRADE_BATCH; + for (path, value) in rows { + if value.is_empty() || is_external_stored_value(&value) || mc.is_v2(&value) { + after = path; + continue; + } + match mc.upgrade(&value) { + Some(new_value) => { + // In a savepoint, so a value the update rejects (v2 is up to 44 chars longer + // than CBC, which a column narrower than 15040 cannot hold) is skipped + // without losing the batch. + let mut sp = sqlx::Acquire::begin(&mut tx).await?; + // Only over the value it was computed from, so a write racing it wins. + match sqlx::query( + "UPDATE variable SET value = $1 \ + WHERE workspace_id = $2 AND path = $3 AND value = $4", + ) + .bind(new_value) + .bind(w_id) + .bind(&path) + .bind(&value) + .execute(&mut *sp) + .await + { + Ok(r) => { + sp.commit().await?; + upgraded += r.rows_affected() as usize; + } + Err(e) => { + sp.rollback().await?; + tracing::warn!( + "secret encryption upgrade: secret variable {path} of workspace {w_id} could not be stored re-encrypted, leaving it as is: {e}" + ); + skipped += 1; + } + } + } + None => { + tracing::warn!( + "secret encryption upgrade: secret variable {path} of workspace {w_id} does not decrypt, leaving it as is" + ); + skipped += 1; + } + } + after = path; + } + tx.commit().await?; + if last_page { + return Ok((upgraded, skipped)); + } + } +} + pub const WM_SCHEDULED_FOR: &str = "WM_SCHEDULED_FOR"; lazy_static::lazy_static! { pub static ref CUSTOM_ENVS_CACHE: Cache)> = Cache::new(100); - pub static ref WORKSPACE_CRYPT_CACHE: Cache = Cache::new(1000); + pub static ref WORKSPACE_CRYPT_CACHE: Cache = Cache::new(1000); } @@ -753,16 +906,24 @@ mod tests { DECRYPT_FAILED, }; use crate::error::Error; + use crate::workspace_crypt::WorkspaceCrypt; + use base64::Engine; use magic_crypt::MagicCryptTrait; #[test] fn decrypt_failures_all_return_the_same_message() { - let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); - let other = magic_crypt::new_magic_crypt!("another-key", 256); + let mc = WorkspaceCrypt::new("workspace-key"); + let legacy = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = WorkspaceCrypt::new("another-key"); + let b64 = base64::engine::general_purpose::STANDARD; + let v2 = mc.upgrade(&legacy.encrypt_str_to_base64("secret")).unwrap(); + let mut tampered = b64.decode(v2).unwrap(); + *tampered.last_mut().unwrap() ^= 1; let failures = [ "not base64 at all!".to_string(), - mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), + legacy.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), other.encrypt_str_to_base64("secret"), + b64.encode(&tampered), ]; for value in failures { match decrypt(&mc, value.clone()) { diff --git a/backend/windmill-common/src/workspace_crypt.rs b/backend/windmill-common/src/workspace_crypt.rs new file mode 100644 index 0000000000..a561060e32 --- /dev/null +++ b/backend/windmill-common/src/workspace_crypt.rs @@ -0,0 +1,255 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The cipher behind `variables::encrypt`/`decrypt`. +//! +//! Values are written as AES-256-GCM ("v2"), standard base64 of +//! `MAGIC (4) || nonce (12) || AES-256-GCM(PKCS#7(plaintext)) || tag (16)`. +//! Values written before are AES-256-CBC (`magic_crypt`) and are still read. +//! +//! The plaintext is padded to the block size so a v2 value decodes to a multiple of 16 bytes, +//! like a CBC one: the CLI and `validate_already_encrypted_secret` tell ciphertext from +//! plaintext by exactly that, and an old CLI that took a v2 value for plaintext would push it +//! to be encrypted again, corrupting the secret. + +use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng}; +use aes_gcm::{Aes256Gcm, Key, Nonce}; +use base64::Engine; +use hkdf::Hkdf; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use sha2::Sha256; + +use crate::min_version::MIN_VERSION_SUPPORTS_AEAD_SECRETS; + +pub const V2_MAGIC: [u8; 4] = *b"wmA2"; +const NONCE_LEN: usize = 12; +const TAG_LEN: usize = 16; +const BLOCK: usize = 16; + +const AEAD_KEY_INFO: &[u8] = b"windmill workspace secret aes-256-gcm v1"; + +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +/// Why a value did not decrypt. Only ever logged, never returned to a caller. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecryptFailure { + Encoding, + Authentication, + Utf8, +} + +impl std::fmt::Display for DecryptFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + DecryptFailure::Encoding => "not base64", + DecryptFailure::Authentication => "authentication or padding check failed", + DecryptFailure::Utf8 => "plaintext is not UTF-8", + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CipherFormat { + LegacyCbc, + V2, +} + +impl CipherFormat { + /// What new values are written as. Old workers read secrets straight from the database with + /// the CBC cipher alone, so v2 is written only once every live worker reads it. + pub fn current() -> Self { + if MIN_VERSION_SUPPORTS_AEAD_SECRETS.met_conservatively() { + CipherFormat::V2 + } else { + CipherFormat::LegacyCbc + } + } +} + +#[derive(Clone)] +pub struct WorkspaceCrypt { + legacy: MagicCrypt256, + aead: Aes256Gcm, +} + +/// Code that drives `magic_crypt` directly on this cipher (app link secrets, AI session +/// objects) keeps the legacy format it always had. +impl std::ops::Deref for WorkspaceCrypt { + type Target = MagicCrypt256; + fn deref(&self) -> &MagicCrypt256 { + &self.legacy + } +} + +impl WorkspaceCrypt { + /// `crypt_key` is the key material the CBC cipher was always built from (workspace key, salt, + /// suffix). The AEAD key is derived from it under its own label, so the two ciphers never + /// share a key. + pub fn new(crypt_key: &str) -> Self { + let mut aead_key = [0u8; 32]; + Hkdf::::new(None, crypt_key.as_bytes()) + .expand(AEAD_KEY_INFO, &mut aead_key) + .expect("32 bytes is a valid HKDF-SHA256 output length"); + Self { + legacy: magic_crypt::new_magic_crypt!(crypt_key, 256), + aead: Aes256Gcm::new(Key::::from_slice(&aead_key)), + } + } + + pub fn encrypt_str(&self, plaintext: &str) -> String { + self.encrypt_str_in(plaintext, CipherFormat::current()) + } + + pub fn encrypt_str_in(&self, plaintext: &str, format: CipherFormat) -> String { + match format { + CipherFormat::V2 => B64.encode(self.seal(plaintext.as_bytes())), + CipherFormat::LegacyCbc => self.legacy.encrypt_str_to_base64(plaintext), + } + } + + pub fn decrypt_str(&self, ciphertext: &str) -> Result { + let bytes = B64 + .decode(ciphertext) + .map_err(|_| DecryptFailure::Encoding)?; + String::from_utf8(self.decrypt_bytes(&bytes)?).map_err(|_| DecryptFailure::Utf8) + } + + /// Whether `ciphertext` is a v2 value under this key. + pub fn is_v2(&self, ciphertext: &str) -> bool { + B64.decode(ciphertext) + .is_ok_and(|bytes| self.open(&bytes).is_some()) + } + + /// The v2 form of a legacy `ciphertext`, checked to decrypt back to the same plaintext. + /// `None` for a v2 value and for anything that does not decrypt under this key. + pub fn upgrade(&self, ciphertext: &str) -> Option { + if self.is_v2(ciphertext) { + return None; + } + let plaintext = self.decrypt_str(ciphertext).ok()?; + let upgraded = self.encrypt_str_in(&plaintext, CipherFormat::V2); + (self.decrypt_str(&upgraded).ok()? == plaintext).then_some(upgraded) + } + + fn decrypt_bytes(&self, bytes: &[u8]) -> Result, DecryptFailure> { + match self.open(bytes) { + Some(plaintext) => Ok(plaintext), + // Also for a value with the magic: a CBC ciphertext starts with it one time in 2^32. + None => self + .legacy + .decrypt_bytes_to_bytes(bytes) + .map_err(|_| DecryptFailure::Authentication), + } + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let pad = BLOCK - plaintext.len() % BLOCK; + let mut padded = Vec::with_capacity(plaintext.len() + pad); + padded.extend_from_slice(plaintext); + padded.resize(plaintext.len() + pad, pad as u8); + let sealed = self + .aead + .encrypt(&nonce, padded.as_slice()) + .expect("AES-GCM encryption fails only past 64 GiB of plaintext"); + let mut out = Vec::with_capacity(V2_MAGIC.len() + NONCE_LEN + sealed.len()); + out.extend_from_slice(&V2_MAGIC); + out.extend_from_slice(&nonce); + out.extend_from_slice(&sealed); + out + } + + fn open(&self, data: &[u8]) -> Option> { + let body = data.strip_prefix(&V2_MAGIC)?; + if body.len() < NONCE_LEN + BLOCK + TAG_LEN || (body.len() - NONCE_LEN) % BLOCK != 0 { + return None; + } + let (nonce, sealed) = body.split_at(NONCE_LEN); + let mut padded = self.aead.decrypt(Nonce::from_slice(nonce), sealed).ok()?; + let pad = *padded.last()? as usize; + if pad == 0 + || pad > BLOCK + || padded[padded.len() - pad..] + .iter() + .any(|b| *b as usize != pad) + { + return None; + } + padded.truncate(padded.len() - pad); + Some(padded) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "workspace-key"; + + #[test] + fn v2_round_trips() { + let wc = WorkspaceCrypt::new(KEY); + for plain in [ + "", + "s", + "exactly sixteen!", + "a longer secret value with ünïcödé", + ] { + let a = wc.encrypt_str_in(plain, CipherFormat::V2); + assert_ne!(a, wc.encrypt_str_in(plain, CipherFormat::V2), "nonce reuse"); + assert!(wc.is_v2(&a)); + assert_eq!(wc.decrypt_str(&a).unwrap(), plain); + } + } + + #[test] + fn legacy_cbc_still_decrypts() { + let wc = WorkspaceCrypt::new(KEY); + let legacy = magic_crypt::new_magic_crypt!(KEY, 256).encrypt_str_to_base64("old secret"); + assert!(!wc.is_v2(&legacy)); + assert_eq!(wc.decrypt_str(&legacy).unwrap(), "old secret"); + let upgraded = wc.upgrade(&legacy).unwrap(); + assert!(wc.is_v2(&upgraded)); + assert_eq!(wc.decrypt_str(&upgraded).unwrap(), "old secret"); + assert_eq!(wc.upgrade(&upgraded), None); + } + + #[test] + fn magic_prefixed_cbc_still_decrypts() { + // Too rare to find by search, so built: with magic_crypt's zero IV and SHA-256(key) AES + // key, the first CBC block is AES(P1), so P1 = AES^-1(MAGIC || zeros). + let mut block = [0u8; 16]; + block[..4].copy_from_slice(&V2_MAGIC); + let mut plaintext = { + use aes_gcm::aes::cipher::{BlockDecrypt, KeyInit}; + use sha2::Digest; + let cipher = aes_gcm::aes::Aes256::new_from_slice(&Sha256::digest(KEY)).unwrap(); + let mut b = aes_gcm::aes::Block::clone_from_slice(&block); + cipher.decrypt_block(&mut b); + b.to_vec() + }; + plaintext.extend_from_slice(b"tail of the secret value"); + let ciphertext = magic_crypt::new_magic_crypt!(KEY, 256).encrypt_bytes_to_bytes(&plaintext); + assert!(ciphertext.starts_with(&V2_MAGIC)); + assert_eq!( + WorkspaceCrypt::new(KEY).decrypt_bytes(&ciphertext).unwrap(), + plaintext + ); + } + + #[test] + fn v2_passes_the_block_multiple_ciphertext_shape_check() { + let wc = WorkspaceCrypt::new(KEY); + for len in 0..64 { + let decoded = B64 + .decode(wc.encrypt_str_in(&"x".repeat(len), CipherFormat::V2)) + .unwrap(); + assert!(!decoded.is_empty() && decoded.len() % 16 == 0, "len {len}"); + } + } +} diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 8f288dc85e..151b75bd85 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -563,7 +563,8 @@ async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { /// /// The check is purely structural and never decrypts, so it cannot act as a /// decryption/padding oracle for a caller who can write but not read secrets. -/// `encrypt` (AES-256-CBC) always yields standard base64 decoding to a non-zero +/// `encrypt` (AES-256-GCM, or legacy AES-256-CBC; see `windmill_common::workspace_crypt`) +/// always yields standard base64 decoding to a non-zero /// multiple of the 16-byte block size; anything else cannot be our ciphertext. /// Values stored by an external backend ($vault:/$aws_sm:/$azure_kv: markers) /// are not workspace ciphertext and are passed through untouched. diff --git a/cli/src/commands/variable/variable.ts b/cli/src/commands/variable/variable.ts index 8963f376d2..88ad108a1a 100644 --- a/cli/src/commands/variable/variable.ts +++ b/cli/src/commands/variable/variable.ts @@ -107,7 +107,8 @@ export interface VariableFile { * plaintext value a user authored by hand. * * Mirrors the server guard (windmill-store/src/variables.rs): workspace - * ciphertext (AES-256-CBC, base64) is standard base64 decoding to a non-zero + * ciphertext (AES-256-GCM or legacy AES-256-CBC, base64) is standard base64 + * decoding to a non-zero * multiple of the 16-byte block size. External secret-backend markers * ($vault:/$aws_sm:/$azure_kv:) are stored verbatim too, so they count as * already-encrypted. This is a shape check only — it never decrypts. From add042ef2f05828a8ea5f6d0a089204bae2322d5 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Tue, 13 Oct 2026 02:52:58 -0400 Subject: [PATCH 56/60] fix: end other sessions when a user changes their password (#11710) Co-authored-by: Claude Opus 5.5 (1M context) --- ...db63d19f1c9ab0366e01c0092aa653ab51877.json | 15 ++++++++++++++ backend/windmill-api/src/users.rs | 20 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json diff --git a/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json new file mode 100644 index 0000000000..be4fbb73c9 --- /dev/null +++ b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877" +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 39a213c860..5b26cf57b8 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -198,6 +198,7 @@ async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, authed: ApiAuthed, + Tokened { token }: Tokened, OptJobAuthed { job_id, .. }: OptJobAuthed, Json(ep): Json, ) -> Result { @@ -206,7 +207,15 @@ async fn set_password( // (GHSA-hfh4-cx4h-3fcr). forbid_elevated_job_token(&db, &authed.email, job_id).await?; let email = authed.email.clone(); - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + &email, + windmill_common::auth::hash_token(&token) + ) + .execute(&db) + .await?; + Ok(res) } async fn set_password_of_user( @@ -219,7 +228,14 @@ async fn set_password_of_user( ) -> Result { require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session'", + &email + ) + .execute(&db) + .await?; + Ok(res) } #[derive(Deserialize)] From 7bbe03227d56688893aafda305a13d65b2bb84f4 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 13 Oct 2026 08:55:31 +0200 Subject: [PATCH 57/60] fix: bump git sync hub scripts to cli 1.830.0 (#11681) Claude-Session: https://claude.ai/code/session_011BzU9zd53JsTShCsbKzYzB Co-authored-by: Claude Opus 5.5 (1M context) --- backend/windmill-common/src/workspaces.rs | 4 ++-- frontend/src/lib/hubPaths.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index f819c5b796..a02bd7f44e 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -184,7 +184,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/29151/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -192,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28993/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/29150/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2c46c48b2c..7e1f965c36 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28950/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28993/git-sync-init-repository-windmill", + "gitInitRepo": "hub/29150/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack", From aeeeb042f93df32214e00040d360f20e8ed8c98a Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 13 Oct 2026 08:56:31 +0200 Subject: [PATCH 58/60] fix: persist hidden languages in edit default scripts drawer (#11693) Co-authored-by: Claude Opus 5.5 (1M context) --- frontend/src/lib/components/DefaultScriptsInner.svelte | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index 48c7a8d9f7..29136e42f7 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -33,6 +33,10 @@ ;[norder[i], norder[i + 1]] = [norder[i + 1], norder[i]] } defaultScripts.update((s) => ({ ...s, order: norder })) + await persist() + } + + async function persist() { await WorkspaceService.editDefaultScripts({ workspace: $operatingWorkspace!, requestBody: $defaultScripts @@ -78,7 +82,7 @@ size="xs" color="red" checked={$defaultScripts?.hidden?.includes(lang)} - on:change={(e) => { + on:change={async (e) => { let toggled = e.detail if (toggled) { defaultScripts.update((s) => ({ @@ -91,6 +95,7 @@ hidden: (s?.hidden ?? []).filter((h) => h != lang) })) } + await persist() }} />
From cd502a5270e787aeb1e5cb14e5a0e0aad8bb97b3 Mon Sep 17 00:00:00 2001 From: AlexRV12 <71396855+AlexRV12@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:59:39 +0200 Subject: [PATCH 59/60] fix: resolve a new item's chosen name in session preview tools (#11688) Co-authored-by: Claude Opus 5.5 --- .../sessions/sessionRuntime.svelte.ts | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index b2a37f1ca0..2995366ae0 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -103,6 +103,7 @@ import { type RawAppRunsProvider, type RawAppScreenshotRequester } from '$lib/components/raw_apps/utils' +import { resolveGlobalDraftStoragePathByKind } from '$lib/components/copilot/chat/global/userDraftAdapter' import type { RawAppDomQuery, RawAppDomRequester, @@ -1120,7 +1121,7 @@ function createRuntime(session: Session): SessionRuntime { }, async requestRuntimeLogs(limit, appPath) { const path = - appPath ?? + (appPath && draftStoragePath(session, 'raw_app', appPath)) ?? activeDomAppPath ?? (runtimeLogRequesters.size === 1 ? [...runtimeLogRequesters.keys()][0] : undefined) if (path === undefined) { @@ -1156,7 +1157,7 @@ function createRuntime(session: Session): SessionRuntime { // its element's app even when another tab is now visible), else the // active preview, else the only one open. const path = - query.appPath ?? + (query.appPath && draftStoragePath(session, 'raw_app', query.appPath)) ?? activeDomAppPath ?? (domRequesters.size === 1 ? [...domRequesters.keys()][0] : undefined) if (path === undefined) return undefined @@ -1372,6 +1373,18 @@ export function removeSession(sessionId: string): void { deleteSessionState(sessionId) } +// The chat calls a new item by its chosen name while its draft is stored at +// `draft_`; resolve the name the way the chat's write tools do. +function draftStoragePath( + session: Session, + kind: 'script' | 'flow' | 'raw_app', + path: string +): string { + return session.workspace_id + ? resolveGlobalDraftStoragePathByKind(session.workspace_id, kind, path) + : path +} + // Register the global open_preview tool handler once at module load. It // dispatches to the *calling* session (sessionId threaded from the tool ctx), // falling back to the UI-active session only when none was passed — so a @@ -1387,7 +1400,10 @@ setOpenPreviewHandler(async ({ sessionId: callerSessionId, kind, path, mode }) = if (!session) { return 'Error: no active session to open the preview in.' } - const target = previewTargetForSessionTarget(kind, path) + const target = previewTargetForSessionTarget( + kind, + mode === 'view' || kind === 'pipeline' ? path : draftStoragePath(session, kind, path) + ) if (!target) { return `Error: ${kind} targets cannot be shown in the preview panel.` } From b57fd3e29a8ff3633a585a8cc07059d45c33c9f6 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Tue, 13 Oct 2026 09:27:24 +0200 Subject: [PATCH 60/60] chore: publish the chat sdk as @windmill-labs/chat-sdk (#11686) * chore: publish the chat sdk as @windmill-labs/chat-sdk * chore: publish the chat sdk with a dedicated npm token * chore: publish the chat sdk with npm trusted publishing --- .github/workflows/npm_on_release.yml | 11 +++++--- chat-sdk/README.md | 28 +++++++++---------- chat-sdk/package-lock.json | 4 +-- chat-sdk/package.json | 2 +- cli/src/guidance/skills.gen.ts | 8 +++--- system_prompts/auto-generated/prompts.ts | 8 +++--- .../auto-generated/skills/raw-app/SKILL.md | 8 +++--- system_prompts/base/raw-app.md | 8 +++--- 8 files changed, 40 insertions(+), 37 deletions(-) diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index 631ab53153..a2bd974daa 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -18,16 +18,19 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_chat_sdk: - runs-on: ubicloud-standard-8 + # Published by npm trusted publishing (OIDC), which npm accepts only from + # GitHub-hosted runners and npm >= 11.5.1: keep both, there is no token to fall back on. + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v7 with: - node-version: "20.x" + node-version: "24.x" registry-url: "https://registry.npmjs.org" - run: cd chat-sdk && npm ci && npm run build && npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/chat-sdk/README.md b/chat-sdk/README.md index 62a5467a47..006d5407d8 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -1,21 +1,21 @@ -# windmill-chat +# @windmill-labs/chat-sdk Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend or from a Windmill raw app. The library is headless: it runs the flow, follows the answer as it streams, keeps the conversation history, and hands you state to render. ``` -npm install windmill-chat +npm install @windmill-labs/chat-sdk ``` -No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for -`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. +No runtime dependencies. Optional peers: `react` for `@windmill-labs/chat-sdk/react`, `ai` for +`@windmill-labs/chat-sdk/ai-sdk`, `@assistant-ui/react` for `@windmill-labs/chat-sdk/assistant-ui`. | You build the UI with | Import | You get | |---|---|---| -| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | -| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | -| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | +| Vercel AI SDK `useChat`, AI Elements | `@windmill-labs/chat-sdk/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `@windmill-labs/chat-sdk/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `@windmill-labs/chat-sdk/react` or `@windmill-labs/chat-sdk` | a hook / a store with messages, status and actions | ## The flow @@ -31,7 +31,7 @@ so an AI agent step remembers earlier turns. The answer is: ```tsx import { useChat } from '@ai-sdk/react' -import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' +import { createWindmillChatTransport } from '@windmill-labs/chat-sdk/ai-sdk' const transport = createWindmillChatTransport({ baseUrl: 'https://app.windmill.dev', @@ -65,7 +65,7 @@ explanatory error. ```tsx import { AssistantRuntimeProvider } from '@assistant-ui/react' -import { useWindmillRuntime } from 'windmill-chat/assistant-ui' +import { useWindmillRuntime } from '@windmill-labs/chat-sdk/assistant-ui' export function Support() { const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) @@ -84,7 +84,7 @@ through `Reasoning`. It takes the same options as `useWindmillChat` below. ## React ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' export function Support() { const chat = useWindmillChat({ @@ -155,7 +155,7 @@ change, and returns the unsubscribe function. That is the Svelte store contract, `$chat` works as is; other frameworks wrap it in a few lines. ```ts -import { createChat } from 'windmill-chat' +import { createChat } from '@windmill-labs/chat-sdk' const chat = createChat({ baseUrl, workspace, flowPath, token }) chat.subscribe((state) => render(state)) @@ -164,7 +164,7 @@ await chat.sendMessage('Hello') ```svelte @@ -331,8 +331,8 @@ custom integrations. ## For AI coding agents When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode -on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, -assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +on. Pick the entry point from the table at the top (`useChat` → `@windmill-labs/chat-sdk/ai-sdk`, +assistant-ui → `@windmill-labs/chat-sdk/assistant-ui`, otherwise `@windmill-labs/chat-sdk/react`). Inside a Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a `token`; for a public page use a token scoped to `jobs:run:flows:` and leave `history` at its default. Render `role`, `content`, `pending`, `success` and diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index 03a55cff86..74f6914f37 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,11 +1,11 @@ { - "name": "windmill-chat", + "name": "@windmill-labs/chat-sdk", "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "windmill-chat", + "name": "@windmill-labs/chat-sdk", "version": "1.830.0", "license": "Apache-2.0", "devDependencies": { diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 96637c376c..26f5f785be 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,5 +1,5 @@ { - "name": "windmill-chat", + "name": "@windmill-labs/chat-sdk", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", "version": "1.830.0", "author": "Ruben Fiszel", diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 8491e837e3..ab4ef31c3e 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6747,16 +6747,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. \`\`\`tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() \`\`\` -\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +\`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -6916,7 +6916,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 1c6541e155..097a7af895 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -1024,16 +1024,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. \`\`\`tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() \`\`\` -\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +\`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -1182,7 +1182,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 8d7788dd94..7afcbd2969 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -378,16 +378,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() ``` -`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +`@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -547,4 +547,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index dea8ebd21e..23b09f1996 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -139,16 +139,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() ``` -`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +`@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -319,4 +319,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream.