diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 4780eb02eb..580a757c21 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -16,7 +16,7 @@ on: default: "global cli" models: description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for" - default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" + default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -28,7 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 62ac9abafe..cc72934156 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.827.0" + ".": "1.828.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b157fa61d..c958457c7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,28 @@ # Changelog +## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08) + + +### Features + +* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864)) +* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d)) +* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4)) +* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6)) +* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24)) + + +### Bug Fixes + +* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775)) +* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba)) +* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687)) + + +### Performance Improvements + +* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996)) + ## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07) diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 22af20d5c3..cf131337fd 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -148,6 +148,15 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-6-astra", }, }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, + }, { id: "gpt-6-sol", label: "GPT-6 Sol", diff --git a/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json new file mode 100644 index 0000000000..1946f55366 --- /dev/null +++ b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217" +} diff --git a/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json new file mode 100644 index 0000000000..036610e08c --- /dev/null +++ b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b" +} diff --git a/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json new file mode 100644 index 0000000000..ba1cc13fac --- /dev/null +++ b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e" +} diff --git a/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json new file mode 100644 index 0000000000..1c8dfcfa57 --- /dev/null +++ b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13" +} diff --git a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json b/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json deleted file mode 100644 index f105dd7825..0000000000 --- a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d" -} diff --git a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json b/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json deleted file mode 100644 index 0b166b32a4..0000000000 --- a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5" -} diff --git a/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json new file mode 100644 index 0000000000..59ed0e1767 --- /dev/null +++ b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d" +} diff --git a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json b/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json deleted file mode 100644 index 2358af807f..0000000000 --- a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665" -} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json deleted file mode 100644 index 7f6fe7a81c..0000000000 --- a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" -} diff --git a/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json new file mode 100644 index 0000000000..ba7a89ef73 --- /dev/null +++ b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8" +} diff --git a/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json new file mode 100644 index 0000000000..48963d9ef7 --- /dev/null +++ b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.id FROM v2_job_queue q\n JOIN native_retry_attempt n ON n.job_id = q.id\n JOIN v2_job r ON r.id = q.id\n JOIN v2_job a ON a.id = r.parent_job\n WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json deleted file mode 100644 index ae893b6791..0000000000 --- a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int8", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" -} diff --git a/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json new file mode 100644 index 0000000000..9ebf9289c2 --- /dev/null +++ b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET draining = true WHERE worker = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919" +} diff --git a/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json new file mode 100644 index 0000000000..9f51c69616 --- /dev/null +++ b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b" +} diff --git a/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json new file mode 100644 index 0000000000..e22953610d --- /dev/null +++ b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323" +} diff --git a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json similarity index 80% rename from backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json rename to backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json index 4ebad828df..1670a5009e 100644 --- a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json +++ b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1 AND j.parent_job IS NULL\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", + "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258" + "hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9" } diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json deleted file mode 100644 index 5f86fb97d8..0000000000 --- a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" -} diff --git a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json similarity index 86% rename from backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json rename to backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json index 583d2bcc6b..5ee5c7571d 100644 --- a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json +++ b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -107,6 +107,11 @@ "ordinal": 20, "name": "native_mode", "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "draining", + "type_info": "Bool" } ], "parameters": { @@ -138,8 +143,9 @@ true, true, true, + false, false ] }, - "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca" + "hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aeb7670d17..b5f3d9a306 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6685,9 +6685,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "inventory" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] @@ -9617,7 +9617,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.16+spec-1.1.0", ] [[package]] @@ -13622,9 +13622,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] @@ -13644,21 +13644,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.16+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722" dependencies = [ "indexmap 2.14.2", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.4+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333" dependencies = [ "winnow 1.0.4", ] @@ -14829,7 +14829,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -14918,7 +14918,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.827.0" +version = "1.828.0" dependencies = [ "async-stream", "async-trait", @@ -14952,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14965,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "argon2", @@ -15105,7 +15105,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15128,7 +15128,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15145,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15171,7 +15171,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.827.0" +version = "1.828.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15181,7 +15181,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15198,7 +15198,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15259,7 +15259,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15281,7 +15281,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15303,7 +15303,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -15353,7 +15353,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15379,7 +15379,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15407,7 +15407,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15429,7 +15429,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15449,7 +15449,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15491,7 +15491,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15520,7 +15520,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.827.0" +version = "1.828.0" dependencies = [ "lazy_static", "serde", @@ -15532,7 +15532,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.827.0" +version = "1.828.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15556,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15570,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.827.0" +version = "1.828.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15609,7 +15609,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.827.0" +version = "1.828.0" dependencies = [ "chrono", "lazy_static", @@ -15624,7 +15624,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15643,7 +15643,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.827.0" +version = "1.828.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15757,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.827.0" +version = "1.828.0" dependencies = [ "chrono", "futures", @@ -15777,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.827.0" +version = "1.828.0" dependencies = [ "regex", "serde", @@ -15794,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15821,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "futures", @@ -15838,7 +15838,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.827.0" +version = "1.828.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15854,7 +15854,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -15875,7 +15875,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -15906,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "arc-swap", @@ -15931,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-stream", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "futures", @@ -15985,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.827.0" +version = "1.828.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15994,7 +15994,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16006,7 +16006,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16018,7 +16018,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "gosyn", @@ -16030,7 +16030,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16042,7 +16042,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16054,7 +16054,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "nu-parser", @@ -16065,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16076,7 +16076,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16088,7 +16088,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16099,7 +16099,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16121,7 +16121,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -16133,7 +16133,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16147,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16164,7 +16164,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16177,7 +16177,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde", @@ -16189,7 +16189,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16207,7 +16207,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16239,7 +16239,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -16253,7 +16253,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16292,7 +16292,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "const_format", @@ -16332,7 +16332,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.827.0" +version = "1.828.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16343,7 +16343,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -16378,7 +16378,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16402,7 +16402,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16435,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16462,7 +16462,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16495,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16515,7 +16515,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16549,7 +16549,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16586,7 +16586,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16609,7 +16609,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16633,7 +16633,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-nats", @@ -16657,7 +16657,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16692,7 +16692,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16720,7 +16720,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-trait", @@ -16745,7 +16745,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16764,7 +16764,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-once-cell", @@ -16883,7 +16883,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.827.0" +version = "1.828.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 06fdc001f0..e0b97275b2 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.827.0" +version = "1.828.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.827.0" +version = "1.828.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index fffcc99883..cc3c1a826c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -1600d49ad5513af8b54ad65688d3d1c7edc977d9 \ No newline at end of file +017710bdd0a10de857f0f40800dde238cdb1ad24 diff --git a/backend/migrations/20261008070335_worker_ping_draining.down.sql b/backend/migrations/20261008070335_worker_ping_draining.down.sql new file mode 100644 index 0000000000..64a5f2949d --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.down.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining; diff --git a/backend/migrations/20261008070335_worker_ping_draining.up.sql b/backend/migrations/20261008070335_worker_ping_draining.up.sql new file mode 100644 index 0000000000..33b5ad5e3e --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.up.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 99d82bb43b..1bdbdd6097 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.827.0" +version = "1.828.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.827.0" +version = "1.828.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.827.0" +version = "1.828.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.827.0" +version = "1.828.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 9ac7f81c7f..b2fd850217 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.827.0" +version = "1.828.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 2a518a7cc8..284a822b47 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -219,7 +219,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) -worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) +worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool) workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 4f58685221..b1f740566f 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its failure must reach the workspace error handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_error_handler_triggered_on_hub_script_failure( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000002"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/fail"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(!completed_job.success, "Hub script should have failed"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace error handler should handle the failed hub script" + ); + + Ok(()) +} diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index f1e042037f..30585c08a1 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() { otel_set_worker_uptime("w-uptime", 3600.0); otel_set_health_status_phase("healthy"); otel_set_health_db_unresponsive(true); + otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024)); + otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35); + otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25); + otel_record_job_memory_peak("python3", 100 * 1024 * 1024); // ── Histograms ────────────────────────────────────────────────── @@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 21 metric names are present ────────────────────── + // ── Verify all metric names are present ───────────────────────── let expected = [ "windmill.queue.push_count", @@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() { "windmill.worker.uptime", "windmill.health.status", "windmill.health.db_unresponsive", + "windmill.worker.memory_usage", + "windmill.worker.windmill_memory_usage", + "windmill.worker.memory_limit", + "windmill.worker.cpu_usage", + "windmill.worker.cpu_periods", + "windmill.worker.cpu_throttled_periods", + "windmill.worker.cpu_throttled_time", + "windmill.worker.occupancy_rate", + "windmill.worker.job_memory_peak", ]; for name in expected { assert!( @@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() { .expect("worker.busy data point with worker=worker-test-1 not found"); assert_eq!(dp.1, 1); + let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu") + }) + .expect("worker.memory_limit data point with worker_group=gpu not found"); + assert_eq!(dp.1, 1024); + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); assert_eq!(gauge_i64_values(m)[0].1, 5); let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index 8fc4182d9e..f876c1984b 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow: Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its success must reach the workspace success handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_hub_script_success( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000003"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/ok"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(completed_job.success, "Hub script should have succeeded"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace success handler should handle the successful hub script" + ); + + Ok(()) +} diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs index 5a83730d24..ca364e5993 100644 --- a/backend/tests/wac_suspend_started_at.rs +++ b/backend/tests/wac_suspend_started_at.rs @@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Res Ok(()) } + +/// Each park clears `started_at`, so the completion would otherwise store only the last +/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that +/// started an hour late. The completed row spans the workflow from its first start, and +/// keeps the last segment aside: that is what is metered and what worker-time readers sum. +#[sqlx::test] +async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment( + db: Pool, +) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \ + 'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \ + VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Two parks, each followed by the pull re-stamping `started_at`: the second park must + // not replace the first start with its own segment's. + for resumed_ago in ["10 minutes", "3 seconds"] { + let mut tx = db.begin().await?; + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?; + tx.commit().await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \ + WHERE id = $1", + ) + .bind(job_id) + .bind(resumed_ago) + .execute(&db) + .await?; + } + + let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db) + .await? + .unwrap(); + let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?; + let (_, span) = windmill_queue::add_completed_job( + &db, + &job, + true, + false, + sqlx::types::Json(&result), + None, + 0, + None, + false, + Some(3000), + false, + ) + .await?; + + let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option) = sqlx::query_as( + "SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \ + (extras->>'wac_last_segment_ms')::bigint \ + FROM v2_job_completed WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + assert!( + (started_ago_s - 3600.0).abs() < 60.0, + "started_at must be the first start, got {started_ago_s}s ago" + ); + assert!( + (duration_ms - 3_600_000).abs() < 60_000, + "duration_ms must be the wall time since the first start, got {duration_ms}" + ); + assert_eq!(last_segment_ms, Some(3000)); + assert_eq!( + span.duration_ms, duration_ms, + "a flow records the step with the span the row holds" + ); + assert!(span.outlasts_run(job.started_at)); + assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter"); + + Ok(()) +} diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs index 772f4193c3..2421e888ef 100644 --- a/backend/tests/worker_ping_ip.rs +++ b/backend/tests/worker_ping_ip.rs @@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres}; use uuid::Uuid; use windmill_common::{ external_ip::UNKNOWN_IP, - worker::{insert_ping_query, update_worker_ping_main_loop_query}, + worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query}, }; async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + register(db, worker, ip, None).await +} + +async fn register( + db: &Pool, + worker: &str, + ip: Option<&str>, + draining: Option, +) -> anyhow::Result<()> { insert_ping_query( "test-instance", worker, @@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> any None, None, false, + draining, db, ) .await?; Ok(()) } +/// A worker re-registers mid-life once its external IP resolves. That registration must not +/// clear a draining flag the same process already published; only a worker starting up under +/// a reclaimed name resets it. +#[sqlx::test] +async fn only_a_starting_worker_resets_draining(db: Pool) -> anyhow::Result<()> { + let draining = || async { + sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'") + .fetch_one(&db) + .await + }; + register(&db, "wk-drain", None, Some(false)).await?; + set_worker_draining_query("wk-drain", &db).await?; + register(&db, "wk-drain", Some("1.2.3.4"), None).await?; + assert!(draining().await?); + + register(&db, "wk-drain", None, Some(false)).await?; + assert!(!draining().await?); + Ok(()) +} + /// The external IP resolves in the background, so the initial ping often has none yet. That must /// not blank the address a previous process wrote to the row this one reclaims — worker names are /// stable across restarts under EXIT_AFTER_N_JOBS. diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index e08b038bb0..7dcddc93a3 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -62,9 +62,13 @@ const REASONING_RULES: &[ReasoningRule] = &[ can_disable: false, completions_tools_need_off: false, }, - // Live-verified: astra takes low..max only, where sol and luna also take `none`. + // Live-verified: gpt-6-astra and gpt-6.1-sol take low..max only, where gpt-6-sol and + // gpt-6-luna also take `none`. ReasoningRule { - matches: |m| base_id(m).starts_with("gpt-6-astra"), + matches: |m| { + let base = base_id(m); + base.starts_with("gpt-6-astra") || base.starts_with("gpt-6.1-sol") + }, can_disable: false, completions_tools_need_off: true, }, diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index dce6a0ff72..b636e11a1c 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -996,6 +996,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui ["job_metrics", "set_progress", ..] => (http_method == "POST", 2), ["job_metrics", "get_progress", ..] => (http_method == "GET", 2), ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2), + ["jobs", "worker_is_draining", ..] => (http_method == "GET", 2), ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3), ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3), _ => return false, diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index ddb94f8c2b..9800d97841 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -417,6 +417,89 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_existing_role_is_taken_over_only_when_asked_and_never_with_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let plain = format!("wmtest_plain_{suffix}"); + let group = format!("wmtest_group_{suffix}"); + let member = format!("wmtest_member_{suffix}"); + for sql in [ + // An expired password and no connection slot: kept as is, the login Windmill sets is dead. + format!( + "CREATE ROLE \"{plain}\" LOGIN PASSWORD 'theirs' VALID UNTIL '2000-01-01' \ + CONNECTION LIMIT 0" + ), + format!("CREATE ROLE \"{group}\" NOLOGIN"), + format!("CREATE ROLE \"{member}\" LOGIN"), + format!("GRANT \"{group}\" TO \"{member}\""), + ] { + sqlx::query(&sql).execute(&db).await?; + } + + let create = |name: String, take_over: bool| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name, "take_over": take_over })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status().as_u16(), resp.text().await?)) + }; + let outcome = async { + let (status, body) = create(plain.clone(), false).await?; + assert_eq!( + status, 409, + "an existing role is not taken without asking: {body}" + ); + + // The group's member would inherit every grant the data table role gets. + let (status, body) = create(group.clone(), true).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains(&member), + "the refusal names the member: {body}" + ); + + let (status, body) = create(plain.clone(), true).await?; + assert_eq!(status, 200, "{body}"); + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; + let taken = catalog + .values() + .find(|r| r.name == plain) + .ok_or_else(|| anyhow::anyhow!("{plain} is not in the catalog: {catalog:?}"))?; + let options = (*db.connect_options()) + .clone() + .username(&plain) + .password(taken.pwd.as_deref().unwrap_or_default()); + let mut login = ::connect_with(&options).await?; + sqlx::query("SELECT 1").execute(&mut login).await?; + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide and outlive this test's database, whatever happened above. + for name in [&member, &group, &plain] { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + #[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs index c3da2f3fb9..2c5d2ee78e 100644 --- a/backend/windmill-api-integration-tests/tests/jobs_authed.rs +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue( Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_worker_is_draining_follows_the_job_worker(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker) + VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version) + VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')", + ) + .execute(&db) + .await?; + + let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}"); + let draining = || async { + let resp = authed(client().get(&url)).send().await?; + anyhow::Ok(resp.json::().await?) + }; + assert!(!draining().await?); + + windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?; + assert!(draining().await?); + + // A parked job keeps its last worker on the queue row without running on it. + sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1") + .bind(job) + .execute(&db) + .await?; + assert!(!draining().await?); + + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 5e08949138..ce6cfc185d 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -33,6 +33,7 @@ use windmill_common::{ flow_conversations::{ add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType, }, + flows::Retry, get_latest_flow_version_info_for_path, jobs::{ check_tag_available_for_workspace_internal, format_result, script_path_to_payload, @@ -126,6 +127,71 @@ pub async fn drop_unclaimable_run_lineage( Ok(()) } +/// Applies the run's `retry` policy by handing `push` a one-step-flow request, which it +/// materializes as a native retryable `Script`. +pub fn with_run_retry( + run_query: &RunJobQuery, + payload: JobPayload, + args: &PushArgs<'_>, + tag: &Option, +) -> error::Result { + let Some(retry) = run_query.retry.as_deref() else { + return Ok(payload); + }; + // Retry attempts are always pushed visible to the owner, which would leak a private run. + if run_query.invisible_to_owner.unwrap_or(false) { + return Err(Error::BadRequest( + "retry cannot be combined with invisible_to_owner".to_string(), + )); + } + let retry: Retry = serde_json::from_str(retry) + .map_err(|e| Error::BadRequest(format!("invalid retry policy: {e}")))?; + let JobPayload::ScriptHash { + hash, + path, + cache_ttl, + cache_ignore_s3_path, + language, + priority, + apply_preprocessor, + debouncing_settings, + concurrency_settings, + .. + } = payload + else { + return Err(Error::BadRequest( + "retry is only supported for workspace scripts".to_string(), + )); + }; + // A failed attempt keeps its raw args and the retry re-push skips the preprocessor, so + // every retry would call `main` with un-preprocessed input. + if apply_preprocessor { + return Err(Error::BadRequest( + "retry is not supported for a script with a preprocessor unless skip_preprocessor is set" + .to_string(), + )); + } + Ok(JobPayload::SingleStepFlow { + path, + hash: Some(hash), + flow_version: None, + language: Some(language), + args: HashMap::from(args), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl, + cache_ignore_s3_path, + priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor, + concurrency_settings, + debouncing_settings, + }) +} + /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for /// a workspace admin anything outside the workspace. A restricted job token never gets the admin @@ -1151,6 +1217,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( let tag = run_query.tag.clone().or(tag); let push_args = PushArgs { args: &args.args, extra: args.extra }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; let return_tx = tx_o.is_some(); diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index f37040bd11..f50cb61451 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -52,6 +52,8 @@ pub struct RunJobQuery { pub trigger_external_id: Option, pub service_name: Option, pub suspended_mode: Option, + /// A JSON-encoded `Retry` policy for a script run; see `with_run_retry`. + pub retry: Option, } impl RunJobQuery { diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index c79b596617..f9f0a993fa 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -58,10 +58,10 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, - AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXTERNAL_INSTANCE_PG_SETTING, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -1238,26 +1238,22 @@ async fn run_setting_pre_write_hook( } } } - GITHUB_APP_WEBHOOK_BASE_URL_SETTING => { - // A bad value here yields a webhook GitHub can never deliver to, and the + GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => { + // A bad webhook base yields a webhook GitHub can never deliver to, and the // failure only shows up much later as "falling back to polling" on a - // repository — so reject it at the boundary instead. + // repository; a bad API base puts an unreachable URL in front of every user. + // So reject both at the boundary instead. match value { // Clearing (delete row) is handled by the caller; allow it through. serde_json::Value::Null => {} serde_json::Value::String(s) if s.trim().is_empty() => {} serde_json::Value::String(s) => { - windmill_common::global_settings::validate_webhook_base_url(s).map_err( - |e| { - error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}" - )) - }, - )?; + windmill_common::global_settings::validate_webhook_base_url(s) + .map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?; } _ => { return Err(error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string" + "{key} must be a URL string" ))); } } @@ -1394,8 +1390,12 @@ async fn set_instance_config( // Applied and audited one setting at a time: the writes are not transactional, so a // batch that fails midway must not leave the settings it did write unrecorded. - let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } = - settings_diff; + let instance_config::SettingsDiff { + upserts, + deletes, + mut previous_values, + unchanged_count, + } = settings_diff; let steps = upserts .into_iter() .map(|(k, v)| (k, Some(v))) diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ac9defcd53..46bd6825e8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -490,7 +490,7 @@ async fn list_user_usage( " SELECT usr.email, usage.executions FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 0102f1b033..cf0b85609a 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -86,6 +86,7 @@ struct WorkerPing { job_isolation: Option, #[serde(skip_serializing_if = "Option::is_none")] native_mode: Option, + draining: bool, } // #[derive(Serialize, Deserialize)] @@ -118,7 +119,7 @@ async fn list_worker_pings( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index ebab6214ce..efaa712481 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -683,6 +683,9 @@ pub(crate) struct CatalogFacts { pub(crate) former_owner: Option, /// For a revoke: each grant it takes back. pub(crate) revoked_grants: Vec, + /// For a grant on what a schema holds: the role cannot reach the schema yet, so the grant + /// would give it nothing it can use. + pub(crate) lacks_schema_usage: bool, } /// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the @@ -1526,7 +1529,24 @@ async fn build_plan( ))); } } - let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let lacks_schema_usage = match (&change, target) { + (AclChange::Grant { scope, .. }, AclTarget::Schema { schema }) + if !matches!(scope, GrantScope::Target) => + { + !has_schema_usage(client, schema, &pg_role).await? + } + (AclChange::Grant { .. }, AclTarget::Table { schema, .. }) => { + !has_schema_usage(client, schema, &pg_role).await? + } + _ => false, + }; + let facts = CatalogFacts { + other_pg_roles, + existing_objects, + former_owner, + revoked_grants, + lacks_schema_usage, + }; let mut plan = crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; if matches!(change, AclChange::SetOwner { .. }) { @@ -1540,6 +1560,27 @@ async fn build_plan( Ok(plan) } +/// Whether `pg_role` may look into `schema`, directly or through a role it inherits from. +async fn has_schema_usage( + client: &tokio_postgres::Client, + schema: &str, + pg_role: &str, +) -> Result { + Ok(client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'USAGE')", + &[&pg_role, &schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the role's privileges on schema {schema}: {}", + pg_error_message(&e) + )) + })? + .get(0)) +} + /// Postgres only hands an object to a role that could have created it: a table to one with /// `CREATE` on its schema, a schema to one with `CREATE` on the database. async fn missing_owner_privilege( diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 8dae1bafcd..4016222d30 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -34993,7 +34993,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -35006,7 +35006,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -47930,7 +47930,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -47943,7 +47943,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 439e4d11a0..d541144231 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -11725,9 +11725,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: >- - Retry with exponential backoff (delay - doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -11738,7 +11736,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -23272,7 +23270,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -23283,7 +23281,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 51d2fbcb81..b6ce4ced20 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.827.0 + version: 1.828.0 title: Windmill API contact: @@ -1765,6 +1765,12 @@ paths: type: string cluster: $ref: "#/components/schemas/DatatableRoleCluster" + take_over: + type: boolean + description: > + manage a Postgres role of that name that already exists on the cluster, + replacing its password, instead of refusing it. A role with cluster-wide + attributes or memberships is refused either way responses: "200": description: the created role @@ -1772,6 +1778,8 @@ paths: application/json: schema: $ref: "#/components/schemas/InstanceDatatableRole" + "409": + description: a Postgres role of that name already exists on the cluster and take_over was not set /settings/datatable_roles/{id}: post: @@ -1911,7 +1919,7 @@ paths: /settings/instance_ui: get: - summary: get the instance-wide announcement banner and accent color + summary: get the instance-wide announcement banner, accent color and API base url description: | Readable by any authenticated user; every page load reads it. Each value is the raw stored setting, or null when unset. @@ -1920,7 +1928,7 @@ paths: - setting responses: "200": - description: the instance_banner and accent_color settings + description: the instance_banner, accent_color and api_base_url settings content: application/json: schema: @@ -1928,6 +1936,7 @@ paths: properties: instance_banner: {} accent_color: {} + api_base_url: {} /settings_u/ruff_config: get: @@ -12004,6 +12013,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -16053,6 +16063,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) @@ -17357,6 +17368,27 @@ paths: schema: type: string + /w/{workspace}/jobs/worker_is_draining/{id}: + get: + summary: whether the worker running this job is draining + description: | + True once the worker running the job has received its shutdown signal. The worker + still waits for the job to finish, so a long-running job can poll this to exit early. + False when the job is not currently running on a worker. + operationId: workerIsDraining + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: whether the worker running this job is draining + content: + application/json: + schema: + type: boolean + /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs @@ -27867,6 +27899,12 @@ components: in: query schema: type: string + RunRetry: + name: retry + description: JSON-encoded retry policy (same shape as a schedule's retry) re-running the job when it fails. The returned id is the first attempt's; each retry is a job of its own, recorded as a child of the first attempt. Not supported for hub scripts, with invisible_to_owner, or for a script whose preprocessor would run. + in: query + schema: + type: string NewJobId: name: job_id description: @@ -34627,6 +34665,11 @@ components: type: boolean cluster: $ref: "#/components/schemas/DatatableRoleCluster" + in_use: + type: boolean + description: >- + Whether any data table, in any workspace, lets someone connect as this role. Only set + in a data table's available_roles. DatatableRoleTenants: type: object @@ -35982,6 +36025,9 @@ components: type: string native_mode: type: boolean + draining: + type: boolean + description: the worker received its shutdown signal and exits once its current job is done required: - worker - worker_instance diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f276e6d143..26b369ce74 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -979,7 +979,14 @@ async fn get_raw_app_data( // the opaque, sandboxed wrapper (incl. under a cross-origin-isolated // / COEP `require-corp` embedder). .header("X-Content-Type-Options", "nosniff") - .header("Cross-Origin-Resource-Policy", "cross-origin"); + .header("Cross-Origin-Resource-Policy", "cross-origin") + // The secret names one app version, whose bundle is written once at deploy, + // so the URL's content never changes. `private`: the URL is a capability, so + // shared caches must not keep serving it after the app is deleted. + .header( + http::header::CACHE_CONTROL, + "private, max-age=31536000, immutable", + ); Ok(res.body(body).unwrap()) } else { return Err(Error::NotFound("File not found".to_string())); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b08b083695..5b0da21c50 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -378,6 +378,10 @@ pub fn workspaced_service() -> Router { "/wac_approval_urls/{job_id}/{step_key}", get(get_wac_approval_urls).layer(cors.clone()), ) + .route( + "/worker_is_draining/{id}", + get(worker_is_draining).layer(cors.clone()), + ) .route( "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), @@ -613,6 +617,53 @@ async fn get_root_job( Ok(Json(res)) } +/// Whether the worker running this job has received its shutdown signal. A draining worker +/// never interrupts its job, so a long-running script polls this to exit on its own terms. +async fn worker_is_draining( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> windmill_common::error::JsonResult { + // A job always may ask about itself. Anyone else must be able to see the job: the + // caller's row policies on v2_job decide, narrowed by the token's tag filter. + let draining = if authed.job_id == Some(id) { + sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker + WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + id, + w_id + ) + .fetch_optional(&db) + .await? + } else { + let tags = get_scope_tags(&authed) + .map(|tags| tags.into_iter().map(str::to_string).collect::>()); + let mut tx = user_db.begin(&authed).await?; + let draining = sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job j + JOIN v2_job_queue q ON q.id = j.id + JOIN worker_ping wp ON wp.worker = q.worker + WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL + AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_deref() + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + draining + } + .unwrap_or(false); + windmill_common::feature_usage::log_feature_usage( + "worker_draining", + "check", + if draining { "draining" } else { "not_draining" }, + ); + Ok(Json(draining)) +} + async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, @@ -8541,6 +8592,11 @@ pub async fn stream_job( run_query: RunJobQuery, is_get: bool, ) -> error::Result { + if run_query.retry.is_some() { + return Err(error::Error::BadRequest( + "retry is only supported when running a script asynchronously".to_string(), + )); + } let args = if is_get { let payload_as_args = run_query.payload_as_args()?; @@ -10483,26 +10539,29 @@ pub async fn run_job_by_hash_inner( ) }; + let job_payload = JobPayload::ScriptHash { + hash: ScriptHash(hash), + path: path, + concurrency_settings, + debouncing_settings, + cache_ttl, + cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + labels, + job_token_scopes, + }; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; + let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, &w_id, - JobPayload::ScriptHash { - hash: ScriptHash(hash), - path: path, - concurrency_settings, - debouncing_settings, - cache_ttl, - cache_ignore_s3_path, - language, - dedicated_worker, - priority, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - labels, - job_token_scopes, - }, + job_payload, push_args, authed.display_username(), email, diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 1962710578..73b1fd1792 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -2024,7 +2024,7 @@ You should get the schema of the script or flow before creating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2037,7 +2037,7 @@ You should get the schema of the script or flow before creating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -2231,7 +2231,7 @@ You should get the schema of the script or flow before updating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2244,7 +2244,7 @@ You should get the schema of the script or flow before updating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index f355e733cb..e7e490a6a4 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -331,6 +331,12 @@ pub async fn converge_connect_grants_with( /// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user` on `cluster`. No /// privileges beyond that — an admin grants them through SQL or the ACL editor. /// +/// A role of that name that already exists is refused with a 409, unless `take_over`: then its +/// password is replaced and it is managed from here on like one created here. A role holding +/// cluster-wide privileges, or inheriting another role's, is refused either way, since every tenant +/// would get them. Returns whether an existing role was taken over: undoing a failed take-over +/// must not drop a role that was there before. +/// /// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external /// cluster is another server: the role is created there before `tx` commits, and callers MUST drop /// it again ([`drop_datatable_role`]) if `tx` then fails to commit. @@ -343,8 +349,10 @@ pub async fn create_datatable_role( cluster: DatatableRoleCluster, name: &str, password: &str, -) -> Result<()> { - crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await + take_over: bool, +) -> Result { + crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password, take_over) + .await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index a97a936827..33d711e7f9 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -125,7 +125,8 @@ mod ce { _cluster: DatatableRoleCluster, _name: &str, _password: &str, - ) -> Result<()> { + _take_over: bool, + ) -> Result { Err(unavailable()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index e1a731f3e6..1cd3609820 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -181,6 +181,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP +/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on. +/// For deployments that serve the API on a separate domain or behind a gateway. +/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]: +/// every user is shown URLs built from it. +pub const API_BASE_URL_SETTING: &str = "api_base_url"; /// Instance-wide announcement rendered above every page of the app (maintenance /// windows, incidents). Readable by any authenticated user, unlike most settings: /// the banner exists to be shown to everyone, so it must never hold anything the @@ -313,12 +319,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { pub struct InstanceUi { pub instance_banner: Option, pub accent_color: Option, + pub api_base_url: Option, } pub async fn get_instance_ui(db: &Pool) -> error::Result { let rows = sqlx::query!( "SELECT name, value FROM global_settings WHERE name = ANY($1)", - &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + &[ + INSTANCE_BANNER_SETTING, + ACCENT_COLOR_SETTING, + API_BASE_URL_SETTING + ] as &[&str] ) .fetch_all(db) .await?; @@ -327,17 +338,22 @@ pub async fn get_instance_ui(db: &Pool) -> error::Result { match row.name.as_str() { INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + API_BASE_URL_SETTING => ui.api_base_url = Some(row.value), _ => {} } } Ok(ui) } -/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. +const SHELL_ACTIVE_CHARS: [char; 16] = [ + '$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!', +]; + +/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value. /// -/// The receiver path is appended to it verbatim, so anything that doesn't -/// concatenate into a URL GitHub can POST to must be rejected at write time -/// rather than silently producing an unreachable hook: a wrong scheme +/// A path is appended to it verbatim, so anything that doesn't concatenate into a +/// URL an external caller can reach must be rejected at write time rather than +/// silently producing an unreachable hook or endpoint: a wrong scheme /// (`httpss://`), a missing host, embedded whitespace, or a query/fragment /// (appending a path after `?`/`#` keeps it inside the query/fragment). /// @@ -353,7 +369,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?; if !url.username().is_empty() || url.password().is_some() { return Err( - "must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(), + "must not embed a username or password: the URL is readable by users who are not instance admins".to_string(), ); } if !matches!(url.scheme(), "http" | "https") { @@ -366,13 +382,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { } if url.query().is_some() || url.fragment().is_some() { return Err( - "must not include a query string or fragment, since the webhook path is appended to it" + "must not include a query string or fragment, since a path is appended to it" .to_string(), ); } if value.chars().any(char::is_whitespace) { return Err("must not contain whitespace".to_string()); } + // The UI pastes the URL into shell commands users copy, where double quotes + // do not stop `$(...)` or backticks from running. + if value.contains(SHELL_ACTIVE_CHARS) { + return Err( + "must not contain shell metacharacters, since it is shown inside commands users copy" + .to_string(), + ); + } // Matches the sibling `base_url` / `hub_base_url` convention. The receiver // builder trims it anyway, so this is about keeping the stored value canonical // rather than about reachability. @@ -1101,6 +1125,9 @@ mod tests { "https://hooks.example.com#", "https://user:password@hooks.example.com", "https://user@hooks.example.com", + "https://hooks.example.com/$(uname)", + "https://hooks.example.com/`uname`", + "https://hooks.example.com/a;b", ]; for v in accept { assert!( diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 700ba71df5..bd7c28628c 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -281,6 +281,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub api_base_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub email_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub hub_base_url: Option, @@ -1375,8 +1377,8 @@ pub fn diff_worker_configs( } /// Declaratively replace the global settings, rejecting a -/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the -/// API would reject. +/// `github_app_webhook_base_url`, `api_base_url` or +/// `http_route_default_allowed_origins` the API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1399,29 +1401,35 @@ pub async fn sync_global_settings_declarative( current: &BTreeMap, desired: &BTreeMap, ) -> anyhow::Result<()> { - let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING; - // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a - // bool/number/object through as if the key were absent, and the diff below would - // then persist it — where the HTTP path answers "must be a URL string". - match desired.get(webhook_key) { - None | Some(serde_json::Value::Null) => {} - Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} - Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s) - .map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?, - // Names the JSON kind rather than printing it: this is the last message on - // this path that could report submitted content, and an object or array could - // carry a secret into `sync-config` output and operator logs. - Some(other) => { - let kind = match other { - serde_json::Value::Bool(_) => "a boolean", - serde_json::Value::Number(_) => "a number", - serde_json::Value::Array(_) => "an array", - serde_json::Value::Object(_) => "an object", - _ => "a non-string value", - }; - return Err(anyhow::anyhow!( - "{webhook_key} must be a URL string, got {kind}" - )); + for url_key in [ + crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, + crate::global_settings::API_BASE_URL_SETTING, + ] { + // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a + // bool/number/object through as if the key were absent, and the diff below would + // then persist it — where the HTTP path answers "must be a URL string". + match desired.get(url_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(serde_json::Value::String(s)) => { + crate::global_settings::validate_webhook_base_url(s) + .map_err(|e| anyhow::anyhow!("{url_key}: {e}"))? + } + // Names the JSON kind rather than printing it: this is the last message on + // this path that could report submitted content, and an object or array could + // carry a secret into `sync-config` output and operator logs. + Some(other) => { + let kind = match other { + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + _ => "a non-string value", + }; + return Err(anyhow::anyhow!( + "{url_key} must be a URL string, got {kind}" + )); + } } } diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index bd0a5202df..95e635dd8f 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -132,3 +132,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {} #[cfg(not(feature = "private"))] pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_memory( + _worker: &str, + _worker_group: &str, + _usage: Option, + _windmill_usage: Option, + _limit: Option, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_add_worker_cpu( + _worker: &str, + _worker_group: &str, + _usage_secs: f64, + _periods: u64, + _throttled_periods: u64, + _throttled_secs: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_occupancy_rate( + _worker: &str, + _worker_group: &str, + _window: &'static str, + _rate: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1feb4182ab..b3774cdda4 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -14,7 +14,7 @@ use crate::utils::WarnAfterExt; use crate::worker::Connection; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; -use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; @@ -375,13 +375,15 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { mc.encrypt_str_to_base64(value) } +const DECRYPT_FAILED: &str = + "Could not decrypt value. The value may have been encrypted with a different key."; + +/// Every failure returns the same message so responses carry no signal about why decryption +/// failed (padding, base64, UTF-8); the cause is only logged. pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { - mc.decrypt_base64_to_string(value).map_err(|e| match e { - MagicCryptError::DecryptError(_) => error::Error::internal_err( - "Could not decrypt value. The value may have been encrypted with a different key." - .to_string(), - ), - _ => error::Error::internal_err(e.to_string()), + mc.decrypt_base64_to_string(value).map_err(|e| { + tracing::warn!("decryption failed: {e}"); + error::Error::internal_err(DECRYPT_FAILED.to_string()) }) } @@ -746,7 +748,31 @@ pub async fn get_variable_or_self_as( #[cfg(test)] mod tests { - use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier}; + use super::{ + can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier, + DECRYPT_FAILED, + }; + use crate::error::Error; + use magic_crypt::MagicCryptTrait; + + #[test] + fn decrypt_failures_all_return_the_same_message() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let failures = [ + "not base64 at all!".to_string(), + mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), + other.encrypt_str_to_base64("secret"), + ]; + for value in failures { + match decrypt(&mc, value.clone()) { + Err(Error::InternalErrLoc { message, .. }) => { + assert_eq!(message, DECRYPT_FAILED, "for {value:?}") + } + other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"), + } + } + } #[test] fn is_valid_js_identifier_gates_prologue_injection() { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index e185d1d445..017c244670 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1783,6 +1783,73 @@ pub fn get_cpu_period() -> Option { Some(100000) } +/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job +/// peak-memory histograms so the two exports stay comparable. +pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = { + let mut buckets = [0.0; 12]; + let mut i = 0; + while i < 12 { + buckets[i] = (16u64 << (20 + i)) as f64; + i += 1; + } + buckets +}; + +/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0 +/// when the cgroup has no CPU controller, which is also what an unthrottled one reports. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct CpuStat { + pub usage_usec: u64, + pub nr_periods: u64, + pub nr_throttled: u64, + pub throttled_usec: u64, +} + +fn cpu_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let (k, v) = line.split_once(' ')?; + (k == key).then(|| v.trim().parse().ok()).flatten() + }) +} + +fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpu_stat_field(stat, "usage_usec")?, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0), + }) +} + +/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds. +fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpuacct_usage.trim().parse::().ok()? / 1000, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000, + }) +} + +#[cfg(not(windows))] +pub fn get_cpu_stat() -> Option { + const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage"; + if Path::new(V1_USAGE).exists() { + parse_cgroup_v1_cpu_stat( + &std::fs::read_to_string(V1_USAGE).ok()?, + &std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(), + ) + } else { + let cgroup_path = get_cgroupv2_path()?; + parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?) + } +} + +#[cfg(windows)] +pub fn get_cpu_stat() -> Option { + None +} + /// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count /// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds /// the quota in, so a fraction of a CPU makes it report a single-core machine. @@ -1952,6 +2019,7 @@ pub enum PingType { MainLoop, Job, InitScript, + Draining, } #[derive(Serialize, Deserialize)] pub struct Ping { @@ -1974,6 +2042,8 @@ pub struct Ping { pub occupancy_rate_30m: Option, pub job_isolation: Option, pub native_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draining: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -2026,6 +2096,7 @@ pub async fn update_ping_http( insert_ping.memory, insert_ping.job_isolation, insert_ping.native_mode.unwrap_or(false), + insert_ping.draining, db, ) .await?; @@ -2054,10 +2125,21 @@ pub async fn update_ping_http( ) .await? } + PingType::Draining => set_worker_draining_query(worker_name, db).await?, } Ok(()) } +pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET draining = true WHERE worker = $1", + worker_name + ) + .execute(db) + .await?; + Ok(()) +} + #[derive(Debug, Serialize, Deserialize)] pub struct JobCancelled { pub canceled_by: String, @@ -2156,6 +2238,9 @@ pub async fn fetch_raw_script_from_app_query( /// `started_at` and `jobs_executed` are the only two columns carried over, being the /// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP /// lookup has not resolved yet and the predecessor's address is still the best guess. +/// +/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering +/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime. pub async fn insert_ping_query( worker_instance: &str, worker_name: &str, @@ -2169,6 +2254,7 @@ pub async fn insert_ping_query( memory: Option, job_isolation: Option, native_mode: bool, + draining: Option, db: &DB, ) -> anyhow::Result { // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and @@ -2176,8 +2262,8 @@ pub async fn insert_ping_query( // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The // literal below must stay equal to `external_ip::UNKNOWN_IP`. let previous_jobs_executed = sqlx::query_scalar!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) - DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker) + DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining) RETURNING jobs_executed", worker_instance, worker_name, @@ -2191,6 +2277,7 @@ pub async fn insert_ping_query( memory, job_isolation.as_deref(), native_mode, + draining, ) .fetch_one(db) .await?; @@ -2899,6 +2986,32 @@ mod tests { use super::*; use std::collections::HashMap; + #[test] + fn cgroup_cpu_stat_is_normalized_to_microseconds() { + let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\ + nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n"; + let expected = CpuStat { + usage_usec: 2_500_000, + nr_periods: 40, + nr_throttled: 7, + throttled_usec: 350_000, + }; + assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected)); + + let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n"; + assert_eq!( + parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat), + Some(expected) + ); + + // No CPU controller: usage is still reported, throttling reads as zero. + assert_eq!( + parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"), + Some(CpuStat { usage_usec: 12, ..Default::default() }) + ); + assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None); + } + #[test] fn datatable_role_is_read_from_the_leading_comment_block() { let role = |code| SqlAnnotations::datatable_role(code); diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 207c8ee176..389ae1d112 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -359,6 +359,22 @@ ORDER BY depth, id .filter_map(|r| r.id.clone()) .collect_vec(); + // A native retry is the child of its first attempt, which has completed by the time the + // retry is queued, so the walk above stops before reaching it. + let tree = [&[job.id][..], &jobs_to_cancel[..]].concat(); + let pending_retries = sqlx::query_scalar!( + "SELECT q.id FROM v2_job_queue q + JOIN native_retry_attempt n ON n.job_id = q.id + JOIN v2_job r ON r.id = q.id + JOIN v2_job a ON a.id = r.parent_job + WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + tree.as_slice(), + w_id + ) + .fetch_all(&mut *tx) + .await?; + jobs_to_cancel.extend(pending_retries); + jobs_to_cancel.reverse(); if !jobs_to_cancel.is_empty() { tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len()); @@ -1205,7 +1221,7 @@ pub async fn add_completed_job_pre_shaped_failure, -) -> Result<(), Error> { +) -> Result { record_failure_metrics(completed_job, worker_name).await; tracing::error!( @@ -1213,7 +1229,7 @@ pub async fn add_completed_job_pre_shaped_failure, ) -> Result { + add_completed_job_error_with_span( + db, + completed_job, + mem_peak, + canceled_by, + e, + worker_name, + flow_is_done, + duration, + ) + .await + .map(|(wrapped, _)| wrapped) +} + +/// `add_completed_job_error`, also handing back what the completed row holds. +pub async fn add_completed_job_error_with_span( + db: &Pool, + completed_job: &MiniCompletedJob, + mem_peak: i32, + canceled_by: Option, + e: serde_json::Value, + worker_name: &str, + flow_is_done: bool, + duration: Option, +) -> Result<(WrappedError, CompletedSpan), Error> { record_failure_metrics(completed_job, worker_name).await; let result = WrappedError { error: e }; @@ -1250,7 +1291,7 @@ pub async fn add_completed_job_error( completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); - let _ = add_completed_job( + let (_, span) = add_completed_job( db, &completed_job, false, @@ -1265,7 +1306,7 @@ pub async fn add_completed_job_error( ) .warn_after_seconds(10) .await?; - Ok(result) + Ok((result, span)) } lazy_static::lazy_static! { @@ -1304,7 +1345,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, CompletedSpan), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1338,7 +1379,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { + let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1379,7 +1420,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, span)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1391,7 +1432,18 @@ pub async fn add_completed_job( .then(|| completed_job.parent_job) .flatten() } else if !success && !skipped && retry_pending { - Some(completed_job.parent_job.unwrap_or(completed_job.id)) + let is_retry_attempt = sqlx::query_scalar!( + "SELECT attempt FROM native_retry_attempt WHERE job_id = $1", + completed_job.id, + ) + .fetch_optional(db) + .await + .unwrap_or_else(|e| { + tracing::error!("reading the retry marker of {}: {e:#}", completed_job.id); + None + }) + .is_some(); + Some(native_retry_root(completed_job, is_retry_attempt)) } else { None }; @@ -1409,7 +1461,7 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, completed_job, duration); + apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( @@ -1426,7 +1478,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, span)) } async fn commit_completed_job( @@ -1444,7 +1496,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, bool)> { +) -> windmill_common::error::Result<(Option, CompletedSpan, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1511,11 +1563,11 @@ async fn commit_completed_job( && monitor_parent.is_none() && !is_scheduled { - let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else { + let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else { return Err(not_in_queue_error(db, job_id).await); }; - log_completed_job(completed_job, duration, success); - return Ok((None, duration, false, false)); + log_completed_job(completed_job, completed.duration_ms, success); + return Ok((None, completed, false, false)); } let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1523,7 +1575,8 @@ async fn commit_completed_job( // The parent's rows are locked ahead of the child's own queue row (see // `record_child_completion` for the order this must keep), so the duration it stamps is read // before the completion: the one the completed row will hold. `now()` is fixed for the - // transaction, and a completed row already there keeps its own duration. + // transaction, and a completed row already there keeps its own duration. For a child that + // itself parked it is the last run's, matching the last start its timeline entry carries. let mut wac_parent_ready = false; if let Some(parent_job) = wac_parent { let Some(duration) = sqlx::query_scalar!( @@ -1550,7 +1603,7 @@ async fn commit_completed_job( .await?; } - let Some(duration) = completion.execute(&mut *tx).await? else { + let Some(completed) = completion.execute(&mut *tx).await? else { return Err(not_in_queue_error(&mut *tx, job_id).await); }; @@ -1760,16 +1813,35 @@ async fn commit_completed_job( tx.commit().warn_after_seconds(10).await?; - log_completed_job(completed_job, duration, success); + log_completed_job(completed_job, completed.duration_ms, success); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); Ok(( None, - duration, + completed, _skip_downstream_error_handlers, wac_parent_ready, )) } +/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's. +/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered; +/// it differs from `duration_ms` only for a job that parked. +#[derive(Debug, Clone, Copy)] +pub struct CompletedSpan { + pub started_at: Option>, + pub duration_ms: i64, + pub last_run_ms: i64, +} + +impl CompletedSpan { + /// Whether the row spans more than the run that started at `last_started_at`, i.e. the + /// job parked and resumed. A flow records such a step with the row's span, so the step + /// reads like the job does. + pub fn outlasts_run(&self, last_started_at: Option>) -> bool { + matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last) + } +} + /// What a job's completion writes, for `Completion::execute`. struct Completion<'a> { completed_job: &'a MiniCompletedJob, @@ -1786,6 +1858,11 @@ impl Completion<'_> { /// Moves the job from the queue to the completed jobs and refreshes a flow step's parent /// ping, as one statement. Returns `None` when the job was no longer in the queue. /// + /// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's + /// `started_at` and `duration_ms` span the whole workflow from its first start, and the + /// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from + /// completed rows. + /// /// The completion takes its cancellation from the queue row it deletes, not only from /// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still /// being written, so the deleted row is the final word on whether the job was canceled. @@ -1793,7 +1870,7 @@ impl Completion<'_> { /// It locks the queue row before the completed row's key. Any other writer completing a job /// (the monitor's zombie fallback, debounce) must take them in the same order, or the two /// deadlock. - async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { + async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { let Completion { completed_job, success, @@ -1824,16 +1901,20 @@ impl Completion<'_> { .then_some(completed_job.parent_job) .flatten() else { - return sqlx::query_scalar!( + return sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1844,8 +1925,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1861,9 +1945,11 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1883,16 +1969,20 @@ impl Completion<'_> { // A canceled flow is pinged too: it is completed by its next transition like any other // flow, and the zombie flow monitor needs the ping to finish the cancel if that // transition is lost. - sqlx::query_scalar!( + sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1903,8 +1993,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1920,14 +2013,16 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ), parent_ping AS ( UPDATE v2_job_runtime r SET ping = now() FROM v2_job_queue q WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12 AND EXISTS (SELECT 1 FROM completed) ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1994,7 +2089,7 @@ async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, bool), Error>> { +) -> Option, CompletedSpan, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -2194,18 +2289,17 @@ async fn restart_job_if_perpetual_inner( /// resolution too; `ON CONFLICT DO NOTHING` keeps a human's note intact. /// /// Membership of the chain must be *proven* per row, never inferred from `parent_job` -/// alone: `root` is `job.parent_job.unwrap_or(job.id)`, so for a job launched with an -/// explicit `parent_job` (WAC inline children, SDK-launched children) `root` is the -/// *calling* job, and its other failed children are unrelated. Resolving those would hide -/// exactly the failures this feature exists to surface, so each row must be either +/// alone: the success-side caller passes the succeeding job's `parent_job`, which for a +/// job launched with an explicit parent (WAC inline children, SDK-launched children) is +/// the *calling* job, and its other failed children are unrelated. Resolving those would +/// hide exactly the failures this feature exists to surface, so each row must be either /// - a job carrying a `native_retry_attempt` marker under `root` (provably an attempt), or -/// - `root` itself as the original attempt, which is only provable when `root` is -/// parentless and unmarked. +/// - `root` itself as the original attempt: unmarked, and proven a chain root by the +/// marked success under it, since only a chain root is ever a marked job's parent +/// (see [`native_retry_root`]). /// /// Both arms also require the same `runnable_id` as the succeeding attempt, since every -/// attempt in a chain runs the same runnable. The deliberate cost is a miss, not an -/// over-reach: when the original attempt had a parent it is an unmarked sibling -/// indistinguishable from any other child of the caller, so it stays red. +/// attempt in a chain runs the same runnable. /// /// Nothing is trusted of the caller (this writes through a privileged pool): the gate is /// "some marked attempt under `root` running `runnable_id` has succeeded", evaluated in @@ -2249,7 +2343,7 @@ pub async fn resolve_retry_chain_if_succeeded( AND ( (j.parent_job = $1 AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) - OR (c.id = $1 AND j.parent_job IS NULL + OR (c.id = $1 AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) ) ON CONFLICT (job_id) DO NOTHING", @@ -2303,6 +2397,18 @@ async fn eval_retry_if( false } +/// The first attempt of `job`'s native retry chain, which every later attempt is pushed +/// with as `parent_job` and whose id seeds theirs. A first attempt may have a parent of its +/// own (a script dispatched from another job), so only a marked attempt's parent is the root: +/// rooting at that outer parent would merge the chains of its retried children. +fn native_retry_root(job: &MiniCompletedJob, is_retry_attempt: bool) -> Uuid { + if is_retry_attempt { + job.parent_job.unwrap_or(job.id) + } else { + job.id + } +} + /// Native script retry. When a failed `Script` or `Script_Hub` job carries a retry /// policy (via `runnable_settings_handle`) and has attempts left, enqueue a fresh /// attempt of the same script after the policy's backoff delay — instead of having @@ -2346,6 +2452,18 @@ pub async fn maybe_enqueue_native_script_retry( if !policy.has_attempts() { return Ok(false); } + // Cancelling a job that has a parent (every retry attempt does) only flags its queue row, + // and the worker that pulls it completes it without a `canceled_by`: read the flag here. + let soft_canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + job.id, + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if soft_canceled { + return Ok(false); + } // Attempt counter for this job: its `native_retry_attempt` marker, or 0 for // the first (un-marked) attempt. The marker is persistent (unlike the queue @@ -2358,7 +2476,7 @@ pub async fn maybe_enqueue_native_script_retry( .fetch_optional(db) .await? .unwrap_or(0) as u32; - let root = job.parent_job.unwrap_or(job.id); + let root = native_retry_root(job, prev_attempts > 0); // Scheduled chains keep the schedule trigger so the terminal attempt drives // the schedule completion handlers (on_failure/on_success); `parent_job` // keeps every retry out of the per-occurrence handler counting queries. diff --git a/backend/windmill-queue/tests/native_retry_test.rs b/backend/windmill-queue/tests/native_retry_test.rs index 9b0e05a74c..4cf278cf83 100644 --- a/backend/windmill-queue/tests/native_retry_test.rs +++ b/backend/windmill-queue/tests/native_retry_test.rs @@ -561,4 +561,123 @@ mod native_retry { .unwrap(); assert_eq!(status, "failure", "resolving must not change job status"); } + + // A script dispatched from another job keeps that job as its parent, so its chain is + // rooted at the first attempt itself: sibling dispatches must not share retry ids, and + // a parented first attempt is still resolved once its retry succeeds. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn parented_first_attempts_root_their_own_chains( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 1, seconds: 0 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let dispatcher = Uuid::new_v4(); + let (a, b) = (Uuid::new_v4(), Uuid::new_v4()); + for first in [a, b] { + let attempt = mini(first, Some(dispatcher), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + } + assert!( + retry_by_attempt(&db, a, 1).await.is_some(), + "a's retry hangs off a" + ); + assert!( + retry_by_attempt(&db, b, 1).await.is_some(), + "b's retry is not mistaken for a's" + ); + + let (c, c_retry, c_sibling) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + seed_job(&db, c, Some(dispatcher), false, "failure").await; + seed_job(&db, c_sibling, Some(dispatcher), false, "failure").await; + seed_job(&db, c_retry, Some(c), true, "success").await; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, c, WS, None).await?; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, dispatcher, WS, None).await?; + let resolved: Vec = sqlx::query_scalar("SELECT job_id FROM job_resolution") + .fetch_all(&db) + .await?; + assert_eq!( + resolved, + vec![c], + "only the parented first attempt resolves, not its sibling" + ); + Ok(()) + } + + // Cancelling the job that dispatched a script reaches that script's pending retry, which + // hangs off a first attempt that has already completed. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn cancelling_the_caller_cancels_a_pending_retry( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 2, seconds: 60 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let (caller, first) = (Uuid::new_v4(), Uuid::new_v4()); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, kind, runnable_path) VALUES ($1, $2, 'script', $3)", + ) + .bind(caller) + .bind(WS) + .bind(SCRIPT) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running) VALUES ($1, $2, now(), true)", + ) + .bind(caller) + .bind(WS) + .execute(&db) + .await?; + seed_job(&db, first, Some(caller), false, "failure").await; + let attempt = mini(first, Some(caller), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + let (retry_id, ..) = retry_by_attempt(&db, first, 1) + .await + .expect("retry is queued"); + + let tx = db.begin().await?; + let (tx, _) = + windmill_queue::jobs::cancel_job("test-user", None, caller, WS, tx, &db, false, false) + .await?; + tx.commit().await?; + + let canceled_by: Option = + sqlx::query_scalar("SELECT canceled_by FROM v2_job_queue WHERE id = $1") + .bind(retry_id) + .fetch_one(&db) + .await?; + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + // The worker that pulls the canceled retry completes it without a `canceled_by`. + let (_, _, _, _, retry_handle) = retry_by_attempt(&db, first, 1).await.unwrap(); + let canceled_retry = mini(retry_id, Some(first), retry_handle); + assert!( + !maybe_enqueue_native_script_retry(&db, &canceled_retry, &None, &no_result).await?, + "a canceled retry does not enqueue the next attempt" + ); + Ok(()) + } } diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index e2e4ab866b..364795510a 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -54,11 +54,45 @@ pub struct VolumeMount { pub target: String, } +/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while +/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A +/// write within the filesystem's timestamp granularity of the hand-off is not seen. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileStamp { + size: u64, + changed: Option<(i64, i64)>, +} + +impl FileStamp { + /// Whether the file is as it was at `earlier`. An unreadable change time never matches. + pub fn unchanged_since(&self, earlier: &FileStamp) -> bool { + self.changed.is_some() && self == earlier + } + + pub fn of(meta: &std::fs::Metadata) -> Self { + // ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it. + #[cfg(unix)] + let changed = { + use std::os::unix::fs::MetadataExt; + Some((meta.ctime(), meta.ctime_nsec())) + }; + #[cfg(not(unix))] + let changed = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64)); + Self { size: meta.len(), changed } + } +} + pub struct VolumeState { pub mount: VolumeMount, pub local_dir: PathBuf, pub manifest: HashMap, pub symlinks: HashMap, + /// Stamp of each pulled file as the job received it, keyed like `manifest`. + pub stamps: HashMap, } pub struct DownloadStats { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index bb47d3e4be..1ae588ef11 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output( // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) - UPDATE v2_job_queue q SET running = false, started_at = null + UPDATE v2_job_queue q SET running = false, started_at = null, + extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras + ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at)) + END FROM prev WHERE q.id = $1 RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", job.id, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 01523eba59..745f202826 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script( wm_memory_usage: None, job_isolation: None, native_mode: None, + draining: None, ping_type: PingType::InitScript, }, ) diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index fa06edf6bf..84fd547641 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -1049,6 +1049,7 @@ where let wm_memory_usage = get_windmill_memory_usage(); tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); + crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref()); if job_id != Uuid::nil() { if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await { tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); @@ -1140,8 +1141,8 @@ fn child_joined_output_stream( .take() .expect("child did not have a handle to stdout"); - let stdout = BufReader::new(stdout).lines(); - let stderr = BufReader::new(stderr).lines(); + let stdout = BufReader::new(stdout).split(b'\n'); + let stderr = BufReader::new(stderr).split(b'\n'); stream::select( lines_to_stream(stderr, true, job_id.clone(), w_id.clone()) .map(|l| l.map(|line| OutputLine { stderr: true, line })), @@ -1151,18 +1152,32 @@ fn child_joined_output_stream( } pub fn lines_to_stream( - mut lines: tokio::io::Lines, + mut lines: tokio::io::Split, stderr: bool, job_id: Uuid, w_id: String, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) - .poll_next_line(cx) - .map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id)) + .poll_next_segment(cx) + .map(|result| { + let result = result.map(|segment| segment.map(decode_output_line)); + process_streaming_log_lines(result, stderr, &job_id, &w_id) + }) }) } +/// Child output is not guaranteed to be UTF-8 (e.g. pwsh on Windows writes in the +/// console code page), so decode lossily: a strict decode errors out and ends log +/// capture for the rest of the job. Strips a trailing `\r` so CRLF output doesn't keep it. +pub(crate) fn decode_output_line(mut segment: Vec) -> String { + if segment.last() == Some(&b'\r') { + segment.pop(); + } + String::from_utf8(segment) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) +} + pub fn process_status( program: &str, status: ExitStatus, @@ -1209,4 +1224,33 @@ mod tests { let left = JobDeadline(Some(Instant::now() + Duration::from_secs(120))); assert!(matches!(left.remaining_secs(), Some(s) if (118..=120).contains(&s))); } + + // "Prüfe" in CP850/1252 is not UTF-8. A strict decode errors on that line and + // would stop log capture, dropping every line after it. + #[tokio::test] + async fn a_non_utf8_line_is_decoded_lossily_and_later_lines_still_arrive() { + let output: &[u8] = + b"start\r\nPr\x81fe Benutzer\nPr\xfcfe\nGr\xc3\xbc\xc3\x9fe\r\nafter\n\nlast"; + let lines: Vec = lines_to_stream( + BufReader::new(output).split(b'\n'), + false, + Uuid::nil(), + "w".into(), + ) + .map(|line| line.expect("reading from a byte slice cannot fail")) + .collect() + .await; + assert_eq!( + lines, + vec![ + "start", + "Pr\u{FFFD}fe Benutzer", + "Pr\u{FFFD}fe", + "Grüße", + "after", + "", + "last", + ] + ); + } } diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 6059936af6..aa1d9ab209 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -81,6 +81,7 @@ mod prepare_deps; mod python_executor; #[cfg(feature = "python")] mod python_versions; +mod resource_metrics; pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; diff --git a/backend/windmill-worker/src/resource_metrics.rs b/backend/windmill-worker/src/resource_metrics.rs new file mode 100644 index 0000000000..53c7dc5b4d --- /dev/null +++ b/backend/windmill-worker/src/resource_metrics.rs @@ -0,0 +1,235 @@ +use std::sync::{atomic::Ordering, Mutex, OnceLock}; + +use windmill_common::{ + otel_oss::{ + otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory, + otel_set_worker_occupancy_rate, + }, + worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP}, + OTEL_METRICS_ENABLED, +}; + +use crate::common::OccupancyResult; + +#[cfg(feature = "prometheus")] +struct PromMetrics { + memory_usage: prometheus::IntGaugeVec, + windmill_memory_usage: prometheus::IntGaugeVec, + memory_limit: prometheus::IntGaugeVec, + cpu_usage: prometheus::CounterVec, + cpu_periods: prometheus::IntCounterVec, + cpu_throttled_periods: prometheus::IntCounterVec, + cpu_throttled_seconds: prometheus::CounterVec, + occupancy_rate: prometheus::GaugeVec, + job_memory_peak: prometheus::HistogramVec, +} + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + static ref PROM_METRICS: Option = + if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) { + const WORKER: &[&str] = &["name", "worker_group"]; + Some(PromMetrics { + memory_usage: prometheus::register_int_gauge_vec!( + "worker_memory_usage_bytes", + "Memory used by the worker's container, page cache excluded", + WORKER + ) + .unwrap(), + windmill_memory_usage: prometheus::register_int_gauge_vec!( + "worker_windmill_memory_usage_bytes", + "Resident memory of the worker process itself", + WORKER + ) + .unwrap(), + memory_limit: prometheus::register_int_gauge_vec!( + "worker_memory_limit_bytes", + "Memory limit of the worker's container", + WORKER + ) + .unwrap(), + cpu_usage: prometheus::register_counter_vec!( + "worker_cpu_usage_seconds_total", + "CPU time consumed by the worker's container", + WORKER + ) + .unwrap(), + cpu_periods: prometheus::register_int_counter_vec!( + "worker_cpu_periods_total", + "CPU enforcement periods elapsed for the worker's container", + WORKER + ) + .unwrap(), + cpu_throttled_periods: prometheus::register_int_counter_vec!( + "worker_cpu_throttled_periods_total", + "CPU enforcement periods in which the worker's container was throttled", + WORKER + ) + .unwrap(), + cpu_throttled_seconds: prometheus::register_counter_vec!( + "worker_cpu_throttled_seconds_total", + "Time the worker's container spent throttled by its CPU limit", + WORKER + ) + .unwrap(), + occupancy_rate: prometheus::register_gauge_vec!( + "worker_occupancy_rate", + "Share of time the worker spent executing jobs, over the window", + &["name", "worker_group", "window"] + ) + .unwrap(), + job_memory_peak: prometheus::register_histogram_vec!( + "worker_job_memory_peak_bytes", + "Peak memory of a job's main process", + &["tag"], + windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec() + ) + .unwrap(), + }) + } else { + None + }; +} + +fn metrics_enabled() -> bool { + #[cfg(feature = "prometheus")] + if PROM_METRICS.is_some() { + return true; + } + OTEL_METRICS_ENABLED.load(Ordering::Relaxed) +} + +// Memory and CPU are read from the cgroup, which every worker of a process shares +// (native mode runs several). Only the first worker to report exports them, so that +// summing the series over workers does not count one container several times. +static CONTAINER_SERIES_OWNER: OnceLock = OnceLock::new(); +static MEMORY_LIMIT: Mutex>> = Mutex::new(None); + +/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits. +pub(crate) fn set_memory_limit(limit: Option) { + *MEMORY_LIMIT.lock().unwrap() = Some(limit); +} +static LAST_CPU_STAT: Mutex = + Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 }); + +/// Exports the worker's resource readings. Meant for the ping ticks, which have already +/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read. +pub(crate) fn record_worker_resources( + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy: Option<&OccupancyResult>, +) { + if !metrics_enabled() { + return; + } + let worker_group = WORKER_GROUP.as_str(); + + if let Some(occupancy) = occupancy { + let windows = [ + ("total", Some(occupancy.occupancy_rate)), + ("15s", occupancy.occupancy_rate_15s), + ("5m", occupancy.occupancy_rate_5m), + ("30m", occupancy.occupancy_rate_30m), + ]; + for (window, rate) in windows { + let Some(rate) = rate.filter(|r| r.is_finite()) else { + continue; + }; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.occupancy_rate + .with_label_values(&[worker_name, worker_group, window]) + .set(rate as f64); + } + otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64); + } + } + + if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name { + return; + } + + let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory); + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + if let Some(v) = memory_usage { + m.memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = wm_memory_usage { + m.windmill_memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = memory_limit { + m.memory_limit.with_label_values(labels).set(v); + } + } + otel_set_worker_memory( + worker_name, + worker_group, + memory_usage, + wm_memory_usage, + memory_limit, + ); + + let Some(stat) = get_cpu_stat() else { + return; + }; + let delta = { + let mut last = LAST_CPU_STAT.lock().unwrap(); + // A reading below the previous one means the cgroup changed under the process: + // restart from it rather than exporting a negative increase. + let delta = if stat.usage_usec < last.usage_usec { + CpuStat::default() + } else { + CpuStat { + usage_usec: stat.usage_usec - last.usage_usec, + nr_periods: stat.nr_periods.saturating_sub(last.nr_periods), + nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled), + throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec), + } + }; + *last = stat; + delta + }; + let usage_secs = delta.usage_usec as f64 / 1e6; + let throttled_secs = delta.throttled_usec as f64 / 1e6; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + m.cpu_usage.with_label_values(labels).inc_by(usage_secs); + m.cpu_periods + .with_label_values(labels) + .inc_by(delta.nr_periods); + m.cpu_throttled_periods + .with_label_values(labels) + .inc_by(delta.nr_throttled); + m.cpu_throttled_seconds + .with_label_values(labels) + .inc_by(throttled_secs); + } + otel_add_worker_cpu( + worker_name, + worker_group, + usage_secs, + delta.nr_periods, + delta.nr_throttled, + throttled_secs, + ); +} + +/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process +/// to sample or the reading failed, and those are not observations. +pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) { + if mem_peak_kb <= 0 { + return; + } + let bytes = mem_peak_kb as u64 * 1024; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.job_memory_peak + .with_label_values(&[tag]) + .observe(bytes as f64); + } + otel_record_job_memory_peak(tag, bytes); +} diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0490ed1754..12edd7823e 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1776,7 +1776,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &job, true, @@ -1835,7 +1835,10 @@ pub async fn process_completed_job( true, canceled_by, result, - started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }), + span.started_at.or(started_at).map(|x| FlowJobDuration { + started_at: x, + duration_ms: span.duration_ms, + }), StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -1867,9 +1870,10 @@ pub async fn process_completed_job( // `wm_failure` field) so a real runtime failure whose raw // result happens to contain a `wm_failure` field still goes // through the standard `WrappedError { error: ... }` wrap path. - let downstream_result: Arc> = if is_pre_shaped_wm_failure_result(result.get()) - { - windmill_queue::add_completed_job_pre_shaped_failure( + let (downstream_result, span): (Arc>, _) = if is_pre_shaped_wm_failure_result( + result.get(), + ) { + let span = windmill_queue::add_completed_job_pre_shaped_failure( db, &job, mem_peak.to_owned(), @@ -1880,9 +1884,9 @@ pub async fn process_completed_job( None, ) .await?; - result.clone() + (result.clone(), span) } else { - let wrapped = add_completed_job_error( + let (wrapped, span) = windmill_queue::add_completed_job_error_with_span( db, &job, mem_peak.to_owned(), @@ -1895,7 +1899,10 @@ pub async fn process_completed_job( None, ) .await?; - Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()) + ( + Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()), + span, + ) }; #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { @@ -1927,12 +1934,19 @@ pub async fn process_completed_job( false, canceled_by, downstream_result, - duration.and_then(|d| { - job.started_at.map(|started_at| FlowJobDuration { - started_at: started_at, - duration_ms: d, + if span.outlasts_run(job.started_at) { + span.started_at.map(|started_at| FlowJobDuration { + started_at, + duration_ms: span.duration_ms, }) - }), + } else { + duration.and_then(|d| { + job.started_at.map(|started_at| FlowJobDuration { + started_at: started_at, + duration_ms: d, + }) + }) + }, StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 45d76c74e8..3cc9c1e23a 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -113,6 +113,11 @@ pub enum WacPark { /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. /// +/// The first park keeps that first start in `extras.wac_first_started_at`, so the +/// completed row can still say when the workflow began and how long it took overall +/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline +/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way. +/// /// Call it before any write to the parent's `v2_job_status` row in the same /// transaction: a child's completion locks the queue row and then the status row /// (`record_child_completion`), and taking them the other way round here can @@ -156,7 +161,10 @@ pub async fn suspend_wac_parent( sqlx::query!( "UPDATE v2_job_queue - SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null, + extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras + ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at)) + END WHERE id = $1 AND workspace_id = $2", job_id, w_id, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2acafa539f..5b5df51b30 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -155,7 +155,7 @@ use crate::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, tally_unfinished_dependency_deploy, }, - worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full}, + worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full}, }; #[cfg(feature = "rust")] @@ -2713,7 +2713,7 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); let mut reported_ip = cached_ip(); - let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn) + let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn) .await .expect("initial ping could be sent"); @@ -3160,6 +3160,31 @@ pub async fn run_worker( let mut killpill_rx2 = killpill_rx.resubscribe(); + // The loop below only sees the killpill once the running job is done, which is too late + // for that job to learn its worker is going away. + { + let mut killpill_rx = killpill_rx.resubscribe(); + let conn = conn.clone(); + let worker_name = worker_name.clone(); + tokio::spawn(async move { + if matches!( + killpill_rx.recv().await, + Err(broadcast::error::RecvError::Closed) + ) { + return; + } + for attempt in 1..=3 { + match mark_worker_draining(&conn, &worker_name).await { + Ok(()) => return, + Err(e) => { + tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}"); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + }); + } + loop { let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst); if last_processing_duration_secs > 5 { @@ -3235,7 +3260,7 @@ pub async fn run_worker( // which costs at most the last job's id here: no job of this worker is in flight at this // point in the loop, and the next one refills them. if ip_just_resolved && conn.as_sql().is_none() { - if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await { + if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await { tracing::warn!( worker = %worker_name, hostname = %hostname, "failed to re-register with the resolved external IP: {e}" @@ -5105,6 +5130,7 @@ pub async fn handle_queued_job( // WAC v2 job suspended while waiting for child jobs — don't complete it return Ok(JobOutcome::Completed); } + crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak); process_result( cjob, result.map(|x| Arc::new(x)), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6d76d0d771..4f67780f18 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; let duration = if success { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, true, @@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms } else { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, false, @@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms }; flow_job_duration = flow_job .started_at diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 3a313e9360..c634c13b6a 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -7,9 +7,9 @@ use windmill_common::{ external_ip::UNKNOWN_IP, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, - insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, - update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED, - WORKER_CONFIG, WORKER_GROUP, + insert_ping_query, set_worker_draining_query, update_job_ping_query, + update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping, + PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, DB, }; @@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full( let wm_memory_usage = get_windmill_memory_usage(); let (vcpus, memory) = if read_cgroups { - (get_vcpus(), get_memory()) + let memory = get_memory(); + crate::resource_metrics::set_memory_limit(memory); + (get_vcpus(), memory) } else { (None, None) }; + let occupancy = occupancy_metrics.update_occupancy_metrics(); + crate::resource_metrics::record_worker_resources( + worker_name, + memory_usage, + wm_memory_usage, + Some(&occupancy), + ); let OccupancyResult { occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, - } = occupancy_metrics.update_occupancy_metrics(); + } = occupancy; let ping_start = std::time::Instant::now(); if let Err(e) = (|| { @@ -164,6 +173,7 @@ async fn update_worker_ping_full_inner( wm_memory_usage: get_windmill_memory_usage(), job_isolation: None, native_mode: Some(native_mode), + draining: None, ping_type: PingType::MainLoop, }, ) @@ -179,6 +189,7 @@ pub async fn insert_ping( worker_instance: &str, worker_name: &str, ip: Option<&str>, + reset_draining: bool, db: &Connection, ) -> anyhow::Result { let (tags, dw, dws, native_mode) = { @@ -224,6 +235,7 @@ pub async fn insert_ping( memory, job_isolation, native_mode, + reset_draining.then_some(false), db, ) .await; @@ -256,6 +268,7 @@ pub async fn insert_ping( wm_memory_usage: get_windmill_memory_usage(), job_isolation, native_mode: Some(native_mode), + draining: reset_draining.then_some(false), ping_type: PingType::Initial, }, ) @@ -314,6 +327,7 @@ pub async fn update_worker_ping_from_job( &Ping { last_job_executed: Some(job_id.clone()), last_job_workspace_id: Some(w_id.to_string()), + draining: None, ping_type: PingType::Job, worker_instance: None, ip: None, @@ -342,6 +356,45 @@ pub async fn update_worker_ping_from_job( Ok(()) } +/// Publishes on the worker's ping row that it has received its killpill, so the job it is +/// still running can find out and exit early. +pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> { + match conn { + Connection::Sql(db) => set_worker_draining_query(worker_name, db).await, + Connection::Http(client) => { + client + .post::( + UPDATE_PING_URL, + None, + &Ping { + last_job_executed: None, + last_job_workspace_id: None, + draining: None, + ping_type: PingType::Draining, + worker_instance: None, + ip: None, + tags: None, + dw: None, + dws: None, + version: None, + vcpus: None, + memory: None, + memory_usage: None, + wm_memory_usage: None, + jobs_executed: None, + occupancy_rate: None, + occupancy_rate_15s: None, + occupancy_rate_5m: None, + occupancy_rate_30m: None, + job_isolation: None, + native_mode: None, + }, + ) + .await + } + } +} + pub async fn ping_job_status( conn: &Connection, job_id: &Uuid, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 281ba460b6..df2044d419 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.827.0"; +export const VERSION = "v1.828.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index 692e5d33f5..c6691e0048 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.827.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.827.0", + "version": "1.828.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 287636a9f8..a119b8645d 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.827.0", + "version": "1.828.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/commands/user/user.ts b/cli/src/commands/user/user.ts index b5ed0e3196..99bea072a8 100644 --- a/cli/src/commands/user/user.ts +++ b/cli/src/commands/user/user.ts @@ -1,6 +1,8 @@ import { writeFile } from "node:fs/promises"; import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import { GlobalOptions, isSuperset, @@ -69,6 +71,32 @@ async function list(opts: GlobalOptions) { .render(); } +async function listWorkspaceUsers(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const users = await wmill.listUsers({ workspace: workspace.workspaceId }); + + if (opts.json) { + console.log(JSON.stringify(users)); + } else { + new Table() + .header(["username", "email", "role"]) + .padding(2) + .border(true) + .body( + users.map((u) => [ + u.username, + u.email, + u.is_admin ? "admin" : u.operator ? "operator" : "developer", + ]) + ) + .render(); + } +} + function rdString() { return Math.random().toString(36).slice(2, 7); } @@ -518,6 +546,9 @@ export async function pushInstanceGroups( const command = new Command() .description("user related commands") .action(list as any) + .command("list", "List the users of the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(listWorkspaceUsers as any) .command("add", "Create a user") .arguments(" [password:string]") .option("--superadmin", "Specify to make the new user superadmin.") diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index d05c4e60a7..860a3ddede 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.827.0"; +export const VERSION = "1.828.0"; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 4aaf6d7e3c..eb5d46ef57 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -767,21 +767,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -837,6 +839,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1600,21 +1610,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1670,6 +1682,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2539,21 +2559,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -2609,6 +2631,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -4434,10 +4464,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -4610,6 +4644,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -5969,6 +6015,10 @@ can call one, not an agent used as a tool. - \`results.step_id.property\` - Access specific property from a previous step output only when that step result is in scope - \`flow_input.iter.value\` - Current iteration value inside a \`forloopflow\`; in a \`whileloopflow\` it is just the iteration index (a plain number, same as \`flow_input.iter.index\`) - \`flow_input.iter.index\` - Current loop index when inside a loop (\`forloopflow\` or \`whileloopflow\`) +- After a suspend step (approval), the next step's javascript expressions also get: + - \`resume\` - the last resume payload (the resume form values); with \`continue_on_disapprove_timeout\`, a disapproval or timeout yields \`{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }\` instead + - \`resumes\` - every resume payload, in arrival order + - \`approvers\` - a plain array of strings, one per resume: for a resume through the \`resume\`/\`cancel\` URL from \`getResumeUrls(approver)\`, the \`approver\` passed there if any; otherwise (including the \`approvalPage\` link) the resuming user's username, else \`"anonymous"\`, e.g. \`approvers[0]\`. It is NOT the \`{ resume_id, approver }\` objects stored in the flow status, so \`approvers[0].approver\` is always undefined ## Loop Structure Rules @@ -6280,7 +6330,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} `, "raw-app": `--- name: raw-app @@ -7865,10 +7915,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -7912,7 +8000,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -8711,6 +8799,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. @@ -9061,7 +9151,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9072,7 +9163,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9160,7 +9252,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9171,7 +9264,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9244,7 +9338,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9255,7 +9350,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9365,7 +9461,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9376,7 +9473,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9528,7 +9626,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9539,7 +9638,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9697,7 +9797,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9708,7 +9809,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9823,7 +9925,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9834,7 +9937,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9921,7 +10025,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9932,7 +10037,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10013,7 +10119,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10024,7 +10131,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10111,7 +10219,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10122,7 +10231,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10262,7 +10372,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10273,7 +10384,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10463,7 +10575,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10474,7 +10587,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/docs/pipeline-local-dev.md b/docs/pipeline-local-dev.md index e05acec9f0..fb8d49b296 100644 --- a/docs/pipeline-local-dev.md +++ b/docs/pipeline-local-dev.md @@ -69,7 +69,7 @@ makes the client own the whole cascade so the backend dispatcher never double-fi - `frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts` — reusable run primitives (`makeLaunch`, `makeWaitJobTerminal`, `runDownstreamCascade`, `runBoundedCascade`) wrapping `cascadeOrchestrator` + `graphTraversal`. -- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/pipeline_dev/+page.svelte`. +- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/(styled)/pipeline_dev/+page.svelte`. **Modified** - `cli/src/commands/pipeline/pipeline.ts` — `--local` on `show`/`run`; `show` split into graph diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2f5ea5b721..28ecbebf42 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.827.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.827.0", + "version": "1.828.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 4515459ba3..cc42f5424a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.827.0", + "version": "1.828.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/app.html b/frontend/src/app.html index aad967fd91..3c9e232e0f 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -12,6 +12,17 @@ fill: #3b82f6; } + /* Public app pages render their shell before app.css loads, if at all. */ + html[data-wm-public-app] #svelte-global-loader { + display: none; + } + html[data-wm-public-app] body { + margin: 0; + } + html[data-wm-public-app='dark'] body { + background-color: #3b4252; + } + /* Background color for dark mode */ @media (prefers-color-scheme: dark) { #mainbg { @@ -123,6 +134,64 @@ } })() + %sveltekit.head% diff --git a/frontend/src/lib/apiBaseUrl.svelte.ts b/frontend/src/lib/apiBaseUrl.svelte.ts new file mode 100644 index 0000000000..f87a33f0d8 --- /dev/null +++ b/frontend/src/lib/apiBaseUrl.svelte.ts @@ -0,0 +1,22 @@ +import { fromStore } from 'svelte/store' +import { base } from '$lib/base' +import { instanceUi } from '$lib/instanceUi' + +const ui = fromStore(instanceUi) + +/** + * Base of every URL shown for an external client to call (webhooks, HTTP routes, push + * trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the + * origin being browsed. Append `/api/...` to it. + * + * Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links + * to pages): those must stay on the browsing origin, where the session cookie lives. + * + * Reactive: the setting lands after page load, so read it inside a `$derived` or the + * template rather than caching the result. + */ +export function apiBaseUrl(): string { + const setting = ui.current?.api_base_url + const configured = typeof setting === 'string' ? setting.trim() : '' + return configured || `${window.location.origin}${base}` +} diff --git a/frontend/src/lib/components/AutosaveIndicator.svelte b/frontend/src/lib/components/AutosaveIndicator.svelte index 76dc030c58..1e318ed9b2 100644 --- a/frontend/src/lib/components/AutosaveIndicator.svelte +++ b/frontend/src/lib/components/AutosaveIndicator.svelte @@ -142,7 +142,13 @@ let hintTimer: ReturnType | undefined const kindLabel = $derived( - itemKind === 'flow' ? 'flow' : itemKind === 'app' || itemKind === 'raw_app' ? 'app' : 'script' + itemKind === 'flow' + ? 'flow' + : itemKind === 'app' || itemKind === 'raw_app' + ? 'app' + : itemKind === 'data_pipeline' + ? 'pipeline' + : 'script' ) // Each false → true transition fires a hint (others > loaded). Seeded to diff --git a/frontend/src/lib/components/BreadcrumbItemContent.svelte b/frontend/src/lib/components/BreadcrumbItemContent.svelte new file mode 100644 index 0000000000..9e945586c7 --- /dev/null +++ b/frontend/src/lib/components/BreadcrumbItemContent.svelte @@ -0,0 +1,16 @@ + + + +{@render icon?.()} +{label} diff --git a/frontend/src/lib/components/BreadcrumbSegment.svelte b/frontend/src/lib/components/BreadcrumbSegment.svelte index 8a266e4fc4..4d55675f26 100644 --- a/frontend/src/lib/components/BreadcrumbSegment.svelte +++ b/frontend/src/lib/components/BreadcrumbSegment.svelte @@ -6,7 +6,7 @@ state — switching popovers relies on melt-ui's `closeOnOtherPopoverOpen` to close siblings. -->
diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 375e73480e..9696836dfd 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -1,9 +1,9 @@ + +{@render children()} diff --git a/frontend/src/lib/components/CronBuilder.svelte b/frontend/src/lib/components/CronBuilder.svelte index bcf8508c12..3bceabb417 100644 --- a/frontend/src/lib/components/CronBuilder.svelte +++ b/frontend/src/lib/components/CronBuilder.svelte @@ -15,7 +15,7 @@ contentClasses="p-4" > {#snippet trigger()} - {/snippet} diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 5204a876b9..837569c575 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -375,7 +375,8 @@ } if (!isInstanceDatatable(datatable)) return { - reason: 'Only data tables stored in the instance database have roles and grants.', + reason: + "Roles and grants are not supported on data tables backed by a Postgres resource. Store the data table on Windmill's database or on an external Postgres instance (set up by a superadmin in instance settings) to use them.", ee: false } if (needsAdmin && !canManageDatatable) @@ -458,6 +459,11 @@ // default to open — actually be folded; a plain "expanded" set could never // close them, since the default would keep winning. // What the permissions drawer is open on, if anything: a schema, or a table. + let aclEditor = $state(undefined) + /** The open access editor lists the data table's roles, which the roles drawer can change. */ + export function refreshAcl() { + aclEditor?.refresh() + } let aclDrawer = $state<{ datatable: string | undefined; target: AclTarget } | undefined>( undefined ) @@ -1466,7 +1472,7 @@ (aclDrawer = undefined)}> (aclDrawer = undefined)} tooltip="Who owns this, and what each role may do with it." > @@ -1474,7 +1480,18 @@ {@const dt = aclDrawer.datatable ?? currentDatatable} {#if dt} {#key `${dt}~${JSON.stringify(aclDrawer.target)}`} - + onDatatableAction?.(dt, 'roles'), + blocker: accessBlocker(dt, true)?.reason + } + : undefined} + /> {/key} {/if} {/if} diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index bd016544b0..30fc93377d 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -311,6 +311,7 @@ function refreshRoles() { datatables.refetch() usableRoles.refetch() + dbManagerContent?.dbManager()?.refreshAcl() } async function handleExportSchema(explicitSource?: string) { diff --git a/frontend/src/lib/components/DatatablePicker.svelte b/frontend/src/lib/components/DatatablePicker.svelte index 53617acf36..17ad6f3361 100644 --- a/frontend/src/lib/components/DatatablePicker.svelte +++ b/frontend/src/lib/components/DatatablePicker.svelte @@ -17,6 +17,9 @@ selectInputClass?: string class?: string onClear?: () => void + /** Size the input to its content, like Path's folder picker. */ + useContentEditable?: boolean + RightIcon?: any } let { @@ -27,7 +30,9 @@ placeholder = undefined, selectInputClass = '', class: className = '', - onClear = undefined + onClear = undefined, + useContentEditable = false, + RightIcon = undefined }: Props = $props() let datatables = usePromise(() => @@ -47,6 +52,8 @@ {placeholder} inputClass={selectInputClass} {onClear} + {useContentEditable} + {RightIcon} /> {#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })} diff --git a/frontend/src/lib/components/DevWorkspaceBadge.svelte b/frontend/src/lib/components/DevWorkspaceBadge.svelte new file mode 100644 index 0000000000..e86e4593c8 --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceBadge.svelte @@ -0,0 +1,56 @@ + + + +{devBadgeText(label)} diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte index e148d90c22..ff9932fa87 100644 --- a/frontend/src/lib/components/DevWorkspaceSetting.svelte +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -1,7 +1,8 @@ -
+
{#if !hidePath}
@@ -162,9 +111,9 @@ {#if segments} {#each segments.dirs as dir, i (dir.fullPath)} {@const dKey = dirKey('all', dir.fullPath)} + {/each} + {/if} + {#if segments?.leaf} {@const leafKey = leafKeyFor(kind, segments.leaf.fullPath)} {@const leafParent = segments.dirs[segments.dirs.length - 1]?.fullPath} + - + {#if pathEditable} - pathPopoverOpen, setPathPopoverOpen} - > - {#snippet trigger()} -
{/if} diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index b4bc4471a2..0a65701b06 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -303,7 +303,12 @@ {#if showBanner} - -
-
- -
-
- -
- - {isDevWorkspace - ? `${devLabelWord(currentWorkspaceData?.dev_workspace_label)} workspace of` - : 'Fork of'} - {parentWorkspaceData?.name}{#if parentWorkspaceData?.name !== parentWorkspaceId} - ({parentWorkspaceId}){/if} - -
- - {#if loading} - Checking for changes... - {:else if error} - - {error} - - {:else if comparison} -
- {#if comparison.summary.total_diffs > 0} - - {forkAheadBehindMessage(changesAhead, changesBehind)} - {parentWorkspaceId} over {comparison - .summary.total_diffs} items - - - - - {#if ciTestTotal > 0} - - - {#if ciTestFailing > 0} -
- - CI: {ciTestFailing} failing -
- {:else if ciTestRunning > 0} -
- - CI: {ciTestRunning} running -
- {:else} -
- - CI: {ciTestPassing} passing -
- {/if} - {/if} - - {#if comparison.summary.conflicts > 0} - - -
- - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 - ? 's' - : ''} -
- {/if} - {:else if comparison.skipped_comparison} - - This {currentNoun} was created before the addition of certain windmill features, and - therefore the changes with its parent workspace cannot be displayed. - {:else if showDraftsOnly} - - This workspace has {draftCount} draft{draftCount !== 1 ? 's' : ''} - - {:else} - Everything is up to date - {/if} -
- {/if} -
- -
- -
-
-
-
+ + {/if} + +{#snippet forkAction()} + {@const total = comparison?.summary.total_diffs ?? 0} + {@const conflicts = comparison?.summary.conflicts ?? 0} + + {#if loading} + Checking for changes… + {:else if error} + Comparison failed + {:else if showDraftsOnly} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + {:else if total > 0} + + + {#if changesAhead > 0} + openCompareDir('deploy_to')} + title="{changesAhead} ahead of {parentWorkspaceId} — review what this {currentNoun} has to give" + > + + {changesAhead}{compactCounts ? '' : ' ahead'} + + {/if} + {#if changesBehind > 0} + openCompareDir('update')} + title="{changesBehind} behind {parentWorkspaceId} — review what this {currentNoun} has yet to take" + > + + {changesBehind}{compactCounts ? '' : ' behind'} + + {/if} + {#if conflicts > 0} + + openCompareDir('deploy_to')} + title="{conflicts} conflicting item{conflicts !== 1 ? 's' : ''} — review them" + > + + {conflicts}{compactCounts ? '' : ` conflict${conflicts !== 1 ? 's' : ''}`} + + {/if} + + {#if draftCount > 0} + + + {draftCount}{compactCounts ? '' : ` draft${draftCount !== 1 ? 's' : ''}`} + + {/if} + {/if} + {#if showDraftsOnly} + + {:else} + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index 64ca912608..e0712d66c7 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -15,6 +15,7 @@ import { isWindmillTooBigObject } from './job_args' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' interface Props { id?: string | undefined @@ -246,7 +247,7 @@ ${Object.entries(displayArgs) If using the wmill client in your code, set the following env variables:
BASE_URL="{window.location.origin}"
+				>BASE_URL="{apiBaseUrl()}"
 WM_TOKEN="{''}"
diff --git a/frontend/src/lib/components/NavBreadcrumb.svelte b/frontend/src/lib/components/NavBreadcrumb.svelte new file mode 100644 index 0000000000..3081bcca4c --- /dev/null +++ b/frontend/src/lib/components/NavBreadcrumb.svelte @@ -0,0 +1,434 @@ + + + +{#snippet workspaceDisc()} + + + + + + + +{/snippet} + +{#snippet itemKindIcon()} + + {#if item?.kind}{/if} +{/snippet} + +{#snippet sectionKindIcon()} + {#if section?.kind}{/if} +{/snippet} + +{#snippet folderIcon()} + +{/snippet} + +{#snippet userIcon()} + +{/snippet} + +{#snippet envBadgeMark()} + {#if envBadge === 'dev'} + + {:else if envBadge === 'prod'} + prod + {/if} +{/snippet} + +{#snippet slash(extra = '')} + + +{/snippet} + + diff --git a/frontend/src/lib/components/NoDirectDeployAlert.svelte b/frontend/src/lib/components/NoDirectDeployAlert.svelte index d1017f7347..a3a249caf5 100644 --- a/frontend/src/lib/components/NoDirectDeployAlert.svelte +++ b/frontend/src/lib/components/NoDirectDeployAlert.svelte @@ -16,7 +16,8 @@ import { findCanonicalDevWorkspace } from '$lib/utils/workspaceHierarchy' import { devLabelKey, devLabelNoun } from '$lib/utils/devWorkspaceLabel' import { switchWorkspace } from '$lib/storeUtils' - import { Badge, Button } from './common' + import { Button, ButtonType } from './common' + import { twMerge } from 'tailwind-merge' import Popover from './meltComponents/Popover.svelte' import Toggle from './Toggle.svelte' import { GitFork, Lock, ShieldOff } from 'lucide-svelte' @@ -83,28 +84,35 @@ ? `Edits in ${devLabelKey(canonicalDev.dev_workspace_label)}` : 'Edits restricted' ) + + // The Popover renders its trigger as a + - {#if customUi?.topBar?.path != false} -
- onNavigate?.(item)} - /> -
+ {#if ownsPageHeader} + + + + {:else} +
+
+
+ {@render languageButton(condensedHeader ? 18 : 24)} + {#if customUi?.topBar?.path != false} +
+ onNavigate?.(item)} + /> +
+ {/if} + {@render autosaveIndicator()} + {#if !condensedHeader} + {@render settingsBadges()} + {/if} +
+ + +
+ + {#if $enterpriseLicense && initialPath != '' && !inSessionPane} + {/if} - {#if opWorkspace} - - {/if} - {#if !condensedHeader} - {@const canOpenRuntime = - customUi?.topBar?.settings != false && - customUi?.settingsPanel?.disableRuntime !== true} - { - selectedTab = 'runtime' - metadataOpen = true - } - : undefined} + + +
+ + {@render rightButtons()} +
+
+ {/if} + + {#snippet scriptSummary()} + + {#if script} + +
+ {#snippet summaryText()} + {emptyString(script.summary) ? 'Add a summary...' : script.summary} + {/snippet} + {#if customUi?.topBar?.editablePath != false || customUi?.topBar?.editableSummary != false} + + {:else} + {@render summaryText()} {/if}
+ {/if} + {/snippet} - -
+ + {#snippet scriptPathTrigger(pathLabel: Snippet, triggerClass: string)} + + {#if script} + + {:else} + {@render pathLabel()} + {/if} + {/snippet} + {#snippet scriptMarks()} + {#if script} + + {@render settingsBadges('pl-1.5')} + {/if} + {/snippet} + + {#snippet scriptHeaderActions()} + {#if script} + + {@render autosaveIndicator()} {#if $enterpriseLicense && initialPath != '' && !inSessionPane} {/if} + {@render rightButtons()} + {/if} + {/snippet} - -
+ {#snippet languageButton(size: number)} + + {/snippet} - {#snippet settingsButton()} - {#if customUi?.topBar?.settings != false} - - {/if} - {/snippet} - {#snippet diffButton()} - {#if customUi?.topBar?.diff != false} - {@const isDraftOnly = savedScript?.no_deployed === true} - {@const diffDisabled = !savedScript || !diffDrawer || isDraftOnly} - {@const diffTitle = isDraftOnly - ? 'Deploy this script once to compare against the deployed version' - : 'Diff'} - -
- -
- {/if} - {/snippet} - {#if compactTopbar} - - {#snippet buttonReplacement()} - + + {/if} + {/snippet} + {#if compactTopbar} + + {#snippet buttonReplacement()} + - - - + + {/if} + +{#snippet draftsAction()} + + {#if phone} + + {:else} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/WorkspaceItemKindIcon.svelte b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte new file mode 100644 index 0000000000..9d07f6ef89 --- /dev/null +++ b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte @@ -0,0 +1,19 @@ + + + +{#if kind === 'flow'} + +{:else if kind === 'app'} + +{:else} + +{/if} diff --git a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte index 77fe52b421..37a27f3812 100644 --- a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte +++ b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte @@ -2,8 +2,8 @@ import { userWorkspaces, type UserWorkspace } from '$lib/stores' import { findWorkspaceRoot } from '$lib/utils/workspaceHierarchy' import { forkAccentStyle } from '$lib/utils/forkColor' - import { Badge, Button } from '$lib/components/common' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import { Button } from '$lib/components/common' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { Building, ChevronDown, EllipsisVertical, GitFork } from 'lucide-svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { type Item } from '$lib/utils' @@ -171,12 +171,12 @@ {#if pendingFork} (new) {:else if currentWs?.is_dev_workspace} - {devBadgeText(currentWs.dev_workspace_label)} + {/if} {/if} diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 073d6c3e7d..ac0d2b1cb6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -86,7 +86,8 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: AppEditorProps = $props() migrateApp(untrack(() => app)) @@ -885,6 +886,7 @@ {#if !$userStore?.operator} {#if $appStore} hideBottomPanel()} /> {#if $mode === 'preview'} - +
{/if} - +
void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } let { @@ -142,10 +147,11 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: Props = $props() - /** Mirror of the path the user is editing in the pen popover. Initialized + /** Mirror of the path the user is editing in the rename popover. Initialized * once from `newPath` (or a synthesized path for new apps) and only * updated by user input from then on — we deliberately do NOT sync from * `newPath` afterwards so the user's in-flight rename isn't clobbered by @@ -223,6 +229,18 @@ // the preview to the deployed version. const inSessionPane = !!getContext('aiChatManager') + // The header's buttons render under the page header, not under this component, so the contexts + // they look up have to travel with them. The app's own two carry everything the canvas toggles, + // the panel buttons, Debug runs and the preview switch read. + /** Held by the rename popover while it is open; the band's trail reads it so the path does not + * reflow under the pointer as the user types. */ + let pathSnapshot = $state(undefined) + + const headerContexts = new Map([ + ['AppViewerContext', getContext('AppViewerContext')], + ['AppEditorContext', getContext('AppEditorContext')] + ]) + const loading = $state({ publish: false, save: false @@ -243,7 +261,24 @@ // Top-bar responsive collapse — container width, not viewport. let topbarWidth = $state(0) - const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720) + // In the page header the buttons share the row with the breadcrumb, so they shed the canvas + // toggles and fold Debug runs into the menu sooner: the whole group is ~840px and the trail + // with the summary takes ~430px of the same row. + // Measured on the band rather than estimated: with the canvas toggles shown the actions alone + // are 826-911px and the trail with the summary another ~500, so the row overflows the bar at + // every width up to about 1470 — 1350 was not high enough either. Below this the toggles stand + // down and the menu carries all three, which is what keeps them reachable rather than merely + // out of the way. + const compactBelow = $derived(ownsPageHeader ? 1500 : 720) + const compactTopbar = $derived.by(() => { + const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth + return w > 0 && w < compactBelow + }) + // A phone's bar holds the trail and Deploy. Which panels are open, the export and the + // editor/preview switch join the menu compact already shows. + const phoneTopbar = $derived( + ownsPageHeader && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR + ) function closeSaveDrawer() { saveDrawerOpen = false @@ -590,7 +625,49 @@ selectedJobId = $jobs[$jobs.length - 1] } $jobsDrawerOpen = true - }, + } + }, + // The canvas toggles stand down with the bar, and the breakpoint among them is the + // only way onto the mobile layout — so each one keeps an entry here. + { + displayName: $breakpoint === 'sm' ? 'Computer view' : 'Mobile view', + icon: $breakpoint === 'sm' ? Laptop2 : Smartphone, + action: () => ($breakpoint = $breakpoint === 'sm' ? 'lg' : 'sm') + }, + // The setting the mobile view is edited for, which otherwise lives beside the + // toggle group that stood down: without it here a deployed app could be put on + // the mobile layout at this width and never taken off it. + ...($breakpoint === 'sm' + ? [ + { + displayName: $app?.mobileViewOnSmallerScreens + ? 'Mobile view on small screens: on' + : 'Mobile view on small screens: off', + icon: Smartphone, + action: () => { + if ($app) $app.mobileViewOnSmallerScreens = !$app.mobileViewOnSmallerScreens + } + } + ] + : []), + { + displayName: $app?.fullscreen ? 'Centered canvas' : 'Full-width canvas', + icon: $app?.fullscreen ? AlignHorizontalSpaceAround : Expand, + action: () => { + if ($app) $app.fullscreen = !$app.fullscreen + } + }, + { + displayName: + $app?.darkMode === undefined + ? 'Theme: automatic' + : $app.darkMode + ? 'Theme: dark' + : 'Theme: light', + icon: $app?.darkMode === undefined ? SunMoon : $app.darkMode ? Moon : Sun, + // Cycles auto → light → dark, the order the toggle group reads in. + action: () => + setTheme($app?.darkMode === undefined ? false : $app.darkMode ? undefined : true), separatorBottom: true } ] @@ -913,141 +990,232 @@ -
- + ` placeholder, and the trail would name that instead of the path Deploy + // will create. Same fallback chain the rename sites read. + // Frozen while the rename popover is open so the trail holds still as the user types. + kind: 'app', + path: pathSnapshot ?? (newEditedPath || $appPath || newPath || undefined), + summaryContent: appSummary, + pathTrigger: appPathTrigger + }} + actions={appHeaderActions} + contexts={headerContexts} + separator="always" + /> +{:else} +
+ -
-
- (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} - /> -
-
- {#if $app} - { - $app.fullscreen = detail === 'true' - }} - > - {#snippet children({ item })} - - - {/snippet} - - {/if} - {#if $app} - { - const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined - setTheme(theme) - }} - selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} - > - {#snippet children({ item })} - - - - {/snippet} - - {/if} -
- - {#snippet children({ item })} - - - {#if $breakpoint === 'sm'} - - {/if} - {/snippet} - -
-
- {#if $workspaceStore} -
- +
+ (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} />
- {/if} -
+ {@render canvasToggles()} + {@render autosaveIndicator()} +
+ {@render panelButtons()} + {@render awarenessMark()} + {@render appActions()} +
+{/if} + + +{#snippet appPathTrigger(pathLabel: Snippet, triggerClass: string)} + +{/snippet} + +{#snippet appSummary()} + +
+ {#snippet summaryText()} + {emptyString($summary) ? 'Add a summary...' : $summary} + {/snippet} + +
+{/snippet} + +{#snippet appHeaderActions()} + +
+ {@render canvasToggles()} + {@render autosaveIndicator()} + {#if !phoneTopbar} + {@render panelButtons()} + {@render awarenessMark()} + {/if} + {@render appActions()} +
+{/snippet} + +{#snippet canvasToggles()} +
+ {#if $app} + { + $app.fullscreen = detail === 'true' + }} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} + {#if $app} + { + const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined + setTheme(theme) + }} + selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} + > + {#snippet children({ item })} + + + + {/snippet} + + {/if} +
+ + {#snippet children({ item })} + + + {#if $breakpoint === 'sm'} + + {/if} + {/snippet} + +
+
+{/snippet} + +{#snippet autosaveIndicator()} + {#if $workspaceStore} +
+ +
+ {/if} +{/snippet} + +{#snippet panelButtons()} {#if $mode !== 'preview'}
{/if} +{/snippet} + +{#snippet awarenessMark()} {#if $enterpriseLicense && $appPath != '' && !inSessionPane}
{/if} +{/snippet} + +{#snippet appActions()}
@@ -1131,14 +1305,19 @@
+ - +
-
+{/snippet} diff --git a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte index e004c17a04..cb56b37498 100644 --- a/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte +++ b/frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte @@ -13,13 +13,18 @@ import { sendUserToast } from '$lib/toast' import PublicApp from '$lib/components/apps/editor/PublicApp.svelte' import PublicAppFrame from '$lib/components/apps/editor/PublicAppFrame.svelte' + import PageHeaderContent from '$lib/components/PageHeaderContent.svelte' + import DropdownV2 from '$lib/components/DropdownV2.svelte' + import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte' + import type { Item } from '$lib/utils' import { Button } from '$lib/components/common' import { AppService, OpenAPI } from '$lib/gen' - import type { UserExt } from '$lib/stores' + import { userStore, type UserExt } from '$lib/stores' import { canWrite } from '$lib/utils' import { getUserExt } from '$lib/user' - import { Pen } from 'lucide-svelte' + import { ExternalLink, Pen } from 'lucide-svelte' import { page } from '$app/state' + import { isMenuHidden } from '$lib/components/sessions/sessionMode.svelte' import { setOperatingWorkspace, useOperatingUser @@ -30,10 +35,15 @@ path, onEdit, onLoadState, - syncHashToUrl = true + syncHashToUrl = true, + ownsPageHeader = false }: { workspace: string path: string + /** True for the route's own page: the app becomes what the page header names. A host that + * renders this inside something else — a session's preview panel, whose band names the + * session — leaves it false, and the app keeps its Edit over its own canvas. */ + ownsPageHeader?: boolean /** Handle Edit in place instead of following `editHref`. An AI session shows * this viewer inside a preview tab, where a plain link would navigate the whole * page out of the session rather than flipping the tab to its editor. */ @@ -70,6 +80,12 @@ * app point at /apps/get all over the app), so only the app can say which * editor the Edit button must open. */ let isRawApp = $state(false) + /** The app's execution mode, from the same lite read as the permissions and for the same + * reason: a sandboxed low-code app loads inside the opaque iframe, so `app` here stays + * undefined and cannot say whether the app answers at a public url. `policy.execution_mode` is + * where the lite payload carries it; the flat field beside it is the apps *list* row's + * spelling (ListableApp, which is what AppRow reads). */ + let executionMode = $state(undefined) let editHref = $derived(`${base}/${isRawApp ? 'apps_raw' : 'apps'}/edit/${path}?nodraft=true`) let refresh: (() => void) | undefined @@ -85,6 +101,11 @@ const hideEditBtn = page.url.searchParams.get('hideEditBtn') === 'true' const hideRefreshBar = page.url.searchParams.get('hideRefreshBar') === 'true' + const showEdit = $derived(canWriteApp && !hideEditBtn) + // Decides where Edit goes, not whether it appears: without the workspace navigation there is + // no band to put it in. Same rule the layout hides the sidebar by. + const menuHidden = $derived(isMenuHidden(page.url)) + // Embedder side: mint a scoped embed token (by path) from the member's session. async function fetchEmbedToken(opts?: { sdkConsent?: boolean }): Promise<{ token?: string }> { const headers: Record = {} @@ -155,48 +176,142 @@ const lite: any = await AppService.getAppLiteByPath({ workspace, path }) appPerms = { path: lite?.path ?? path, extraPerms: lite?.extra_perms ?? {} } isRawApp = !!lite?.raw_app + executionMode = lite?.policy?.execution_mode ?? lite?.execution_mode } catch (_) { appPerms = undefined } } + /** Whether the app answers at a public url at all. Read from the app this viewer has already + * loaded, the way the apps table decides the same thing (AppRow) — the secret itself is a + * request, and asking for one on every app view to decide whether to draw a button is a + * request per view that most views never use. */ + const hasPublicUrl = $derived(executionMode === 'anonymous') + /** Resolved when the button is used, not when it is drawn. */ + async function openPublicUrl() { + // The tab is opened inside the click itself and filled once the secret arrives. Safari only + // lets a gesture open the tab it opens synchronously, so opening it after the await is + // blocked there — silently, since a blocked popup reports nothing. + const tab = window.open('about:blank', '_blank') + try { + const secret = await AppService.getPublicSecretOfApp({ workspace, path }) + if (!secret) { + tab?.close() + sendUserToast('This app has no public url', true) + return + } + // Built from this viewer's workspace rather than the navigation one, like everything + // else here: the two differ inside a session's preview panel. + const url = `${window.location.origin}${base}/public/${workspace}/${secret}` + if (tab) tab.location.href = url + else sendUserToast('Allow pop-ups for this site to open the app’s public url', true) + } catch (e: any) { + tab?.close() + sendUserToast('Could not open the public url: ' + (e?.body ?? e?.message ?? e), true) + } + } + $effect(() => { if (workspace && path) loadPerms() }) + + // Both of this page's buttons carry a label, and a phone's bar has room for neither beside the + // app's path — so below that width they become one menu. Unmeasured (0) counts as wide: the bar + // measures itself on mount, and starting compact would pop them out a frame later. + const compact = $derived(pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR) + const compactItems: Item[] = $derived([ + ...(hasPublicUrl + ? [ + { + displayName: 'Public url', + icon: ExternalLink, + action: () => openPublicUrl() + } + ] + : []), + { + displayName: 'Edit', + icon: Pen, + href: onEdit ? undefined : editHref, + action: onEdit ? () => onEdit() : undefined + } + ]) - { - refresh = requestTokenRefresh - loadApp() - }} -> - {#snippet viewer()} - loadApp()} - > - {/snippet} - +
+ { + refresh = requestTokenRefresh + loadApp() + }} + > + {#snippet viewer()} + loadApp()} + > + {/snippet} + +
-{#if canWriteApp && !hideEditBtn} -
- + +{#if ownsPageHeader} + +{/if} + + +{#if showEdit && (menuHidden || !ownsPageHeader)} +
+ {@render editAction()}
{/if} + +{#snippet editAction()} + {#if compact} + + + {:else} + {#if hasPublicUrl} + + + {/if} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte index 7a72ac62b3..4e20301043 100644 --- a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte +++ b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte @@ -8,9 +8,11 @@ interface Props { loading?: boolean + /** Drops the two labels, for a bar with no room for them: the pen and the eye carry it. */ + iconOnly?: boolean } - let { loading = false }: Props = $props() + let { loading = false, iconOnly = false }: Props = $props() const { mode, jobs, jobsById } = getContext('AppViewerContext') @@ -24,7 +26,8 @@ > {#snippet children({ item })} import { applyDarkModeVariant } from '$lib/darkModeVariant' - import { enterpriseLicense, userStore, type UserExt } from '$lib/stores' - import { base } from '$app/paths' - import { page } from '$app/state' - import Login from '$lib/components/Login.svelte' + import type { UserExt } from '$lib/stores' import { isCloudHosted } from '$lib/cloud' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import WindmillIcon from '$lib/components/icons/WindmillIcon.svelte' - import { getContext, setContext } from 'svelte' - import { - EMBED_NAV_CONTEXT_KEY, - IS_APP_PUBLIC_CONTEXT_KEY, - type EditorBreakpoint, - type EmbedNav - } from '../types' - import { UserService, type AppWithLastVersion, type GlobalWhoamiResponse } from '$lib/gen' - import { urlParamsToObject } from '$lib/utils' - import { goto } from '$app/navigation' - import { loadAppPreview } from './loadAppPreview' + import { setContext } from 'svelte' + import { IS_APP_PUBLIC_CONTEXT_KEY } from '../types' + import type { AppWithLastVersion } from '$lib/gen/types.gen' import RawAppPreview from '$lib/components/raw_apps/RawAppPreview.svelte' import type { Runnable } from '$lib/components/raw_apps/rawAppPolicy' - import { twMerge } from 'tailwind-merge' - import { writable } from 'svelte/store' + import { loadAppCss } from './publicAppApi' let { notExists, @@ -33,6 +20,7 @@ app, workspace, user = undefined, + license = undefined, inWorkspace = false, hideRefreshBar = false, syncHashToUrl = true @@ -46,11 +34,13 @@ onLoginSuccess: () => void app: (AppWithLastVersion & { value: any; workspace_id?: string }) | undefined workspace: string | undefined - /** The viewer's membership in `workspace`, for the app's `ctx`. Without it `ctx` falls - * back to `userStore`, which describes the workspace the page is navigated to — the - * same one on every standalone viewer route, but not inside an AI session's preview - * tab, where the app belongs to the session's workspace. */ + /** The viewer's membership in `workspace`, for the app's `ctx`. Callers pass it + * explicitly — `$userStore` describes the workspace the page is navigated to, which + * is not the app's inside an AI session's preview tab. */ user?: UserExt | undefined + /** The instance license, which fades the "Powered by" badge and lets a low-code app's + * custom CSS apply. */ + license?: string | undefined /** * In-workspace rendering (`/apps/get`, `/app_embed`): keep exact parity * with the pre-sandbox member viewer — no "Powered by Windmill" badge, no @@ -65,23 +55,11 @@ // Use workspace from props or from app.workspace_id (for custom path responses) let effectiveWorkspace = $derived(workspace ?? app?.workspace_id) - // `ctx.username` / `ctx.groups` are per-workspace, and they sit next to `ctx.workspace` - // in the same object, so they have to come from the same workspace it names. - let ctxUser = $derived(user ?? $userStore) - // On the public surfaces (untrusted distribution) runnable-authored html/svg needs // the viewer's approval before it renders, unless the app sandbox isolates it. The // in-workspace viewer renders it verbatim. See getAppMarkupTrust. setContext(IS_APP_PUBLIC_CONTEXT_KEY, !inWorkspace) - // WIN-2006: inside the opaque viewer iframe, navigations to other routes - // (navbar "app" items) must happen on the TOP page — the iframe is cookieless, - // so navigating it would just show a login screen. PublicAppFrame provides the - // relay; outside the opaque viewer this is undefined and goto works directly. - const embedNav = getContext(EMBED_NAV_CONTEXT_KEY) - - const breakpoint = writable('lg') - const darkMode = window.localStorage.getItem('dark-mode') ?? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light') @@ -93,135 +71,75 @@ } // This route bypasses the (root) layout, so restore the variant class too. applyDarkModeVariant() - - let globalUser = $state(undefined) - async function loadGlobalUser() { - try { - globalUser = await UserService.globalWhoami() - } catch (error) { - console.error(error) - } - // const user = await fetch('/api/global/user') - // console.log(user) - } - - // Only the no-access page reads it, to tell a signed-in non-member which - // workspace the app belongs to. - $effect(() => { - if (noPermission && !guestAppPath && !$userStore && !globalUser) { - loadGlobalUser() - } - }) + {#if !inWorkspace} - + Powered by   Windmill +
+ {/await} {/if} -{#if notExists} -
There was an error loading the app, is the url correct? Go to Windmill -
-{:else if noPermission} - {#if guestAppPath && !$userStore} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
-
- {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} - You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of - {:else}You must be logged in and have read access to this app{/if}
- {/if} -
- {#if !jwtError} - - {/if} -
+{#if notExists || noPermission} + {#await Promise.all([loadAppCss(), import('./PublicAppAccess.svelte')]) then [, { default: PublicAppAccess }]} + + {/await} {:else if app} {#key app} {#if app.raw_app && effectiveWorkspace} - } - {syncHashToUrl} - /> + +
+ } + {syncHashToUrl} + /> +
{:else if app.raw_app && !effectiveWorkspace} -
- - Unable to load raw app: workspace information is missing. - -
+ {#await Promise.all([loadAppCss(), import('$lib/components/common/alert/Alert.svelte')]) then [, { default: Alert }]} +
+ + Unable to load raw app: workspace information is missing. + +
+ {/await} {:else} -
- {#await loadAppPreview()} - - {:then Module} - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> - {:catch} -
- Reload the page to try again. -
- {/await} -
+ {#await Promise.all([loadAppCss(), import('./PublicLowCodeApp.svelte')]) then [, { default: PublicLowCodeApp }]} + + {/await} {/if} {/key} {:else} - + {#await loadAppCss() then} + + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte new file mode 100644 index 0000000000..9a993b4972 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte @@ -0,0 +1,81 @@ + + +{#if notExists} +
There was an error loading the app, is the url correct? Go to Windmill +
+{:else} + {#if guestAppPath && !user} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+
+ {#if user}You are logged in but have no read access to this app{:else if globalUser && workspace} + You are logged in but are not a member of the workspace {workspace} this app is part of + {:else}You must be logged in and have read access to this app{/if}
+ {/if} +
+ {#if !jwtError} + { + dropPrefetched() + onLoginSuccess() + }} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> + {/if} +
+{/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index e994767488..dbb9b4b88b 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -23,19 +23,43 @@ * bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`. */ import { BROWSER } from 'esm-env' - import { OpenAPI, UserService } from '$lib/gen' + import { OpenAPI } from '$lib/gen/core/OpenAPI' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import { base } from '$app/paths' import { goto } from '$app/navigation' - import Login from '$lib/components/Login.svelte' - import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' + import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils/queryParams' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' import { loadAppPreview } from './loadAppPreview' - import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' - import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' + import { dropPrefetched, loadAppCss } from './publicAppApi' + import { + hasStoredSdkConsent, + sdkConsentCovers, + storeSdkConsent + } from '$lib/components/raw_apps/sdkScopes' + + // Everything but a rendered app (sign-in, errors, the SDK consent prompt) loads on + // demand: a raw app that renders right away needs none of it. See publicAppApi.ts. + let frameUi: + | Promise<{ + Alert: typeof import('$lib/components/common/alert/Alert.svelte').default + Login: typeof import('$lib/components/Login.svelte').default + RawAppSdkConsent: typeof import('$lib/components/raw_apps/RawAppSdkConsent.svelte').default + }> + | undefined + function loadFrameUi() { + return (frameUi ??= Promise.all([ + loadAppCss(), + import('$lib/components/common/alert/Alert.svelte'), + import('$lib/components/Login.svelte'), + import('$lib/components/raw_apps/RawAppSdkConsent.svelte') + ]).then(([, alert, login, consent]) => ({ + Alert: alert.default, + Login: login.default, + RawAppSdkConsent: consent.default + }))) + } type EmbedToken = { token?: string | null @@ -231,7 +255,8 @@ if (offerSignIn && accountSession === 'unknown') { // Workspace-less, so it answers for an account and never for a guest // (pinned to its workspace) or for nobody. - UserService.getCurrentEmail() + import('$lib/gen') + .then(({ UserService }) => UserService.getCurrentEmail()) .then(() => (accountSession = 'held')) .catch(() => (accountSession = 'none')) } @@ -245,8 +270,12 @@ let staleGuestLogoutFailed = $state(false) $effect(() => { if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) { - UserService.logout() - .then(() => (staleGuestCleared = true)) + import('$lib/gen') + .then(({ UserService }) => UserService.logout()) + .then(() => { + dropPrefetched() + staleGuestCleared = true + }) .catch(() => (staleGuestLogoutFailed = true)) } }) @@ -388,7 +417,7 @@ appPath = resp.app_path ?? appPath workspaceId = resp.workspace_id ?? workspaceId const granted = resp.sdk_scopes ?? [] - if (!granted.every((s) => approved.includes(s))) { + if (!sdkConsentCovers(approved, granted)) { // The app was redeployed with more scopes between the prompt and the // mint. Never inject a token carrying scopes the viewer wasn't shown: // drop it and ask again with the new set. @@ -423,7 +452,10 @@ // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed // raw app. A low-code app's runtime downloads alongside the app payload. - if (!isRaw) loadAppPreview().catch(() => {}) + if (!isRaw) { + loadAppCss().catch(() => {}) + loadAppPreview().catch(() => {}) + } onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -533,6 +565,7 @@ onMount(() => { if (isViewer) { // Only a sandboxed low-code app is ever framed as a viewer. + loadAppCss().catch(() => {}) loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() @@ -565,105 +598,120 @@ {#if viewerReady} {@render viewer()} {:else if viewerOrphaned} -
- - This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your own - page, use the app's public URL without the wm_embed parameter. - -
+ {#await loadFrameUi() then { Alert }} +
+ + This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your + own page, use the app's public URL without the wm_embed parameter. + +
+ {/await} {:else} - + {#await loadAppCss() then} + + {/await} {/if} {:else if status === 'loading'} - -{:else if status === 'notExists' && !offerSignIn} -
- - There was an error loading the app, is the url correct? - Go to Windmill - -
-{:else if status === 'sdkPrompt'} - - -{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} - -{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} -
- - The app could not be reached to find out who may open it. Reload to try again. - -
-{:else if offerSignIn} - - {#if signInDidNotHelp} - -
- You are signed in, but this app is not open to you -
-
- It is open to the people it was shared with{guestAppPath - ? ', and to guests who have no Windmill account' - : ''}. Ask the person who shared it to give your account access. -
+ {#await loadAppCss() then} + + {/await} +{:else if status === 'ready'} + {#if unsandboxed} + + {@render viewer()} + {:else if isRaw} + + {@render viewer()} {:else} - {#if guestAppPath} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
- {/if} - {#if signInError} -
- {signInError} -
- {/if} -
- { - signInError = undefined - accountSession = 'unknown' - initEmbedder() - }} - onLoginError={(message) => (signInError = message)} - popup - guestApp={guestAppPath} - rd={page.url.pathname + page.url.search + page.url.hash} - /> -
+ + {/if} -{:else if unsandboxed} - - {@render viewer()} -{:else if isRaw} - - {@render viewer()} {:else} - - + {#await loadFrameUi() then { Alert, Login, RawAppSdkConsent }} + {#if status === 'notExists' && !offerSignIn} +
+ + There was an error loading the app, is the url correct? + Go to Windmill + +
+ {:else if status === 'sdkPrompt'} + + + {:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} + + {:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} +
+ + The app could not be reached to find out who may open it. Reload to try again. + +
+ {:else if offerSignIn} + + {#if signInDidNotHelp} + +
+ You are signed in, but this app is not open to you +
+
+ It is open to the people it was shared with{guestAppPath + ? ', and to guests who have no Windmill account' + : ''}. Ask the person who shared it to give your account access. +
+ {:else} + {#if guestAppPath} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+ {/if} + {#if signInError} +
+ {signInError} +
+ {/if} +
+ { + dropPrefetched() + signInError = undefined + accountSession = 'unknown' + initEmbedder() + }} + onLoginError={(message) => (signInError = message)} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> +
+ {/if} + {/if} + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte new file mode 100644 index 0000000000..f74be67166 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte @@ -0,0 +1,91 @@ + + +
+ {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
+ Reload the page to try again. +
+ {/await} +
diff --git a/frontend/src/lib/components/apps/editor/publicAppApi.ts b/frontend/src/lib/components/apps/editor/publicAppApi.ts new file mode 100644 index 0000000000..674cb0b622 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/publicAppApi.ts @@ -0,0 +1,82 @@ +// The public app routes (`/public/...`, `/a/...`) render a raw app as little more than its +// bundle's iframe, so they must not statically import the generated client, `$lib/stores` +// or `$lib/utils`: each drags in most of the app shell. `vite.config.js` +// (`assertLeanPublicAppRoutes`) fails the build if they do. Anything heavier is imported +// lazily by the branch that needs it. +import { OpenAPI } from '$lib/gen/core/OpenAPI' +import type { AppWithLastVersion, User } from '$lib/gen/types.gen' +import type { UserExt } from '$lib/stores' +import { mapUserToUserExt } from '$lib/userExt' + +export type PublicAppValue = AppWithLastVersion & { value: any; workspace_id?: string } + +declare global { + interface Window { + /** Responses `app.html` requested before any module loaded, keyed by URL. */ + __wmPublicAppPrefetch?: Record> + } +} + +/** Carries `status` like the generated client's `ApiError`, which callers branch on. */ +export class PublicApiError extends Error { + constructor(public status: number) { + super(`Request failed with status ${status}`) + } +} + +/** Prefetched responses answer for the credential the page loaded with; call this when it + * changes (a sign-in or logout on the page) so later reads go to the network. */ +export function dropPrefetched(): void { + delete window.__wmPublicAppPrefetch +} + +function takePrefetched(url: string): Promise | undefined { + const prefetched = window.__wmPublicAppPrefetch + const res = prefetched?.[url] + if (res) delete prefetched![url] + return res +} + +/** GET an API path with the page credential: the bearer token in `OpenAPI.TOKEN` when the + * share link carries one, the session cookie otherwise. A prefetched response is only + * taken in the cookie case, which is the only credential `app.html` sends. */ +export async function publicGet(path: string): Promise { + const url = `${OpenAPI.BASE}${path}` + const token = typeof OpenAPI.TOKEN === 'string' && OpenAPI.TOKEN ? OpenAPI.TOKEN : undefined + const res = await ((!token && takePrefetched(url)) || + fetch(url, { + // As the generated client does: the opaque viewer frame is cross-origin to the + // API, whose `Access-Control-Allow-Origin: *` refuses a credentialed request. + credentials: OpenAPI.WITH_CREDENTIALS ? OpenAPI.CREDENTIALS : undefined, + headers: token ? { Authorization: `Bearer ${token}` } : {} + })) + if (!res.ok) throw new PublicApiError(res.status) + return (res.headers.get('content-type')?.includes('json') ? res.json() : res.text()) as T +} + +export async function getUserExtOrUndefined(workspace: string): Promise { + try { + return mapUserToUserExt( + await publicGet(`/w/${encodeURI(workspace)}/users/whoami`), + workspace + ) + } catch { + return undefined + } +} + +/** Only fades the "Powered by" badge, so a failure is not worth surfacing. */ +export async function getLicenseOrUndefined(): Promise { + try { + return (await publicGet('/ee_license')) || undefined + } catch { + return undefined + } +} + +let appCss: Promise | undefined +/** The app-wide stylesheet, which these routes need only off the raw-app path: sign-in, + * errors, skeletons and low-code apps. A raw app's bundle brings its own styles. */ +export function loadAppCss(): Promise { + return (appCss ??= import('$lib/assets/app.css')) +} diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 75de2d7faf..a72d54507e 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -188,6 +188,8 @@ export interface AppEditorProps { // with its author and time. `version` and `head` differ when another deploy landed // beside this one, which is when the out-of-date prompt has something to say. onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } export type App = { diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte new file mode 100644 index 0000000000..49a222148b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte @@ -0,0 +1,150 @@ + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte new file mode 100644 index 0000000000..2a54fe11d4 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte @@ -0,0 +1,73 @@ + + + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte new file mode 100644 index 0000000000..8ebd24455c --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte @@ -0,0 +1,23 @@ + + + + + + + Add downstream step + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte index 880abb685a..57a31e34d8 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte @@ -1,145 +1,36 @@ - - {#snippet trigger()} - - - {/snippet} - + +
+ + {#snippet trigger()} + + + {/snippet} + +
diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 27f6ce6be7..87b085593c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -1,8 +1,11 @@ -
+
onNodePointerEnter(node.id)} + onnodepointerleave={onNodePointerLeave} + onedgepointerenter={onEdgeEnter} + onedgepointerleave={onEdgeLeave} --background-color={false} >
- + + {#if assetsOnlyToggle} + + { + assetsOnly = !e.detail + if (assetsOnlyFolder != undefined) setAssetsOnlyFolder(assetsOnlyFolder, assetsOnly) + }} + /> + {#if withoutPassive.dropped > 0} + { + showPassiveReads = e.detail + try { + localStorage.setItem(SHOW_PASSIVE_READS_KEY, String(e.detail)) + } catch {} + }} + /> + {/if} + + {/if} {#if showMinimap} + +{/if} + +{#if scheduleWizard} + +{/if} + +{#if onAddPipelineScript} + +{/if} + + - + { @@ -2467,7 +2550,12 @@ addEventListener('message', function (e) { {onTakeLatest} {draftBaseVersion} {onRestore} - {onSavedNewAppPath} + onSavedNewAppPath={(newAppPath) => { + if (opWorkspace && $userStore?.email) { + movePreviewSdkConsent($userStore.email, opWorkspace, path, newAppPath) + } + onSavedNewAppPath?.(newAppPath) + }} {policy} {diffDrawer} {newApp} @@ -2484,6 +2572,7 @@ addEventListener('message', function (e) { {getBundle} {onNavigate} {onDeploy} + {onDetails} {onResetToDeployed} {loadedFromDraft} {othersDraftsCount} @@ -2496,6 +2585,7 @@ addEventListener('message', function (e) { sidebarCollapsed={sidebarCollapsed.val} onToggleSidebar={() => (sidebarCollapsed.val = !sidebarCollapsed.val)} {condensedHeader} + {ownsPageHeader} /> {/key} {/if} - {#if buildError} + {#if sdkPrompt} + +
+ {#if sdkPrompt.declined} +
+

+ The preview is blocked until you grant the permissions this app declares. +

+ +
+ {:else} + sdkPrompt && (sdkPrompt.declined = true)} + /> + {/if} +
+ {:else if buildError} + + + {#if !phoneTopbar} {#snippet fallback()} -
-
+ } + ] + : []) + ] + : undefined} + > + Deploy + +{/snippet} diff --git a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte index 557b2a13ad..9eced2d0e6 100644 --- a/frontend/src/lib/components/raw_apps/RawAppPreview.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppPreview.svelte @@ -1,6 +1,6 @@ - +{#await runnerModule then { default: Runner }} + +{:catch} + {#if requestRefused} + +
+ Could not load the app runtime. Reload the page to try again. +
+ {/if} +{/await} {#if iframeSrc} + {/if} diff --git a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte index 1f28cd02c0..28940eb02a 100644 --- a/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppSdkConsent.svelte @@ -7,16 +7,19 @@ let { scopes, onContinue, - onDecline + onDecline, + declineLabel = 'Open without granting' }: { /** Scopes the app policy declares for its frontend SDK token. */ scopes: string[] /** Fired when the viewer accepts; `dontAskAgain` persists the consent for * this app path so the prompt is skipped until the declared scopes grow. */ onContinue: (dontAskAgain: boolean) => void - /** Fired when the viewer declines: the app still renders, its frontend code - * just gets no token (SDK calls fail). */ + /** Fired when the viewer declines; no token is minted. */ onDecline: () => void + /** Says what declining does where the prompt is shown: the deployed app + * still renders tokenless, the editor preview stays blocked. */ + declineLabel?: string } = $props() let dontAskAgain = $state(false) @@ -34,17 +37,17 @@
    {#each scopes as scope (scope)}
  • - {sdkScopeLabel(scope)} - {#if sdkScopeDescription(scope)} - — {sdkScopeDescription(scope)} - {/if} + {sdkScopeLabel(scope)}{#if sdkScopeDescription(scope)}: {sdkScopeDescription(scope)}{/if}
  • {/each}
- +
diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.test.ts b/frontend/src/lib/components/raw_apps/sdkScopes.test.ts index 68aba9c545..f72918e311 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.test.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it } from 'vitest' -import { hasStoredSdkConsent, storeSdkConsent } from './sdkScopes' +import { hasStoredSdkConsent, movePreviewSdkConsent, storeSdkConsent } from './sdkScopes' describe('stored frontend SDK consent', () => { beforeEach(() => localStorage.clear()) @@ -20,6 +20,17 @@ describe('stored frontend SDK consent', () => { expect(hasStoredSdkConsent('a@w.dev', 'other', 'u/a/app', ['users:read'])).toBe(false) }) + it('keeps the editor preview and the deployed app apart, and moves preview consent', () => { + storeSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read']) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'], true)).toBe(false) + storeSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'])).toBe(false) + + movePreviewSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', 'u/a/real') + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/real', ['users:read'], true)).toBe(true) + expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/draft_1', ['users:read'], true)).toBe(false) + }) + it('treats unreadable storage as no consent', () => { localStorage.setItem('wm_sdk_consent:a@w.dev:ws:u/a/app', 'not json') expect(hasStoredSdkConsent('a@w.dev', 'ws', 'u/a/app', ['users:read'])).toBe(false) diff --git a/frontend/src/lib/components/raw_apps/sdkScopes.ts b/frontend/src/lib/components/raw_apps/sdkScopes.ts index b0177bb660..238f1b7278 100644 --- a/frontend/src/lib/components/raw_apps/sdkScopes.ts +++ b/frontend/src/lib/components/raw_apps/sdkScopes.ts @@ -52,8 +52,17 @@ export function sdkScopeDescription(scope: string): string | undefined { // Keyed by viewer as well as app: this localStorage lives on the shared embedder // origin, so without the viewer one person's "do not ask again" would silently // suppress the prompt for the next person to use the same browser profile. -function sdkConsentKey(viewer: string, workspace: string, path: string): string { - return `wm_sdk_consent:${viewer}:${workspace}:${path}` +// `preview` keeps the editor's consent apart from the deployed app's: the editor +// runs a draft nobody may have reviewed, so trust given to one is not trust given +// to the other. +function sdkConsentKey(viewer: string, workspace: string, path: string, preview: boolean): string { + return `wm_sdk_consent:${preview ? 'p:' : ''}${viewer}:${workspace}:${path}` +} + +/** True when `approved` (a consent the viewer gave) covers every scope the app now + * declares. Fewer scopes need no new consent; one more does. */ +export function sdkConsentCovers(approved: unknown, scopes: string[]): boolean { + return Array.isArray(approved) && scopes.every((s) => approved.includes(s)) } /** True when a previously stored "do not ask again" consent covers every @@ -62,13 +71,14 @@ export function hasStoredSdkConsent( viewer: string, workspace: string, path: string, - scopes: string[] + scopes: string[], + preview = false ): boolean { try { const stored = JSON.parse( - localStorage.getItem(sdkConsentKey(viewer, workspace, path)) ?? 'null' + localStorage.getItem(sdkConsentKey(viewer, workspace, path, preview)) ?? 'null' ) - return Array.isArray(stored) && scopes.every((s) => stored.includes(s)) + return sdkConsentCovers(stored, scopes) } catch (_) { return false } @@ -78,9 +88,29 @@ export function storeSdkConsent( viewer: string, workspace: string, path: string, - scopes: string[] + scopes: string[], + preview = false ): void { try { - localStorage.setItem(sdkConsentKey(viewer, workspace, path), JSON.stringify(scopes)) + localStorage.setItem(sdkConsentKey(viewer, workspace, path, preview), JSON.stringify(scopes)) + } catch (_) {} +} + +/** Follows the editor's app to its new path (first deploy of a draft, or a rename), + * so a stored "do not ask again" consent neither is asked again nor stays behind + * under the old path. A plain Continue is not stored and does not carry over. */ +export function movePreviewSdkConsent( + viewer: string, + workspace: string, + from: string, + to: string +): void { + if (from === to) return + try { + const fromKey = sdkConsentKey(viewer, workspace, from, true) + const stored = localStorage.getItem(fromKey) + if (stored === null) return + localStorage.setItem(sdkConsentKey(viewer, workspace, to, true), stored) + localStorage.removeItem(fromKey) } catch (_) {} } diff --git a/frontend/src/lib/components/raw_apps/utils.ts b/frontend/src/lib/components/raw_apps/utils.ts index bdbf62ef8a..3070572785 100644 --- a/frontend/src/lib/components/raw_apps/utils.ts +++ b/frontend/src/lib/components/raw_apps/utils.ts @@ -92,6 +92,8 @@ export type RawAppPreviewLogs = { /** Files were sent to the UI Builder and its build has not reported back in time. */ buildPending: boolean buildLogs: string + /** The app is held back until the user answers its SDK permission prompt, so it is not running. */ + sdkConsentPending: boolean } export type RawAppRuntimeLogRequester = (limit: number) => Promise @@ -292,6 +294,9 @@ export function unsandboxedRawAppHtml( App + +
@@ -299,6 +438,31 @@ f/{path} · data pipeline + {#if workspaceId} + + {/if} +
+ {#if deployErrors.size > 0} + + {/if} + {#if pendingCount > 0} + + {/if}
+ {#key itemPath} + + {/key} {/await} {:else} {#await import('./RawAppEditorView.svelte')} @@ -365,6 +368,7 @@ path={itemPath} {workspaceId} {onNavigate} + onSeeDetails={seeDetails} {isActiveSession} {active} {newRawApp} diff --git a/frontend/src/lib/components/sessions/RawAppEditorView.svelte b/frontend/src/lib/components/sessions/RawAppEditorView.svelte index 137055204d..de05359bd5 100644 --- a/frontend/src/lib/components/sessions/RawAppEditorView.svelte +++ b/frontend/src/lib/components/sessions/RawAppEditorView.svelte @@ -33,6 +33,7 @@ path, workspaceId, onNavigate, + onSeeDetails, isActiveSession = true, active = true, newRawApp = false @@ -41,6 +42,9 @@ path: string workspaceId: string onNavigate?: (item: WorkspaceItem) => void + /** `Exit & see details` — flips this tab to the item's deployed view rather than + * navigating out of the session. */ + onSeeDetails?: (e: { path: string }) => void /** Forwarded to SessionEditorTarget — only the visible session claims the * workspace's single live-editor slot. */ isActiveSession?: boolean @@ -357,6 +361,7 @@ newApp={!cell.saved.val || cell.saved.val.no_deployed === true} {diffDrawer} {onNavigate} + onDetails={onSeeDetails} condensedHeader={true} onResetToDeployed={reloadDeployed} onDeploy={(e) => runtime.itemDeployed(workspaceId, 'raw_app', path, e.path)} diff --git a/frontend/src/lib/components/sessions/SessionChangesBar.svelte b/frontend/src/lib/components/sessions/SessionChangesBar.svelte index 140300ea30..3929b78ca9 100644 --- a/frontend/src/lib/components/sessions/SessionChangesBar.svelte +++ b/frontend/src/lib/components/sessions/SessionChangesBar.svelte @@ -22,7 +22,7 @@ import { badgeCounts, badgeOf, buildDeployItems, type DeployItem } from './sessionDeployModel' import { useExistingMaskKeys } from './sessionDeployModel.svelte' import { previewTargetForDeployKind } from './sessionPreviewTabs.svelte' - import { pipelineFolderFromBundlePath } from '$lib/pipelinePaths' + import { PIPELINE_DRAFT_KIND, pipelineFolderFromBundlePath } from '$lib/pipelinePaths' import type { PreviewTarget } from './previewRouter' import JobsSegment from '$lib/components/copilot/chat/JobsSegment.svelte' import RowIcon from '$lib/components/common/table/RowIcon.svelte' @@ -378,6 +378,11 @@ onDataChanged={refreshDock} onItemDeployed={(item) => void runtime?.manager.renameModifiedItem(item.draftKind, item.path, item.displayPath)} - onItemDiscarded={(item) => void runtime?.manager.removeModifiedItem(item.draftKind, item.path)} + onItemDiscarded={(item) => { + void runtime?.manager.removeModifiedItem(item.draftKind, item.path) + const folder = pipelineFolderFromBundlePath(item.path) + if (item.draftKind === PIPELINE_DRAFT_KIND && folder) void runtime?.forgetPipelineDrafts(folder) + }} + deployPipeline={runtime?.deployPipeline} /> {/if} diff --git a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte index fe5ab2d199..d921e435b0 100644 --- a/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte +++ b/frontend/src/lib/components/sessions/SessionDiffDrawer.svelte @@ -2,10 +2,10 @@ import WorkspaceDiffDrawer from './WorkspaceDiffDrawer.svelte' import { ArrowRight, GitFork, Pencil } from 'lucide-svelte' import { userWorkspaces } from '$lib/stores' - import Badge from '$lib/components/common/badge/Badge.svelte' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { useSessionDeployModel } from './sessionDeployModel.svelte' import type { DeployItem } from './sessionDeployModel' + import type { DeployResult } from '$lib/utils_workspace_deploy' // Session "Edits" drawer. Builds the deploy model over the session // workspace's drafts and hands it to WorkspaceDiffDrawer, which renders the @@ -19,7 +19,8 @@ keys, onDataChanged, onItemDeployed, - onItemDiscarded + onItemDiscarded, + deployPipeline }: { workspaceId: string parentWorkspaceId?: string @@ -36,6 +37,8 @@ * drops it. */ onItemDeployed?: (item: DeployItem) => void onItemDiscarded?: (item: DeployItem) => void + /** Forwarded to the deploy model: deploys a pipeline folder's draft bundle. */ + deployPipeline?: (folder: string) => Promise } = $props() const isFork = $derived(!!parentWorkspaceId) @@ -51,7 +54,8 @@ mask: keys, onDataChanged, onItemDeployed, - onItemDiscarded + onItemDiscarded, + deployPipeline })) // Editor URL for a row (every item lives in the session workspace). @@ -95,7 +99,7 @@ {ws?.name ?? workspaceId} {#if ws?.is_dev_workspace} - {devBadgeText(ws.dev_workspace_label)} + {/if} diff --git a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte index 9a7fb20bd6..a14990583d 100644 --- a/frontend/src/lib/components/sessions/SessionModeSwitch.svelte +++ b/frontend/src/lib/components/sessions/SessionModeSwitch.svelte @@ -69,7 +69,7 @@ iconOnly={isCollapsed} tooltip={isCollapsed ? 'Workspace' : undefined} size="sm" - class="w-full justify-center" + class="w-full justify-center whitespace-nowrap" onActivate={onNavActivate} /> {/snippet} diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index 9707b537b4..d4f7fa66ab 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -2,7 +2,7 @@ import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' import Modal from '$lib/components/common/modal/Modal.svelte' import Checkbox from '$lib/components/common/checkbox/Checkbox.svelte' - import { Badge, Button } from '$lib/components/common' + import { Button } from '$lib/components/common' import { Archive, ArchiveRestore, @@ -67,7 +67,7 @@ import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' import { page } from '$app/state' import { base } from '$app/paths' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { dateBucket, GROUP_BY_OPTIONS, @@ -1220,9 +1220,12 @@ {group.label} {#if groupWs?.is_dev_workspace} - - {devBadgeText(groupWs.dev_workspace_label)} - + {/if} {#if groupWsId}
\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -1232,7 +1274,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -1648,21 +1690,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1718,6 +1762,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2210,10 +2262,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2386,6 +2442,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -3475,7 +3543,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands @@ -4206,6 +4274,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml index 70a955d16d..5d5b0f8a0d 100644 --- a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml @@ -84,7 +84,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -95,7 +96,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml index d21d006e8d..e57c92ebc3 100644 --- a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml @@ -71,7 +71,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -82,7 +83,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml index 1d4e81ac2c..79786fc8db 100644 --- a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml @@ -54,7 +54,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -65,7 +66,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml index f35b33ad74..6755d6545c 100644 --- a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml @@ -94,7 +94,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -105,7 +106,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml index f484366783..b428fbf390 100644 --- a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml @@ -133,7 +133,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -144,7 +145,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml index 09217692e5..6549ce1b10 100644 --- a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml @@ -135,7 +135,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -146,7 +147,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml index 2801cc9a73..f167e2965f 100644 --- a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml @@ -96,7 +96,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -107,7 +108,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml index 239d568fcc..e7620683e6 100644 --- a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml @@ -70,7 +70,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -81,7 +82,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml index be196a7bc3..79cca28595 100644 --- a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml @@ -63,7 +63,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -74,7 +75,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/schedule.schema.yaml b/system_prompts/auto-generated/schemas/schedule.schema.yaml index d0ea85d6fe..7097be7d25 100644 --- a/system_prompts/auto-generated/schemas/schedule.schema.yaml +++ b/system_prompts/auto-generated/schemas/schedule.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml index 4f41c32421..32d6400973 100644 --- a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml index 780b0f2245..8184d3815a 100644 --- a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml @@ -172,7 +172,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -183,7 +184,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 0517cf1897..14ca869180 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1567,21 +1567,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1637,6 +1639,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2129,10 +2139,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2305,6 +2319,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 97db78b3d4..13a93ed063 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -60,10 +60,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -236,6 +240,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index 55f59b1f36..55df35ccf7 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -124,21 +124,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -194,6 +196,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 48ea832a31..3f41b5cfba 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -732,6 +732,8 @@ user related commands **Subcommands:** +- `user list` - List the users of the workspace + - `--json` - Output as JSON (for piping to jq) - `user add [password:string]` - Create a user - `--superadmin` - Specify to make the new user superadmin. - `--company ` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 352fe2ed48..4b0889c3d8 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -371,6 +371,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules @@ -682,4 +686,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} diff --git a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md index 064dc600ac..4c339e6222 100644 --- a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md +++ b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md @@ -12,10 +12,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -59,7 +97,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 4b340c0db1..6ad44db068 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -320,21 +320,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -390,6 +392,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 03d40ab044..a5f599a946 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -332,21 +332,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -402,6 +404,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index b15655a6da..ce0c610da7 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -322,21 +322,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -392,6 +394,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index b9c2793f47..4946ed0f6e 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -268,10 +268,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -444,6 +448,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index ddb10fd0fa..ff10f38062 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -289,6 +289,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index 01e9797e32..59924faf41 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -7,10 +7,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -56,7 +94,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index e1ad956dc4..41a8eac37e 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -22,6 +22,6 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 8b51f75dfc..f927acd26e 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -46,7 +46,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" # Build default export by combining client utilities + services # This preserves backward compatibility for `import wmill from "windmill-client"` @@ -63,6 +63,7 @@ import { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, @@ -153,6 +154,7 @@ const wmill = { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 7e84320346..3f7b03117a 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -11,6 +11,7 @@ import { UserService, KafkaTriggerService, } from "./services.gen"; +import type { Retry } from "./types.gen"; import { OpenAPI } from "./core/OpenAPI"; import { isSuspendSignal, stepErrorMarker, taskErrorFromMarker } from "./wacError"; // import type { DenoS3LightClientSettings } from "./index"; @@ -420,12 +421,26 @@ export async function runScriptAsync( return _runScriptAsyncInternal(path, hash_, args, scheduledInSeconds, tag); } +/** Options for {@link runScriptByPathAsync} and {@link runScriptByHashAsync}, passed in + * place of `scheduledInSeconds`; a positional `tag` is then ignored. */ +export interface RunScriptAsyncOptions { + /** Schedule execution for a future time (in seconds) */ + scheduledInSeconds?: number; + /** Override the worker tag the job runs on */ + tag?: string; + /** Re-run the job when it fails, with the same policy a schedule takes, e.g. + * `{ constant: { attempts: 3, seconds: 10 } }`. The returned id is the first + * attempt's; each retry is a job of its own, recorded as a child of it. */ + retry?: Retry; +} + async function _runScriptAsyncInternal( path: string | null = null, hash_: string | null = null, args: Record | null = null, scheduledInSeconds: number | null = null, - tag: string | null = null + tag: string | null = null, + retry: Retry | null = null ): Promise { // Create a script job and return its job id. args = args || {}; @@ -439,6 +454,10 @@ async function _runScriptAsyncInternal( params["tag"] = tag; } + if (retry) { + params["retry"] = JSON.stringify(retry); + } + let parentJobId = getEnv("WM_JOB_ID"); if (parentJobId !== undefined) { params["parent_job"] = parentJobId; @@ -475,34 +494,47 @@ async function _runScriptAsyncInternal( * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByPathAsync( path: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(path, null, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(path, null, args, o.scheduledInSeconds, o.tag, o.retry); } /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByHashAsync( hash_: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(null, hash_, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(null, hash_, args, o.scheduledInSeconds, o.tag, o.retry); +} + +function asyncRunOptions( + scheduledInSeconds: number | null | RunScriptAsyncOptions, + tag: string | null +): RunScriptAsyncOptions { + return typeof scheduledInSeconds === "object" && scheduledInSeconds !== null + ? scheduledInSeconds + : { scheduledInSeconds: scheduledInSeconds ?? undefined, tag: tag ?? undefined }; } /** @@ -692,6 +724,19 @@ export async function getProgress(jobId?: any): Promise { }); } +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +export async function workerIsDraining(jobId?: string): Promise { + return await JobService.workerIsDraining({ + id: jobId ?? getEnv("WM_JOB_ID") ?? "NO_JOB_ID", + workspace: getWorkspace(), + }); +} + /** * Set a flow user state * @param key key of the state diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 615e7885eb..45306830f5 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.827.0", + "version": "1.828.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 59f4b98719..98caf3895a 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.827.0", + "version": "1.828.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index 35d501b3a8..b08184bb19 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.827.0 +1.828.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 295ba3e97a..064996f393 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.827.0", + "version": "1.828.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.827.0", + "version": "1.828.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index 05ba30630f..7e8c7a641d 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.827.0", + "version": "1.828.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",