starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-13 08:02:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into hugo/resource-type-guidance

Browse Source 
# Conflicts:
#	frontend/src/lib/components/copilot/chat/global/core.ts
This commit is contained in:
hugocasa
2026-10-09 10:09:43 +02:00
parent af980dd7a1 fb22e5ce36
commit 5e3c4d72fc
336 changed files with 14263 additions and 3869 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ai-evals-scheduled.yml
+1 -1
View File
@@ -15,7 +15,7 @@ on:
description: "Space-separated modes (global, cli, flow, app, script)"
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for""
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.827.0"
".": "1.828.0"
}
CHANGELOG.md
+23
View File
@@ -1,5 +1,28 @@
# Changelog
## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08)
### Features
* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864))
* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d))
* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4))
* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6))
* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24))
### Bug Fixes
* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775))
* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba))
* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687))
### Performance Improvements
* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996))
## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07)
backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217"
}
backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b"
}
backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json
Generated
+45
View File
@@ -0,0 +1,45 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e"
}
backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13"
}
backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json
Generated
-31
View File
@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d"
}
backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5"
}
backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d"
}
backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665"
}
backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc"
}
backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json
Generated
+43
View File
@@ -0,0 +1,43 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8"
}
backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd"
}
backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET draining = true WHERE worker = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919"
}
backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json
Generated
+34
View File
@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b"
}
backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323"
}
backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed"
}
backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json → backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -107,6 +107,11 @@
"ordinal": 20,
"name": "native_mode",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
@@ -138,8 +143,9 @@
true,
true,
true,
false,
false
]
},
"hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca"
"hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c"
}
backend/Cargo.lock
Generated
+89 -89
View File
@@ -6685,9 +6685,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "inventory"
version = "0.3.24"
version = "0.3.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b"
checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c"
dependencies = [
"rustversion",
]
@@ -9617,7 +9617,7 @@ version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
dependencies = [
"toml_edit 0.25.15+spec-1.1.0",
"toml_edit 0.25.16+spec-1.1.0",
]
[[package]]
@@ -13622,9 +13622,9 @@ dependencies = [
[[package]]
name = "toml_datetime"
version = "1.1.1+spec-1.1.0"
version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7"
checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053"
dependencies = [
"serde_core",
]
@@ -13644,21 +13644,21 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.25.15+spec-1.1.0"
version = "0.25.16+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614"
checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722"
dependencies = [
"indexmap 2.14.2",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_datetime 1.1.2+spec-1.1.0",
"toml_parser",
"winnow 1.0.4",
]
[[package]]
name = "toml_parser"
version = "1.1.3+spec-1.1.0"
version = "1.1.4+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56"
checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333"
dependencies = [
"winnow 1.0.4",
]
@@ -14829,7 +14829,7 @@ dependencies = [
[[package]]
name = "windmill"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-nats",
@@ -14918,7 +14918,7 @@ dependencies = [
[[package]]
name = "windmill-ai"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"async-stream",
"async-trait",
@@ -14952,7 +14952,7 @@ dependencies = [
[[package]]
name = "windmill-alerting"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -14965,7 +14965,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"argon2",
@@ -15105,7 +15105,7 @@ dependencies = [
[[package]]
name = "windmill-api-agent-workers"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15128,7 +15128,7 @@ dependencies = [
[[package]]
name = "windmill-api-assets"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15145,7 +15145,7 @@ dependencies = [
[[package]]
name = "windmill-api-auth"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15171,7 +15171,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"reqwest 0.12.28",
"serde",
@@ -15181,7 +15181,7 @@ dependencies = [
[[package]]
name = "windmill-api-configs"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15198,7 +15198,7 @@ dependencies = [
[[package]]
name = "windmill-api-debug"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"base64 0.22.1",
@@ -15220,7 +15220,7 @@ dependencies = [
[[package]]
name = "windmill-api-embeddings"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15243,7 +15243,7 @@ dependencies = [
[[package]]
name = "windmill-api-flow-conversations"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15259,7 +15259,7 @@ dependencies = [
[[package]]
name = "windmill-api-flows"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15281,7 +15281,7 @@ dependencies = [
[[package]]
name = "windmill-api-groups"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15303,7 +15303,7 @@ dependencies = [
[[package]]
name = "windmill-api-inputs"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15317,7 +15317,7 @@ dependencies = [
[[package]]
name = "windmill-api-integration-tests"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-nats",
@@ -15353,7 +15353,7 @@ dependencies = [
[[package]]
name = "windmill-api-jobs"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15379,7 +15379,7 @@ dependencies = [
[[package]]
name = "windmill-api-npm-proxy"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15407,7 +15407,7 @@ dependencies = [
[[package]]
name = "windmill-api-openapi"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15429,7 +15429,7 @@ dependencies = [
[[package]]
name = "windmill-api-schedule"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15449,7 +15449,7 @@ dependencies = [
[[package]]
name = "windmill-api-scripts"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15491,7 +15491,7 @@ dependencies = [
[[package]]
name = "windmill-api-settings"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15520,7 +15520,7 @@ dependencies = [
[[package]]
name = "windmill-api-sse"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"lazy_static",
"serde",
@@ -15532,7 +15532,7 @@ dependencies = [
[[package]]
name = "windmill-api-users"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"argon2",
"axum 0.8.9",
@@ -15556,7 +15556,7 @@ dependencies = [
[[package]]
name = "windmill-api-workers"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15570,7 +15570,7 @@ dependencies = [
[[package]]
name = "windmill-api-workspaces"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"axum 0.8.9",
"chrono",
@@ -15609,7 +15609,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"chrono",
"lazy_static",
@@ -15624,7 +15624,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"axum 0.8.9",
@@ -15643,7 +15643,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"aes-gcm",
"aho-corasick",
@@ -15757,7 +15757,7 @@ dependencies = [
[[package]]
name = "windmill-dep-map"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"chrono",
"futures",
@@ -15777,7 +15777,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"regex",
"serde",
@@ -15794,7 +15794,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"astral-tokio-tar",
@@ -15821,7 +15821,7 @@ dependencies = [
[[package]]
name = "windmill-jseval"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"futures",
@@ -15838,7 +15838,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15854,7 +15854,7 @@ dependencies = [
[[package]]
name = "windmill-mcp"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15875,7 +15875,7 @@ dependencies = [
[[package]]
name = "windmill-native-triggers"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -15906,7 +15906,7 @@ dependencies = [
[[package]]
name = "windmill-oauth"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"arc-swap",
@@ -15931,7 +15931,7 @@ dependencies = [
[[package]]
name = "windmill-object-store"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-stream",
@@ -15967,7 +15967,7 @@ dependencies = [
[[package]]
name = "windmill-operator"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"futures",
@@ -15985,7 +15985,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15994,7 +15994,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16006,7 +16006,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16018,7 +16018,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"gosyn",
@@ -16030,7 +16030,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16042,7 +16042,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16054,7 +16054,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -16065,7 +16065,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16076,7 +16076,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -16088,7 +16088,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16099,7 +16099,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16121,7 +16121,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -16133,7 +16133,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16147,7 +16147,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -16164,7 +16164,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16177,7 +16177,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde",
@@ -16189,7 +16189,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16207,7 +16207,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -16223,7 +16223,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -16239,7 +16239,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -16253,7 +16253,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16292,7 +16292,7 @@ dependencies = [
[[package]]
name = "windmill-runtime-nativets"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"const_format",
@@ -16332,7 +16332,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"getrandom 0.3.4",
"wasm-bindgen",
@@ -16343,7 +16343,7 @@ dependencies = [
[[package]]
name = "windmill-store"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -16378,7 +16378,7 @@ dependencies = [
[[package]]
name = "windmill-test-utils"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16402,7 +16402,7 @@ dependencies = [
[[package]]
name = "windmill-trigger"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16435,7 +16435,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-amqp"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16462,7 +16462,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-azure"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16495,7 +16495,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-email"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16515,7 +16515,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-gcp"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16549,7 +16549,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-http"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16586,7 +16586,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-kafka"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16609,7 +16609,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-mqtt"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16633,7 +16633,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-nats"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-nats",
@@ -16657,7 +16657,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-postgres"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16692,7 +16692,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-sqs"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16720,7 +16720,7 @@ dependencies = [
[[package]]
name = "windmill-trigger-websocket"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-trait",
@@ -16745,7 +16745,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"bitflags 2.13.2",
@@ -16764,7 +16764,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-once-cell",
@@ -16883,7 +16883,7 @@ dependencies = [
[[package]]
name = "windmill-worker-volumes"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"bytes",
"futures",
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.827.0"
version = "1.828.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.827.0"
version = "1.828.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
6f23819e947b18878000e580ddefccfd01a73166
017710bdd0a10de857f0f40800dde238cdb1ad24
backend/migrations/20261008070335_worker_ping_draining.down.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining;
backend/migrations/20261008070335_worker_ping_draining.up.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE;
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6302,7 +6302,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6314,7 +6314,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"convert_case",
"serde",
@@ -6323,7 +6323,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6335,7 +6335,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6347,7 +6347,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6359,7 +6359,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6371,7 +6371,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6383,7 +6383,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6394,7 +6394,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6405,7 +6405,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6417,7 +6417,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6428,7 +6428,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6450,7 +6450,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6462,7 +6462,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6476,7 +6476,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6493,7 +6493,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6506,7 +6506,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde",
@@ -6518,7 +6518,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6536,7 +6536,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6552,7 +6552,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6568,7 +6568,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6600,7 +6600,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6614,7 +6614,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.827.0"
version = "1.828.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.827.0"
version = "1.828.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/summarized_schema.txt
+1 -1
View File
@@ -219,7 +219,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool)
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
backend/tests/error_handler.rs
+67
View File
@@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its failure must reach the workspace error handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_error_handler_triggered_on_hub_script_failure(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000002";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/fail"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(!completed_job.success, "Hub script should have failed");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace error handler should handle the failed hub script"
);
Ok(())
}
backend/tests/otel.rs
+26 -1
View File
@@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() {
otel_set_worker_uptime("w-uptime", 3600.0);
otel_set_health_status_phase("healthy");
otel_set_health_db_unresponsive(true);
otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024));
otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35);
otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25);
otel_record_job_memory_peak("python3", 100 * 1024 * 1024);
// ── Histograms ──────────────────────────────────────────────────
@@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() {
let metrics = flush_and_get_metrics(&state);
let names = metric_names(&metrics);
// ── Verify all 21 metric names are present ──────────────────────
// ── Verify all metric names are present ─────────────────────────
let expected = [
"windmill.queue.push_count",
@@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() {
"windmill.worker.uptime",
"windmill.health.status",
"windmill.health.db_unresponsive",
"windmill.worker.memory_usage",
"windmill.worker.windmill_memory_usage",
"windmill.worker.memory_limit",
"windmill.worker.cpu_usage",
"windmill.worker.cpu_periods",
"windmill.worker.cpu_throttled_periods",
"windmill.worker.cpu_throttled_time",
"windmill.worker.occupancy_rate",
"windmill.worker.job_memory_peak",
];
for name in expected {
assert!(
@@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() {
.expect("worker.busy data point with worker=worker-test-1 not found");
assert_eq!(dp.1, 1);
let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap();
let values = gauge_i64_values(m);
let dp = values
.iter()
.find(|(attrs, _)| {
attrs
.iter()
.any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu")
})
.expect("worker.memory_limit data point with worker_group=gpu not found");
assert_eq!(dp.1, 1024);
let m = find_metric(&metrics, "windmill.db.pool.active").unwrap();
assert_eq!(gauge_i64_values(m)[0].1, 5);
let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap();
backend/tests/success_handler.rs
+67
View File
@@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow:
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its success must reach the workspace success handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_triggered_on_hub_script_success(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000003";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/ok"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(completed_job.success, "Hub script should have succeeded");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace success handler should handle the successful hub script"
);
Ok(())
}
backend/tests/wac_suspend_started_at.rs
+88
View File
@@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool<Postgres>) -> anyhow::Res
Ok(())
}
/// Each park clears `started_at`, so the completion would otherwise store only the last
/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that
/// started an hour late. The completed row spans the workflow from its first start, and
/// keeps the last segment aside: that is what is metered and what worker-time readers sum.
#[sqlx::test]
async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let job_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \
permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \
VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \
'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)",
)
.bind(job_id)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \
VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')",
)
.bind(job_id)
.execute(&db)
.await?;
// Two parks, each followed by the pull re-stamping `started_at`: the second park must
// not replace the first start with its own segment's.
for resumed_ago in ["10 minutes", "3 seconds"] {
let mut tx = db.begin().await?;
suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?;
tx.commit().await?;
sqlx::query(
"UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \
WHERE id = $1",
)
.bind(job_id)
.bind(resumed_ago)
.execute(&db)
.await?;
}
let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db)
.await?
.unwrap();
let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?;
let (_, span) = windmill_queue::add_completed_job(
&db,
&job,
true,
false,
sqlx::types::Json(&result),
None,
0,
None,
false,
Some(3000),
false,
)
.await?;
let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option<i64>) = sqlx::query_as(
"SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \
(extras->>'wac_last_segment_ms')::bigint \
FROM v2_job_completed WHERE id = $1",
)
.bind(job_id)
.fetch_one(&db)
.await?;
assert!(
(started_ago_s - 3600.0).abs() < 60.0,
"started_at must be the first start, got {started_ago_s}s ago"
);
assert!(
(duration_ms - 3_600_000).abs() < 60_000,
"duration_ms must be the wall time since the first start, got {duration_ms}"
);
assert_eq!(last_segment_ms, Some(3000));
assert_eq!(
span.duration_ms, duration_ms,
"a flow records the step with the span the row holds"
);
assert!(span.outlasts_run(job.started_at));
assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter");
Ok(())
}
backend/tests/worker_ping_ip.rs
+31 -1
View File
@@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres};
use uuid::Uuid;
use windmill_common::{
external_ip::UNKNOWN_IP,
worker::{insert_ping_query, update_worker_ping_main_loop_query},
worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query},
};
async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> anyhow::Result<()> {
register(db, worker, ip, None).await
}
async fn register(
db: &Pool<Postgres>,
worker: &str,
ip: Option<&str>,
draining: Option<bool>,
) -> anyhow::Result<()> {
insert_ping_query(
"test-instance",
worker,
@@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> any
None,
None,
false,
draining,
db,
)
.await?;
Ok(())
}
/// A worker re-registers mid-life once its external IP resolves. That registration must not
/// clear a draining flag the same process already published; only a worker starting up under
/// a reclaimed name resets it.
#[sqlx::test]
async fn only_a_starting_worker_resets_draining(db: Pool<Postgres>) -> anyhow::Result<()> {
let draining = || async {
sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'")
.fetch_one(&db)
.await
};
register(&db, "wk-drain", None, Some(false)).await?;
set_worker_draining_query("wk-drain", &db).await?;
register(&db, "wk-drain", Some("1.2.3.4"), None).await?;
assert!(draining().await?);
register(&db, "wk-drain", None, Some(false)).await?;
assert!(!draining().await?);
Ok(())
}
/// The external IP resolves in the background, so the initial ping often has none yet. That must
/// not blank the address a previous process wrote to the row this one reclaims — worker names are
/// stable across restarts under EXIT_AFTER_N_JOBS.
backend/windmill-api-auth/src/scopes.rs
+1
View File
@@ -996,6 +996,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui
["job_metrics", "set_progress", ..] => (http_method == "POST", 2),
["job_metrics", "get_progress", ..] => (http_method == "GET", 2),
["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2),
["jobs", "worker_is_draining", ..] => (http_method == "GET", 2),
["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3),
["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3),
_ => return false,
backend/windmill-api-integration-tests/tests/jobs_authed.rs
+48
View File
@@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue(
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_worker_is_draining_follows_the_job_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job = Uuid::new_v4();
sqlx::query(
"INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args)
VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)",
)
.bind(job)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker)
VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')",
)
.bind(job)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version)
VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')",
)
.execute(&db)
.await?;
let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}");
let draining = || async {
let resp = authed(client().get(&url)).send().await?;
anyhow::Ok(resp.json::<bool>().await?)
};
assert!(!draining().await?);
windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?;
assert!(draining().await?);
// A parked job keeps its last worker on the queue row without running on it.
sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1")
.bind(job)
.execute(&db)
.await?;
assert!(!draining().await?);
Ok(())
}
backend/windmill-api-settings/src/lib.rs
+17 -17
View File
@@ -58,10 +58,10 @@ use windmill_common::{
error::{self, pg_error_message, JsonResult, Result},
get_database_url,
global_settings::{
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING,
AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXTERNAL_INSTANCE_PG_SETTING,
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING,
APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -1238,26 +1238,22 @@ async fn run_setting_pre_write_hook(
}
}
}
GITHUB_APP_WEBHOOK_BASE_URL_SETTING => {
// A bad value here yields a webhook GitHub can never deliver to, and the
GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => {
// A bad webhook base yields a webhook GitHub can never deliver to, and the
// failure only shows up much later as "falling back to polling" on a
// repository — so reject it at the boundary instead.
// repository; a bad API base puts an unreachable URL in front of every user.
// So reject both at the boundary instead.
match value {
// Clearing (delete row) is handled by the caller; allow it through.
serde_json::Value::Null => {}
serde_json::Value::String(s) if s.trim().is_empty() => {}
serde_json::Value::String(s) => {
windmill_common::global_settings::validate_webhook_base_url(s).map_err(
|e| {
error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}"
))
},
)?;
windmill_common::global_settings::validate_webhook_base_url(s)
.map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?;
}
_ => {
return Err(error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string"
"{key} must be a URL string"
)));
}
}
@@ -1394,8 +1390,12 @@ async fn set_instance_config(
// Applied and audited one setting at a time: the writes are not transactional, so a
// batch that fails midway must not leave the settings it did write unrecorded.
let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } =
settings_diff;
let instance_config::SettingsDiff {
upserts,
deletes,
mut previous_values,
unchanged_count,
} = settings_diff;
let steps = upserts
.into_iter()
.map(|(k, v)| (k, Some(v)))
backend/windmill-api-users/src/users.rs
+1 -1
View File
@@ -490,7 +490,7 @@ async fn list_user_usage(
"
SELECT usr.email, usage.executions
FROM usr, LATERAL (
SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions
SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions
FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.workspace_id = $1
AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')
backend/windmill-api-workers/src/lib.rs
+2 -1
View File
@@ -86,6 +86,7 @@ struct WorkerPing {
job_isolation: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
native_mode: Option<bool>,
draining: bool,
}
// #[derive(Serialize, Deserialize)]
@@ -118,7 +119,7 @@ async fn list_worker_pings(
WorkerPing,
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,
custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode
custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining
FROM worker_ping
WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
ORDER BY ping_at desc LIMIT $2 OFFSET $3",
backend/windmill-api/openapi.yaml
+28 -3
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.827.0
version: 1.828.0
title: Windmill API
contact:
@@ -1919,7 +1919,7 @@ paths:
/settings/instance_ui:
get:
summary: get the instance-wide announcement banner and accent color
summary: get the instance-wide announcement banner, accent color and API base url
description: |
Readable by any authenticated user; every page load reads it. Each value
is the raw stored setting, or null when unset.
@@ -1928,7 +1928,7 @@ paths:
- setting
responses:
"200":
description: the instance_banner and accent_color settings
description: the instance_banner, accent_color and api_base_url settings
content:
application/json:
schema:
@@ -1936,6 +1936,7 @@ paths:
properties:
instance_banner: {}
accent_color: {}
api_base_url: {}
/settings_u/ruff_config:
get:
@@ -17322,6 +17323,27 @@ paths:
schema:
type: string
/w/{workspace}/jobs/worker_is_draining/{id}:
get:
summary: whether the worker running this job is draining
description: |
True once the worker running the job has received its shutdown signal. The worker
still waits for the job to finish, so a long-running job can poll this to exit early.
False when the job is not currently running on a worker.
operationId: workerIsDraining
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/JobId"
responses:
"200":
description: whether the worker running this job is draining
content:
application/json:
schema:
type: boolean
/w/{workspace}/jobs_u/get_logs/{id}:
get:
summary: get job logs
@@ -35901,6 +35923,9 @@ components:
type: string
native_mode:
type: boolean
draining:
type: boolean
description: the worker received its shutdown signal and exits once its current job is done
required:
- worker
- worker_instance
backend/windmill-api/src/apps.rs
+8 -1
View File
@@ -979,7 +979,14 @@ async fn get_raw_app_data(
// the opaque, sandboxed wrapper (incl. under a cross-origin-isolated
// / COEP `require-corp` embedder).
.header("X-Content-Type-Options", "nosniff")
.header("Cross-Origin-Resource-Policy", "cross-origin");
.header("Cross-Origin-Resource-Policy", "cross-origin")
// The secret names one app version, whose bundle is written once at deploy,
// so the URL's content never changes. `private`: the URL is a capability, so
// shared caches must not keep serving it after the app is deleted.
.header(
http::header::CACHE_CONTROL,
"private, max-age=31536000, immutable",
);
Ok(res.body(body).unwrap())
} else {
return Err(Error::NotFound("File not found".to_string()));
backend/windmill-api/src/jobs.rs
+51
View File
@@ -378,6 +378,10 @@ pub fn workspaced_service() -> Router {
"/wac_approval_urls/{job_id}/{step_key}",
get(get_wac_approval_urls).layer(cors.clone()),
)
.route(
"/worker_is_draining/{id}",
get(worker_is_draining).layer(cors.clone()),
)
.route(
"/result_by_id/{job_id}/{node_id}",
get(get_result_by_id).layer(cors.clone()),
@@ -613,6 +617,53 @@ async fn get_root_job(
Ok(Json(res))
}
/// Whether the worker running this job has received its shutdown signal. A draining worker
/// never interrupts its job, so a long-running script polls this to exit on its own terms.
async fn worker_is_draining(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> windmill_common::error::JsonResult<bool> {
// A job always may ask about itself. Anyone else must be able to see the job: the
// caller's row policies on v2_job decide, narrowed by the token's tag filter.
let draining = if authed.job_id == Some(id) {
sqlx::query_scalar!(
"SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker
WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
id,
w_id
)
.fetch_optional(&db)
.await?
} else {
let tags = get_scope_tags(&authed)
.map(|tags| tags.into_iter().map(str::to_string).collect::<Vec<_>>());
let mut tx = user_db.begin(&authed).await?;
let draining = sqlx::query_scalar!(
"SELECT wp.draining FROM v2_job j
JOIN v2_job_queue q ON q.id = j.id
JOIN worker_ping wp ON wp.worker = q.worker
WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL
AND ($3::text[] IS NULL OR j.tag = ANY($3))",
id,
w_id,
tags.as_deref()
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
draining
}
.unwrap_or(false);
windmill_common::feature_usage::log_feature_usage(
"worker_draining",
"check",
if draining { "draining" } else { "not_draining" },
);
Ok(Json(draining))
}
async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result<String> {
let root_job = sqlx::query_scalar!(
r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#,
backend/windmill-common/src/global_settings.rs
+34 -7
View File
@@ -181,6 +181,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app";
/// `base_url` when unset; set it when the browser-facing URL is not reachable
/// from GitHub and a separate ingress fronts the API for inbound webhooks.
pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url";
/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP
/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on.
/// For deployments that serve the API on a separate domain or behind a gateway.
/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]:
/// every user is shown URLs built from it.
pub const API_BASE_URL_SETTING: &str = "api_base_url";
/// Instance-wide announcement rendered above every page of the app (maintenance
/// windows, incidents). Readable by any authenticated user, unlike most settings:
/// the banner exists to be shown to everyone, so it must never hold anything the
@@ -313,12 +319,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> {
pub struct InstanceUi {
pub instance_banner: Option<serde_json::Value>,
pub accent_color: Option<serde_json::Value>,
pub api_base_url: Option<serde_json::Value>,
}
pub async fn get_instance_ui(db: &Pool<Postgres>) -> error::Result<InstanceUi> {
let rows = sqlx::query!(
"SELECT name, value FROM global_settings WHERE name = ANY($1)",
&[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str]
&[
INSTANCE_BANNER_SETTING,
ACCENT_COLOR_SETTING,
API_BASE_URL_SETTING
] as &[&str]
)
.fetch_all(db)
.await?;
@@ -327,17 +338,22 @@ pub async fn get_instance_ui(db: &Pool<Postgres>) -> error::Result<InstanceUi> {
match row.name.as_str() {
INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value),
ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value),
API_BASE_URL_SETTING => ui.api_base_url = Some(row.value),
_ => {}
}
}
Ok(ui)
}
/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value.
const SHELL_ACTIVE_CHARS: [char; 16] = [
'$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!',
];
/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value.
///
/// The receiver path is appended to it verbatim, so anything that doesn't
/// concatenate into a URL GitHub can POST to must be rejected at write time
/// rather than silently producing an unreachable hook: a wrong scheme
/// A path is appended to it verbatim, so anything that doesn't concatenate into a
/// URL an external caller can reach must be rejected at write time rather than
/// silently producing an unreachable hook or endpoint: a wrong scheme
/// (`httpss://`), a missing host, embedded whitespace, or a query/fragment
/// (appending a path after `?`/`#` keeps it inside the query/fragment).
///
@@ -353,7 +369,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> {
url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?;
if !url.username().is_empty() || url.password().is_some() {
return Err(
"must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(),
"must not embed a username or password: the URL is readable by users who are not instance admins".to_string(),
);
}
if !matches!(url.scheme(), "http" | "https") {
@@ -366,13 +382,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> {
}
if url.query().is_some() || url.fragment().is_some() {
return Err(
"must not include a query string or fragment, since the webhook path is appended to it"
"must not include a query string or fragment, since a path is appended to it"
.to_string(),
);
}
if value.chars().any(char::is_whitespace) {
return Err("must not contain whitespace".to_string());
}
// The UI pastes the URL into shell commands users copy, where double quotes
// do not stop `$(...)` or backticks from running.
if value.contains(SHELL_ACTIVE_CHARS) {
return Err(
"must not contain shell metacharacters, since it is shown inside commands users copy"
.to_string(),
);
}
// Matches the sibling `base_url` / `hub_base_url` convention. The receiver
// builder trims it anyway, so this is about keeping the stored value canonical
// rather than about reachability.
@@ -1101,6 +1125,9 @@ mod tests {
"https://hooks.example.com#",
"https://user:password@hooks.example.com",
"https://user@hooks.example.com",
"https://hooks.example.com/$(uname)",
"https://hooks.example.com/`uname`",
"https://hooks.example.com/a;b",
];
for v in accept {
assert!(
backend/windmill-common/src/instance_config.rs
+33 -25
View File
@@ -281,6 +281,8 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub github_app_webhook_base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub api_base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub email_domain: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hub_base_url: Option<String>,
@@ -1375,8 +1377,8 @@ pub fn diff_worker_configs(
}
/// Declaratively replace the global settings, rejecting a
/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the
/// API would reject.
/// `github_app_webhook_base_url`, `api_base_url` or
/// `http_route_default_allowed_origins` the API would reject.
///
/// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's
/// ConfigMap sync) MUST go through this rather than calling
@@ -1399,29 +1401,35 @@ pub async fn sync_global_settings_declarative(
current: &BTreeMap<String, serde_json::Value>,
desired: &BTreeMap<String, serde_json::Value>,
) -> anyhow::Result<()> {
let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING;
// Non-string shapes are rejected rather than ignored: `as_str()` alone would let a
// bool/number/object through as if the key were absent, and the diff below would
// then persist it — where the HTTP path answers "must be a URL string".
match desired.get(webhook_key) {
None | Some(serde_json::Value::Null) => {}
Some(serde_json::Value::String(s)) if s.trim().is_empty() => {}
Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s)
.map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?,
// Names the JSON kind rather than printing it: this is the last message on
// this path that could report submitted content, and an object or array could
// carry a secret into `sync-config` output and operator logs.
Some(other) => {
let kind = match other {
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
_ => "a non-string value",
};
return Err(anyhow::anyhow!(
"{webhook_key} must be a URL string, got {kind}"
));
for url_key in [
crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING,
crate::global_settings::API_BASE_URL_SETTING,
] {
// Non-string shapes are rejected rather than ignored: `as_str()` alone would let a
// bool/number/object through as if the key were absent, and the diff below would
// then persist it — where the HTTP path answers "must be a URL string".
match desired.get(url_key) {
None | Some(serde_json::Value::Null) => {}
Some(serde_json::Value::String(s)) if s.trim().is_empty() => {}
Some(serde_json::Value::String(s)) => {
crate::global_settings::validate_webhook_base_url(s)
.map_err(|e| anyhow::anyhow!("{url_key}: {e}"))?
}
// Names the JSON kind rather than printing it: this is the last message on
// this path that could report submitted content, and an object or array could
// carry a secret into `sync-config` output and operator logs.
Some(other) => {
let kind = match other {
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
_ => "a non-string value",
};
return Err(anyhow::anyhow!(
"{url_key} must be a URL string, got {kind}"
));
}
}
}
backend/windmill-common/src/otel_oss.rs
+33
View File
@@ -132,3 +132,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_worker_memory(
_worker: &str,
_worker_group: &str,
_usage: Option<i64>,
_windmill_usage: Option<i64>,
_limit: Option<i64>,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_add_worker_cpu(
_worker: &str,
_worker_group: &str,
_usage_secs: f64,
_periods: u64,
_throttled_periods: u64,
_throttled_secs: f64,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_set_worker_occupancy_rate(
_worker: &str,
_worker_group: &str,
_window: &'static str,
_rate: f64,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {}
backend/windmill-common/src/variables.rs
+34 -8
View File
@@ -14,7 +14,7 @@ use crate::utils::WarnAfterExt;
use crate::worker::Connection;
use crate::{worker::WORKER_GROUP, BASE_URL, DB};
use chrono::{SecondsFormat, Utc};
use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait};
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
@@ -375,13 +375,15 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String {
mc.encrypt_str_to_base64(value)
}
const DECRYPT_FAILED: &str =
"Could not decrypt value. The value may have been encrypted with a different key.";
/// Every failure returns the same message so responses carry no signal about why decryption
/// failed (padding, base64, UTF-8); the cause is only logged.
pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result<String> {
mc.decrypt_base64_to_string(value).map_err(|e| match e {
MagicCryptError::DecryptError(_) => error::Error::internal_err(
"Could not decrypt value. The value may have been encrypted with a different key."
.to_string(),
),
_ => error::Error::internal_err(e.to_string()),
mc.decrypt_base64_to_string(value).map_err(|e| {
tracing::warn!("decryption failed: {e}");
error::Error::internal_err(DECRYPT_FAILED.to_string())
})
}
@@ -746,7 +748,31 @@ pub async fn get_variable_or_self_as<T: Authable + Sync>(
#[cfg(test)]
mod tests {
use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier};
use super::{
can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier,
DECRYPT_FAILED,
};
use crate::error::Error;
use magic_crypt::MagicCryptTrait;
#[test]
fn decrypt_failures_all_return_the_same_message() {
let mc = magic_crypt::new_magic_crypt!("workspace-key", 256);
let other = magic_crypt::new_magic_crypt!("another-key", 256);
let failures = [
"not base64 at all!".to_string(),
mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]),
other.encrypt_str_to_base64("secret"),
];
for value in failures {
match decrypt(&mc, value.clone()) {
Err(Error::InternalErrLoc { message, .. }) => {
assert_eq!(message, DECRYPT_FAILED, "for {value:?}")
}
other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"),
}
}
}
#[test]
fn is_valid_js_identifier_gates_prologue_injection() {
backend/windmill-common/src/worker.rs
+115 -2
View File
@@ -1783,6 +1783,73 @@ pub fn get_cpu_period() -> Option<i64> {
Some(100000)
}
/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job
/// peak-memory histograms so the two exports stay comparable.
pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = {
let mut buckets = [0.0; 12];
let mut i = 0;
while i < 12 {
buckets[i] = (16u64 << (20 + i)) as f64;
i += 1;
}
buckets
};
/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0
/// when the cgroup has no CPU controller, which is also what an unthrottled one reports.
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct CpuStat {
pub usage_usec: u64,
pub nr_periods: u64,
pub nr_throttled: u64,
pub throttled_usec: u64,
}
fn cpu_stat_field(stat: &str, key: &str) -> Option<u64> {
stat.lines().find_map(|line| {
let (k, v) = line.split_once(' ')?;
(k == key).then(|| v.trim().parse().ok()).flatten()
})
}
fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option<CpuStat> {
Some(CpuStat {
usage_usec: cpu_stat_field(stat, "usage_usec")?,
nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0),
nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0),
throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0),
})
}
/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds.
fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option<CpuStat> {
Some(CpuStat {
usage_usec: cpuacct_usage.trim().parse::<u64>().ok()? / 1000,
nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0),
nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0),
throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000,
})
}
#[cfg(not(windows))]
pub fn get_cpu_stat() -> Option<CpuStat> {
const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage";
if Path::new(V1_USAGE).exists() {
parse_cgroup_v1_cpu_stat(
&std::fs::read_to_string(V1_USAGE).ok()?,
&std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(),
)
} else {
let cgroup_path = get_cgroupv2_path()?;
parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?)
}
}
#[cfg(windows)]
pub fn get_cpu_stat() -> Option<CpuStat> {
None
}
/// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count
/// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds
/// the quota in, so a fraction of a CPU makes it report a single-core machine.
@@ -1952,6 +2019,7 @@ pub enum PingType {
MainLoop,
Job,
InitScript,
Draining,
}
#[derive(Serialize, Deserialize)]
pub struct Ping {
@@ -1974,6 +2042,8 @@ pub struct Ping {
pub occupancy_rate_30m: Option<f32>,
pub job_isolation: Option<String>,
pub native_mode: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draining: Option<bool>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
@@ -2026,6 +2096,7 @@ pub async fn update_ping_http(
insert_ping.memory,
insert_ping.job_isolation,
insert_ping.native_mode.unwrap_or(false),
insert_ping.draining,
db,
)
.await?;
@@ -2054,10 +2125,21 @@ pub async fn update_ping_http(
)
.await?
}
PingType::Draining => set_worker_draining_query(worker_name, db).await?,
}
Ok(())
}
pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET draining = true WHERE worker = $1",
worker_name
)
.execute(db)
.await?;
Ok(())
}
#[derive(Debug, Serialize, Deserialize)]
pub struct JobCancelled {
pub canceled_by: String,
@@ -2156,6 +2238,9 @@ pub async fn fetch_raw_script_from_app_query(
/// `started_at` and `jobs_executed` are the only two columns carried over, being the
/// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP
/// lookup has not resolved yet and the predecessor's address is still the best guess.
///
/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering
/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime.
pub async fn insert_ping_query(
worker_instance: &str,
worker_name: &str,
@@ -2169,6 +2254,7 @@ pub async fn insert_ping_query(
memory: Option<i64>,
job_isolation: Option<String>,
native_mode: bool,
draining: Option<bool>,
db: &DB,
) -> anyhow::Result<i32> {
// A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and
@@ -2176,8 +2262,8 @@ pub async fn insert_ping_query(
// lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The
// literal below must stay equal to `external_ip::UNKNOWN_IP`.
let previous_jobs_executed = sqlx::query_scalar!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)
DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)
RETURNING jobs_executed",
worker_instance,
worker_name,
@@ -2191,6 +2277,7 @@ pub async fn insert_ping_query(
memory,
job_isolation.as_deref(),
native_mode,
draining,
)
.fetch_one(db)
.await?;
@@ -2899,6 +2986,32 @@ mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn cgroup_cpu_stat_is_normalized_to_microseconds() {
let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\
nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n";
let expected = CpuStat {
usage_usec: 2_500_000,
nr_periods: 40,
nr_throttled: 7,
throttled_usec: 350_000,
};
assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected));
let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n";
assert_eq!(
parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat),
Some(expected)
);
// No CPU controller: usage is still reported, throttling reads as zero.
assert_eq!(
parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"),
Some(CpuStat { usage_usec: 12, ..Default::default() })
);
assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None);
}
#[test]
fn datatable_role_is_read_from_the_leading_comment_block() {
let role = |code| SqlAnnotations::datatable_role(code);
backend/windmill-queue/src/jobs.rs
+100 -32
View File
@@ -1221,7 +1221,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
worker_name: &str,
flow_is_done: bool,
duration: Option<i64>,
) -> Result<(), Error> {
) -> Result<CompletedSpan, Error> {
record_failure_metrics(completed_job, worker_name).await;
tracing::error!(
@@ -1229,7 +1229,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
completed_job.id,
completed_job.workspace_id,
);
let _ = add_completed_job(
let (_, span) = add_completed_job(
db,
completed_job,
false,
@@ -1244,7 +1244,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
)
.warn_after_seconds(10)
.await?;
Ok(())
Ok(span)
}
pub async fn add_completed_job_error(
@@ -1257,6 +1257,31 @@ pub async fn add_completed_job_error(
flow_is_done: bool,
duration: Option<i64>,
) -> Result<WrappedError, Error> {
add_completed_job_error_with_span(
db,
completed_job,
mem_peak,
canceled_by,
e,
worker_name,
flow_is_done,
duration,
)
.await
.map(|(wrapped, _)| wrapped)
}
/// `add_completed_job_error`, also handing back what the completed row holds.
pub async fn add_completed_job_error_with_span(
db: &Pool<Postgres>,
completed_job: &MiniCompletedJob,
mem_peak: i32,
canceled_by: Option<CanceledBy>,
e: serde_json::Value,
worker_name: &str,
flow_is_done: bool,
duration: Option<i64>,
) -> Result<(WrappedError, CompletedSpan), Error> {
record_failure_metrics(completed_job, worker_name).await;
let result = WrappedError { error: e };
@@ -1266,7 +1291,7 @@ pub async fn add_completed_job_error(
completed_job.workspace_id,
serde_json::to_string(&result).unwrap_or_else(|_| "".to_string())
);
let _ = add_completed_job(
let (_, span) = add_completed_job(
db,
&completed_job,
false,
@@ -1281,7 +1306,7 @@ pub async fn add_completed_job_error(
)
.warn_after_seconds(10)
.await?;
Ok(result)
Ok((result, span))
}
lazy_static::lazy_static! {
@@ -1320,7 +1345,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
flow_is_done: bool,
duration: Option<i64>,
from_cache: bool,
) -> Result<(Uuid, i64), Error> {
) -> Result<(Uuid, CompletedSpan), Error> {
// tracing::error!("Start");
// let start = tokio::time::Instant::now();
@@ -1354,7 +1379,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
};
let result_columns = result_columns.as_ref();
let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| {
let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| {
commit_completed_job(
db,
completed_job,
@@ -1395,7 +1420,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
if let Some(job_id) = opt_uuid {
return Ok((job_id, duration));
return Ok((job_id, span));
}
// Auto-resolve a retry chain that ultimately worked, from whichever of the two
@@ -1436,7 +1461,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
}
#[cfg(feature = "cloud")]
apply_completed_job_cloud_usage(db, completed_job, duration);
apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms);
#[cfg(all(feature = "enterprise", feature = "private"))]
crate::jobs_ee::apply_completed_job_error_handlers(
@@ -1453,7 +1478,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// tracing::error!("4 {:?}", start.elapsed());
Ok((completed_job.id, duration))
Ok((completed_job.id, span))
}
async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
@@ -1471,7 +1496,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// True when a native script retry was enqueued for this failed attempt, i.e.
// this is not the terminal attempt — schedule completion handlers must wait.
retry_pending: bool,
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool, bool)> {
) -> windmill_common::error::Result<(Option<Uuid>, CompletedSpan, bool, bool)> {
// let start = std::time::Instant::now();
let job_id = completed_job.id;
@@ -1538,11 +1563,11 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
&& monitor_parent.is_none()
&& !is_scheduled
{
let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else {
let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else {
return Err(not_in_queue_error(db, job_id).await);
};
log_completed_job(completed_job, duration, success);
return Ok((None, duration, false, false));
log_completed_job(completed_job, completed.duration_ms, success);
return Ok((None, completed, false, false));
}
let mut tx = db.begin().warn_after_seconds(10).await?;
@@ -1550,7 +1575,8 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// The parent's rows are locked ahead of the child's own queue row (see
// `record_child_completion` for the order this must keep), so the duration it stamps is read
// before the completion: the one the completed row will hold. `now()` is fixed for the
// transaction, and a completed row already there keeps its own duration.
// transaction, and a completed row already there keeps its own duration. For a child that
// itself parked it is the last run's, matching the last start its timeline entry carries.
let mut wac_parent_ready = false;
if let Some(parent_job) = wac_parent {
let Some(duration) = sqlx::query_scalar!(
@@ -1577,7 +1603,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.await?;
}
let Some(duration) = completion.execute(&mut *tx).await? else {
let Some(completed) = completion.execute(&mut *tx).await? else {
return Err(not_in_queue_error(&mut *tx, job_id).await);
};
@@ -1787,16 +1813,35 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
tx.commit().warn_after_seconds(10).await?;
log_completed_job(completed_job, duration, success);
log_completed_job(completed_job, completed.duration_ms, success);
// tracing::info!("completed job: {:?}", start.elapsed().as_micros());
Ok((
None,
duration,
completed,
_skip_downstream_error_handlers,
wac_parent_ready,
))
}
/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's.
/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered;
/// it differs from `duration_ms` only for a job that parked.
#[derive(Debug, Clone, Copy)]
pub struct CompletedSpan {
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub duration_ms: i64,
pub last_run_ms: i64,
}
impl CompletedSpan {
/// Whether the row spans more than the run that started at `last_started_at`, i.e. the
/// job parked and resumed. A flow records such a step with the row's span, so the step
/// reads like the job does.
pub fn outlasts_run(&self, last_started_at: Option<chrono::DateTime<chrono::Utc>>) -> bool {
matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last)
}
}
/// What a job's completion writes, for `Completion::execute`.
struct Completion<'a> {
completed_job: &'a MiniCompletedJob,
@@ -1813,6 +1858,11 @@ impl Completion<'_> {
/// Moves the job from the queue to the completed jobs and refreshes a flow step's parent
/// ping, as one statement. Returns `None` when the job was no longer in the queue.
///
/// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's
/// `started_at` and `duration_ms` span the whole workflow from its first start, and the
/// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from
/// completed rows.
///
/// The completion takes its cancellation from the queue row it deletes, not only from
/// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still
/// being written, so the deleted row is the final word on whether the job was canceled.
@@ -1820,7 +1870,7 @@ impl Completion<'_> {
/// It locks the queue row before the completed row's key. Any other writer completing a job
/// (the monitor's zombie fallback, debounce) must take them in the same order, or the two
/// deadlock.
async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result<Option<i64>> {
async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result<Option<CompletedSpan>> {
let Completion {
completed_job,
success,
@@ -1851,16 +1901,20 @@ impl Completion<'_> {
.then_some(completed_job.parent_job)
.flatten()
else {
return sqlx::query_scalar!(
return sqlx::query_as!(
CompletedSpan,
"WITH deleted AS (
DELETE FROM v2_job_queue WHERE id = $1
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
(extras->>'wac_first_started_at')::timestamptz AS first_started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
), completed AS (
INSERT INTO v2_job_completed AS cj
( workspace_id
, id
, started_at
, duration_ms
, extras
, result
, result_columns
, canceled_by
@@ -1871,8 +1925,11 @@ impl Completion<'_> {
, status
, worker
)
SELECT d.workspace_id, d.id, d.started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),
SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
CASE WHEN d.first_started_at IS NULL THEN d.run_ms
ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
CASE WHEN d.first_started_at IS NOT NULL
THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
$3::text::jsonb, $10,
CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
@@ -1888,9 +1945,11 @@ impl Completion<'_> {
THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
RETURNING duration_ms
RETURNING cj.started_at, cj.duration_ms,
COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
)
SELECT duration_ms AS \"duration_ms!\" FROM completed",
SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
last_run_ms AS \"last_run_ms!\" FROM completed",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result,
@@ -1910,16 +1969,20 @@ impl Completion<'_> {
// A canceled flow is pinged too: it is completed by its next transition like any other
// flow, and the zombie flow monitor needs the ping to finish the cancel if that
// transition is lost.
sqlx::query_scalar!(
sqlx::query_as!(
CompletedSpan,
"WITH deleted AS (
DELETE FROM v2_job_queue WHERE id = $1
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
(extras->>'wac_first_started_at')::timestamptz AS first_started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
), completed AS (
INSERT INTO v2_job_completed AS cj
( workspace_id
, id
, started_at
, duration_ms
, extras
, result
, result_columns
, canceled_by
@@ -1930,8 +1993,11 @@ impl Completion<'_> {
, status
, worker
)
SELECT d.workspace_id, d.id, d.started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),
SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
CASE WHEN d.first_started_at IS NULL THEN d.run_ms
ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
CASE WHEN d.first_started_at IS NOT NULL
THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
$3::text::jsonb, $10,
CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
@@ -1947,14 +2013,16 @@ impl Completion<'_> {
THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
RETURNING duration_ms
RETURNING cj.started_at, cj.duration_ms,
COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
), parent_ping AS (
UPDATE v2_job_runtime r SET ping = now()
FROM v2_job_queue q
WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12
AND EXISTS (SELECT 1 FROM completed)
)
SELECT duration_ms AS \"duration_ms!\" FROM completed",
SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
last_run_ms AS \"last_run_ms!\" FROM completed",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result,
@@ -2021,7 +2089,7 @@ async fn check_result_size<T: ValidableJson>(
db: &Pool<Postgres>,
queued_job: &MiniCompletedJob,
result: Json<&T>,
) -> Option<Result<(Option<Uuid>, i64, bool, bool), Error>> {
) -> Option<Result<(Option<Uuid>, CompletedSpan, bool, bool), Error>> {
let result_size = result.size() / 1024 / 1024;
if result_size > 2 {
if result_size > *MAX_RESULT_SIZE_MB {
backend/windmill-worker-volumes/src/lib.rs
+34
View File
@@ -54,11 +54,45 @@ pub struct VolumeMount {
pub target: String,
}
/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while
/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A
/// write within the filesystem's timestamp granularity of the hand-off is not seen.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FileStamp {
size: u64,
changed: Option<(i64, i64)>,
}
impl FileStamp {
/// Whether the file is as it was at `earlier`. An unreadable change time never matches.
pub fn unchanged_since(&self, earlier: &FileStamp) -> bool {
self.changed.is_some() && self == earlier
}
pub fn of(meta: &std::fs::Metadata) -> Self {
// ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it.
#[cfg(unix)]
let changed = {
use std::os::unix::fs::MetadataExt;
Some((meta.ctime(), meta.ctime_nsec()))
};
#[cfg(not(unix))]
let changed = meta
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64));
Self { size: meta.len(), changed }
}
}
pub struct VolumeState {
pub mount: VolumeMount,
pub local_dir: PathBuf,
pub manifest: HashMap<String, FileEntry>,
pub symlinks: HashMap<String, String>,
/// Stamp of each pulled file as the job received it, keyed like `manifest`.
pub stamps: HashMap<String, FileStamp>,
}
pub struct DownloadStats {
backend/windmill-worker/src/bun_executor.rs
+4 -1
View File
@@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output(
// status row, the order every child completion takes.
let segment_ms = sqlx::query_scalar!(
"WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)
UPDATE v2_job_queue q SET running = false, started_at = null
UPDATE v2_job_queue q SET running = false, started_at = null,
extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras
ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))
END
FROM prev WHERE q.id = $1
RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
job.id,
backend/windmill-worker/src/common.rs
+1
View File
@@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script(
wm_memory_usage: None,
job_isolation: None,
native_mode: None,
draining: None,
ping_type: PingType::InitScript,
},
)
backend/windmill-worker/src/handle_child.rs
+49 -5
View File
@@ -1049,6 +1049,7 @@ where
let wm_memory_usage = get_windmill_memory_usage();
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref());
if job_id != Uuid::nil() {
if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await {
tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
@@ -1140,8 +1141,8 @@ fn child_joined_output_stream(
.take()
.expect("child did not have a handle to stdout");
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
let stdout = BufReader::new(stdout).split(b'\n');
let stderr = BufReader::new(stderr).split(b'\n');
stream::select(
lines_to_stream(stderr, true, job_id.clone(), w_id.clone())
.map(|l| l.map(|line| OutputLine { stderr: true, line })),
@@ -1151,18 +1152,32 @@ fn child_joined_output_stream(
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
mut lines: tokio::io::Split<R>,
stderr: bool,
job_id: Uuid,
w_id: String,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id))
.poll_next_segment(cx)
.map(|result| {
let result = result.map(|segment| segment.map(decode_output_line));
process_streaming_log_lines(result, stderr, &job_id, &w_id)
})
})
}
/// Child output is not guaranteed to be UTF-8 (e.g. pwsh on Windows writes in the
/// console code page), so decode lossily: a strict decode errors out and ends log
/// capture for the rest of the job. Strips a trailing `\r` so CRLF output doesn't keep it.
pub(crate) fn decode_output_line(mut segment: Vec<u8>) -> String {
if segment.last() == Some(&b'\r') {
segment.pop();
}
String::from_utf8(segment)
.unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())
}
pub fn process_status(
program: &str,
status: ExitStatus,
@@ -1209,4 +1224,33 @@ mod tests {
let left = JobDeadline(Some(Instant::now() + Duration::from_secs(120)));
assert!(matches!(left.remaining_secs(), Some(s) if (118..=120).contains(&s)));
}
// "Prüfe" in CP850/1252 is not UTF-8. A strict decode errors on that line and
// would stop log capture, dropping every line after it.
#[tokio::test]
async fn a_non_utf8_line_is_decoded_lossily_and_later_lines_still_arrive() {
let output: &[u8] =
b"start\r\nPr\x81fe Benutzer\nPr\xfcfe\nGr\xc3\xbc\xc3\x9fe\r\nafter\n\nlast";
let lines: Vec<String> = lines_to_stream(
BufReader::new(output).split(b'\n'),
false,
Uuid::nil(),
"w".into(),
)
.map(|line| line.expect("reading from a byte slice cannot fail"))
.collect()
.await;
assert_eq!(
lines,
vec![
"start",
"Pr\u{FFFD}fe Benutzer",
"Pr\u{FFFD}fe",
"Grüße",
"after",
"",
"last",
]
);
}
}
backend/windmill-worker/src/lib.rs
+1
View File
@@ -81,6 +81,7 @@ mod prepare_deps;
mod python_executor;
#[cfg(feature = "python")]
mod python_versions;
mod resource_metrics;
pub mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
backend/windmill-worker/src/resource_metrics.rs
+235
View File
@@ -0,0 +1,235 @@
use std::sync::{atomic::Ordering, Mutex, OnceLock};
use windmill_common::{
otel_oss::{
otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory,
otel_set_worker_occupancy_rate,
},
worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP},
OTEL_METRICS_ENABLED,
};
use crate::common::OccupancyResult;
#[cfg(feature = "prometheus")]
struct PromMetrics {
memory_usage: prometheus::IntGaugeVec,
windmill_memory_usage: prometheus::IntGaugeVec,
memory_limit: prometheus::IntGaugeVec,
cpu_usage: prometheus::CounterVec,
cpu_periods: prometheus::IntCounterVec,
cpu_throttled_periods: prometheus::IntCounterVec,
cpu_throttled_seconds: prometheus::CounterVec,
occupancy_rate: prometheus::GaugeVec,
job_memory_peak: prometheus::HistogramVec,
}
#[cfg(feature = "prometheus")]
lazy_static::lazy_static! {
static ref PROM_METRICS: Option<PromMetrics> =
if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) {
const WORKER: &[&str] = &["name", "worker_group"];
Some(PromMetrics {
memory_usage: prometheus::register_int_gauge_vec!(
"worker_memory_usage_bytes",
"Memory used by the worker's container, page cache excluded",
WORKER
)
.unwrap(),
windmill_memory_usage: prometheus::register_int_gauge_vec!(
"worker_windmill_memory_usage_bytes",
"Resident memory of the worker process itself",
WORKER
)
.unwrap(),
memory_limit: prometheus::register_int_gauge_vec!(
"worker_memory_limit_bytes",
"Memory limit of the worker's container",
WORKER
)
.unwrap(),
cpu_usage: prometheus::register_counter_vec!(
"worker_cpu_usage_seconds_total",
"CPU time consumed by the worker's container",
WORKER
)
.unwrap(),
cpu_periods: prometheus::register_int_counter_vec!(
"worker_cpu_periods_total",
"CPU enforcement periods elapsed for the worker's container",
WORKER
)
.unwrap(),
cpu_throttled_periods: prometheus::register_int_counter_vec!(
"worker_cpu_throttled_periods_total",
"CPU enforcement periods in which the worker's container was throttled",
WORKER
)
.unwrap(),
cpu_throttled_seconds: prometheus::register_counter_vec!(
"worker_cpu_throttled_seconds_total",
"Time the worker's container spent throttled by its CPU limit",
WORKER
)
.unwrap(),
occupancy_rate: prometheus::register_gauge_vec!(
"worker_occupancy_rate",
"Share of time the worker spent executing jobs, over the window",
&["name", "worker_group", "window"]
)
.unwrap(),
job_memory_peak: prometheus::register_histogram_vec!(
"worker_job_memory_peak_bytes",
"Peak memory of a job's main process",
&["tag"],
windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec()
)
.unwrap(),
})
} else {
None
};
}
fn metrics_enabled() -> bool {
#[cfg(feature = "prometheus")]
if PROM_METRICS.is_some() {
return true;
}
OTEL_METRICS_ENABLED.load(Ordering::Relaxed)
}
// Memory and CPU are read from the cgroup, which every worker of a process shares
// (native mode runs several). Only the first worker to report exports them, so that
// summing the series over workers does not count one container several times.
static CONTAINER_SERIES_OWNER: OnceLock<String> = OnceLock::new();
static MEMORY_LIMIT: Mutex<Option<Option<i64>>> = Mutex::new(None);
/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits.
pub(crate) fn set_memory_limit(limit: Option<i64>) {
*MEMORY_LIMIT.lock().unwrap() = Some(limit);
}
static LAST_CPU_STAT: Mutex<CpuStat> =
Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 });
/// Exports the worker's resource readings. Meant for the ping ticks, which have already
/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read.
pub(crate) fn record_worker_resources(
worker_name: &str,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy: Option<&OccupancyResult>,
) {
if !metrics_enabled() {
return;
}
let worker_group = WORKER_GROUP.as_str();
if let Some(occupancy) = occupancy {
let windows = [
("total", Some(occupancy.occupancy_rate)),
("15s", occupancy.occupancy_rate_15s),
("5m", occupancy.occupancy_rate_5m),
("30m", occupancy.occupancy_rate_30m),
];
for (window, rate) in windows {
let Some(rate) = rate.filter(|r| r.is_finite()) else {
continue;
};
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
m.occupancy_rate
.with_label_values(&[worker_name, worker_group, window])
.set(rate as f64);
}
otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64);
}
}
if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name {
return;
}
let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory);
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
let labels = &[worker_name, worker_group];
if let Some(v) = memory_usage {
m.memory_usage.with_label_values(labels).set(v);
}
if let Some(v) = wm_memory_usage {
m.windmill_memory_usage.with_label_values(labels).set(v);
}
if let Some(v) = memory_limit {
m.memory_limit.with_label_values(labels).set(v);
}
}
otel_set_worker_memory(
worker_name,
worker_group,
memory_usage,
wm_memory_usage,
memory_limit,
);
let Some(stat) = get_cpu_stat() else {
return;
};
let delta = {
let mut last = LAST_CPU_STAT.lock().unwrap();
// A reading below the previous one means the cgroup changed under the process:
// restart from it rather than exporting a negative increase.
let delta = if stat.usage_usec < last.usage_usec {
CpuStat::default()
} else {
CpuStat {
usage_usec: stat.usage_usec - last.usage_usec,
nr_periods: stat.nr_periods.saturating_sub(last.nr_periods),
nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled),
throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec),
}
};
*last = stat;
delta
};
let usage_secs = delta.usage_usec as f64 / 1e6;
let throttled_secs = delta.throttled_usec as f64 / 1e6;
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
let labels = &[worker_name, worker_group];
m.cpu_usage.with_label_values(labels).inc_by(usage_secs);
m.cpu_periods
.with_label_values(labels)
.inc_by(delta.nr_periods);
m.cpu_throttled_periods
.with_label_values(labels)
.inc_by(delta.nr_throttled);
m.cpu_throttled_seconds
.with_label_values(labels)
.inc_by(throttled_secs);
}
otel_add_worker_cpu(
worker_name,
worker_group,
usage_secs,
delta.nr_periods,
delta.nr_throttled,
throttled_secs,
);
}
/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process
/// to sample or the reading failed, and those are not observations.
pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) {
if mem_peak_kb <= 0 {
return;
}
let bytes = mem_peak_kb as u64 * 1024;
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
m.job_memory_peak
.with_label_values(&[tag])
.observe(bytes as f64);
}
otel_record_job_memory_peak(tag, bytes);
}
backend/windmill-worker/src/result_processor.rs
+27 -13
View File
@@ -1776,7 +1776,7 @@ pub async fn process_completed_job(
add_time!(bench, "pre add_completed_job");
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&job,
true,
@@ -1835,7 +1835,10 @@ pub async fn process_completed_job(
true,
canceled_by,
result,
started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }),
span.started_at.or(started_at).map(|x| FlowJobDuration {
started_at: x,
duration_ms: span.duration_ms,
}),
StepFailureKind::Normal,
&same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(),
&worker_dir,
@@ -1867,9 +1870,10 @@ pub async fn process_completed_job(
// `wm_failure` field) so a real runtime failure whose raw
// result happens to contain a `wm_failure` field still goes
// through the standard `WrappedError { error: ... }` wrap path.
let downstream_result: Arc<Box<RawValue>> = if is_pre_shaped_wm_failure_result(result.get())
{
windmill_queue::add_completed_job_pre_shaped_failure(
let (downstream_result, span): (Arc<Box<RawValue>>, _) = if is_pre_shaped_wm_failure_result(
result.get(),
) {
let span = windmill_queue::add_completed_job_pre_shaped_failure(
db,
&job,
mem_peak.to_owned(),
@@ -1880,9 +1884,9 @@ pub async fn process_completed_job(
None,
)
.await?;
result.clone()
(result.clone(), span)
} else {
let wrapped = add_completed_job_error(
let (wrapped, span) = windmill_queue::add_completed_job_error_with_span(
db,
&job,
mem_peak.to_owned(),
@@ -1895,7 +1899,10 @@ pub async fn process_completed_job(
None,
)
.await?;
Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap())
(
Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()),
span,
)
};
#[cfg(all(feature = "enterprise", feature = "private"))]
if job.kind == JobKind::DeploymentCallback {
@@ -1927,12 +1934,19 @@ pub async fn process_completed_job(
false,
canceled_by,
downstream_result,
duration.and_then(|d| {
job.started_at.map(|started_at| FlowJobDuration {
started_at: started_at,
duration_ms: d,
if span.outlasts_run(job.started_at) {
span.started_at.map(|started_at| FlowJobDuration {
started_at,
duration_ms: span.duration_ms,
})
}),
} else {
duration.and_then(|d| {
job.started_at.map(|started_at| FlowJobDuration {
started_at: started_at,
duration_ms: d,
})
})
},
StepFailureKind::Normal,
&same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(),
&worker_dir,
backend/windmill-worker/src/wac_executor.rs
+9 -1
View File
@@ -113,6 +113,11 @@ pub enum WacPark {
/// handler — falls back to `now() - started_at`. Left pointing at the first segment,
/// that fallback reports the whole sleep or approval wait as execution time.
///
/// The first park keeps that first start in `extras.wac_first_started_at`, so the
/// completed row can still say when the workflow began and how long it took overall
/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline
/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way.
///
/// Call it before any write to the parent's `v2_job_status` row in the same
/// transaction: a child's completion locks the queue row and then the status row
/// (`record_child_completion`), and taking them the other way round here can
@@ -156,7 +161,10 @@ pub async fn suspend_wac_parent(
sqlx::query!(
"UPDATE v2_job_queue
SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null
SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,
extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras
ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))
END
WHERE id = $1 AND workspace_id = $2",
job_id,
w_id,
backend/windmill-worker/src/worker.rs
+29 -3
View File
@@ -155,7 +155,7 @@ use crate::{
handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
tally_unfinished_dependency_deploy,
},
worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full},
worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full},
};
#[cfg(feature = "rust")]
@@ -2713,7 +2713,7 @@ pub async fn run_worker(
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
let mut reported_ip = cached_ip();
let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn)
let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn)
.await
.expect("initial ping could be sent");
@@ -3160,6 +3160,31 @@ pub async fn run_worker(
let mut killpill_rx2 = killpill_rx.resubscribe();
// The loop below only sees the killpill once the running job is done, which is too late
// for that job to learn its worker is going away.
{
let mut killpill_rx = killpill_rx.resubscribe();
let conn = conn.clone();
let worker_name = worker_name.clone();
tokio::spawn(async move {
if matches!(
killpill_rx.recv().await,
Err(broadcast::error::RecvError::Closed)
) {
return;
}
for attempt in 1..=3 {
match mark_worker_draining(&conn, &worker_name).await {
Ok(()) => return,
Err(e) => {
tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}");
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
}
});
}
loop {
let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst);
if last_processing_duration_secs > 5 {
@@ -3235,7 +3260,7 @@ pub async fn run_worker(
// which costs at most the last job's id here: no job of this worker is in flight at this
// point in the loop, and the next one refills them.
if ip_just_resolved && conn.as_sql().is_none() {
if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await {
if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await {
tracing::warn!(
worker = %worker_name, hostname = %hostname,
"failed to re-register with the resolved external IP: {e}"
@@ -5105,6 +5130,7 @@ pub async fn handle_queued_job(
// WAC v2 job suspended while waiting for child jobs — don't complete it
return Ok(JobOutcome::Completed);
}
crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak);
process_result(
cjob,
result.map(|x| Arc::new(x)),
backend/windmill-worker/src/worker_flow.rs
+4 -4
View File
@@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal(
)
.await?;
let duration = if success {
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&cflow_job,
true,
@@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal(
false,
)
.await?;
duration
span.duration_ms
} else {
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&cflow_job,
false,
@@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal(
false,
)
.await?;
duration
span.duration_ms
};
flow_job_duration = flow_job
.started_at
backend/windmill-worker/src/worker_utils.rs
+58 -5
View File
@@ -7,9 +7,9 @@ use windmill_common::{
external_ip::UNKNOWN_IP,
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
WORKER_CONFIG, WORKER_GROUP,
insert_ping_query, set_worker_draining_query, update_job_ping_query,
update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping,
PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP,
},
KillpillSender, DB,
};
@@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full(
let wm_memory_usage = get_windmill_memory_usage();
let (vcpus, memory) = if read_cgroups {
(get_vcpus(), get_memory())
let memory = get_memory();
crate::resource_metrics::set_memory_limit(memory);
(get_vcpus(), memory)
} else {
(None, None)
};
let occupancy = occupancy_metrics.update_occupancy_metrics();
crate::resource_metrics::record_worker_resources(
worker_name,
memory_usage,
wm_memory_usage,
Some(&occupancy),
);
let OccupancyResult {
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
} = occupancy_metrics.update_occupancy_metrics();
} = occupancy;
let ping_start = std::time::Instant::now();
if let Err(e) = (|| {
@@ -164,6 +173,7 @@ async fn update_worker_ping_full_inner(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation: None,
native_mode: Some(native_mode),
draining: None,
ping_type: PingType::MainLoop,
},
)
@@ -179,6 +189,7 @@ pub async fn insert_ping(
worker_instance: &str,
worker_name: &str,
ip: Option<&str>,
reset_draining: bool,
db: &Connection,
) -> anyhow::Result<i32> {
let (tags, dw, dws, native_mode) = {
@@ -224,6 +235,7 @@ pub async fn insert_ping(
memory,
job_isolation,
native_mode,
reset_draining.then_some(false),
db,
)
.await;
@@ -256,6 +268,7 @@ pub async fn insert_ping(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation,
native_mode: Some(native_mode),
draining: reset_draining.then_some(false),
ping_type: PingType::Initial,
},
)
@@ -314,6 +327,7 @@ pub async fn update_worker_ping_from_job(
&Ping {
last_job_executed: Some(job_id.clone()),
last_job_workspace_id: Some(w_id.to_string()),
draining: None,
ping_type: PingType::Job,
worker_instance: None,
ip: None,
@@ -342,6 +356,45 @@ pub async fn update_worker_ping_from_job(
Ok(())
}
/// Publishes on the worker's ping row that it has received its killpill, so the job it is
/// still running can find out and exit early.
pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> {
match conn {
Connection::Sql(db) => set_worker_draining_query(worker_name, db).await,
Connection::Http(client) => {
client
.post::<Ping, ()>(
UPDATE_PING_URL,
None,
&Ping {
last_job_executed: None,
last_job_workspace_id: None,
draining: None,
ping_type: PingType::Draining,
worker_instance: None,
ip: None,
tags: None,
dw: None,
dws: None,
version: None,
vcpus: None,
memory: None,
memory_usage: None,
wm_memory_usage: None,
jobs_executed: None,
occupancy_rate: None,
occupancy_rate_15s: None,
occupancy_rate_5m: None,
occupancy_rate_30m: None,
job_isolation: None,
native_mode: None,
},
)
.await
}
}
}
pub async fn ping_job_status(
conn: &Connection,
job_id: &Uuid,
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.827.0";
export const VERSION = "v1.828.0";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
chat-sdk/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-chat",
"version": "1.827.0",
"version": "1.828.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-chat",
"version": "1.827.0",
"version": "1.828.0",
"license": "Apache-2.0",
"devDependencies": {
"@ai-sdk/react": "^4.0.102",
chat-sdk/package.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "windmill-chat",
"description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app",
"version": "1.827.0",
"version": "1.828.0",
"author": "Ruben Fiszel",
"license": "Apache-2.0",
"homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme",
cli/src/core/constants.ts
+1 -1
View File
@@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork";
// (e.g. utils.ts) can read it without importing main.ts and creating a circular
// dependency (main → workspace → utils → main) that triggers a TDZ.
// Re-exported from main.ts for backwards compatibility.
export const VERSION = "1.827.0";
export const VERSION = "1.828.0";
cli/src/guidance/skills.gen.ts
Generated
+76 -2
View File
@@ -843,6 +843,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -1679,6 +1687,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -2622,6 +2638,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -4637,6 +4661,18 @@ def set_progress(value: int, job_id: Optional[str] = None)
# Progress value (0-100) or None if not set
def get_progress(job_id: Optional[str] = None) -> Any
# Whether the worker running the job has received its shutdown signal.
#
# A draining worker waits for its current job to finish before exiting. A long-running
# script can poll this and return early so the worker can be replaced.
#
# Args:
# job_id: Job ID (defaults to current WM_JOB_ID)
#
# Returns:
# True if the worker running the job is draining
def worker_is_draining(job_id: Optional[str] = None) -> bool
# Set the user state of a flow at a given key
def set_flow_user_state(key: str, value: Any) -> None
@@ -7915,10 +7951,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step
A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake://<name>/<table>\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else:
- \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live).
- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store.
- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store.
Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal.
## Ingesting from Python / TypeScript
A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table.
Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema:
\`\`\`python
# pipeline
# on schedule
import wmill
def main():
rows = fetch_charges() # list[dict]
lake = wmill.ducklake()
lake.query("""
CREATE OR REPLACE TABLE stripe_charges AS
SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)
""", rows=rows).fetch()
\`\`\`
\`\`\`ts
// pipeline
// on schedule
import * as wmill from "windmill-client"
export async function main() {
const rows = await fetchCharges()
const lake = wmill.ducklake()
await lake\`CREATE OR REPLACE TABLE stripe_charges AS
SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch()
}
\`\`\`
- Use \`INSERT INTO <table> SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first.
- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/<table>\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms.
- \`wmill.ducklake("<name>")\` targets a non-default catalog; \`wmill.datatable("<name>")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above).
- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`.
## Storage prerequisites
A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build.
@@ -7962,7 +8036,7 @@ The key must be a **string literal** — the graph parser is static and cannot f
## Materialize (the managed output)
> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake://<name>/<table>\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table.
> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake://<name>/<table>\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table.
>
> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt://<warehouse>/<schema>/<name>\`, where \`<warehouse>\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt://<warehouse>/<schema>/<name>\` fires when the ingestion node completes.
docs/pipeline-local-dev.md
+1 -1
View File
@@ -69,7 +69,7 @@ makes the client own the whole cascade so the backend dispatcher never double-fi
- `frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts` — reusable run primitives
(`makeLaunch`, `makeWaitJobTerminal`, `runDownstreamCascade`, `runBoundedCascade`) wrapping
`cascadeOrchestrator` + `graphTraversal`.
- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/pipeline_dev/+page.svelte`.
- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/(styled)/pipeline_dev/+page.svelte`.
**Modified**
- `cli/src/commands/pipeline/pipeline.ts` — `--local` on `show`/`run`; `show` split into graph
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@windmill-labs/components",
"version": "1.827.0",
"version": "1.828.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@windmill-labs/components",
"version": "1.827.0",
"version": "1.828.0",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@windmill-labs/components",
"version": "1.827.0",
"version": "1.828.0",
"scripts": {
"dev": "vite dev",
"dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev",
frontend/src/app.html
+69
View File
@@ -12,6 +12,17 @@
fill: #3b82f6;
}
/* Public app pages render their shell before app.css loads, if at all. */
html[data-wm-public-app] #svelte-global-loader {
display: none;
}
html[data-wm-public-app] body {
margin: 0;
}
html[data-wm-public-app='dark'] body {
background-color: #3b4252;
}
/* Background color for dark mode */
@media (prefers-color-scheme: dark) {
#mainbg {
@@ -123,6 +134,64 @@
}
})()
</script>
<script>
// Public app pages (`/public/<ws>/<secret>`, `/a/<custom path>`): skip the loading
// screen, and start the API calls the page makes first so they overlap the module
// download instead of following it. `publicAppApi.ts` takes these responses by URL,
// so each URL here must be exactly what it requests. Only the cookie credential is
// prefetched: a share link carrying a JWT, or the opaque viewer frame (`wm_embed`),
// sends its own and fetches normally. The bundle of an unsandboxed raw app is
// requested as soon as its secret is known, for the iframe to find in the HTTP cache.
;(function () {
var path = location.pathname
var pub = /^\/public\/([^/]+)\/([^/]+)\/?$/.exec(path)
var custom = /^\/a\/(.+?)\/?$/.exec(path)
if (!pub && !custom && !/^\/public\//.test(path)) return
// Dark as PublicApp decides it, painted before any stylesheet so that a dark app
// does not load over a white page.
var dark = false
try {
var mode = localStorage.getItem('dark-mode')
dark = mode ? mode === 'dark' : matchMedia('(prefers-color-scheme: dark)').matches
} catch (_) {}
document.documentElement.setAttribute('data-wm-public-app', dark ? 'dark' : '')
if (new URLSearchParams(location.search).get('wm_embed') === '1') return
var prefetched = (window.__wmPublicAppPrefetch = {})
function get(url) {
var res = fetch(url)
res.catch(function () {})
prefetched[url] = res
return res
}
function warmBundle(appRes, workspace) {
appRes
.then(function (r) {
return r.ok ? r.clone().json() : null
})
.then(function (app) {
if (!app || !app.raw_app || !app.bundle_secret || (app.policy || {}).sandbox) return
var ws = workspace || app.workspace_id
var v = '/api/w/' + ws + '/apps_u/get_data/v/' + app.bundle_secret
fetch(v + '.css').catch(function () {})
fetch(v + '.js').catch(function () {})
})
.catch(function () {})
}
if (pub) {
var w = '/api/w/' + pub[1]
get(w + '/apps_u/embed_token/' + pub[2])
warmBundle(get(w + '/apps_u/public_app/' + pub[2]), pub[1])
get(w + '/apps_u/guest_entry/' + pub[2])
get(w + '/users/whoami')
} else if (custom && custom[1].indexOf('.') === -1) {
// A `.` marks a trailing JWT segment, which no custom path contains.
get('/api/apps_u/embed_token_by_custom_path/' + custom[1])
warmBundle(get('/api/apps_u/public_app_by_custom_path/' + custom[1]))
get('/api/apps_u/guest_entry_by_custom_path/' + custom[1])
}
get('/api/ee_license')
})()
</script>
%sveltekit.head%
</head>
<body data-sveltekit-preload-code="viewport" class="outline-none focus:outline-none">
frontend/src/lib/apiBaseUrl.svelte.ts
+22
View File
@@ -0,0 +1,22 @@
import { fromStore } from 'svelte/store'
import { base } from '$lib/base'
import { instanceUi } from '$lib/instanceUi'
const ui = fromStore(instanceUi)
/**
* Base of every URL shown for an external client to call (webhooks, HTTP routes, push
* trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the
* origin being browsed. Append `/api/...` to it.
*
* Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links
* to pages): those must stay on the browsing origin, where the session cookie lives.
*
* Reactive: the setting lands after page load, so read it inside a `$derived` or the
* template rather than caching the result.
*/
export function apiBaseUrl(): string {
const setting = ui.current?.api_base_url
const configured = typeof setting === 'string' ? setting.trim() : ''
return configured || `${window.location.origin}${base}`
}
frontend/src/lib/components/AutosaveIndicator.svelte
+7 -1
View File
@@ -142,7 +142,13 @@
let hintTimer: ReturnType<typeof setTimeout> | undefined
const kindLabel = $derived(
itemKind === 'flow' ? 'flow' : itemKind === 'app' || itemKind === 'raw_app' ? 'app' : 'script'
itemKind === 'flow'
? 'flow'
: itemKind === 'app' || itemKind === 'raw_app'
? 'app'
: itemKind === 'data_pipeline'
? 'pipeline'
: 'script'
)
// Each false → true transition fires a hint (others > loaded). Seeded to
frontend/src/lib/components/CliHelpBox.svelte
+2 -1
View File
@@ -2,10 +2,11 @@
import ClipboardPanel from './details/ClipboardPanel.svelte'
import Section from './Section.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
const operatingWorkspace = useOperatingWorkspace()
let url = $derived(`${window.location.protocol}//${window.location.hostname}/`)
let url = $derived(`${apiBaseUrl()}/`)
</script>
<Section label="CLI quick setup">
frontend/src/lib/components/CronBuilder.svelte
+1 -1
View File
@@ -15,7 +15,7 @@
contentClasses="p-4"
>
{#snippet trigger()}
<Button variant="accent" size="xs" nonCaptureEvent={true} startIcon={{ icon: Clock }}>
<Button variant="default" unifiedSize="sm" nonCaptureEvent={true} startIcon={{ icon: Clock }}>
Cron builder
</Button>
{/snippet}
frontend/src/lib/components/DatatablePicker.svelte
+8 -1
View File
@@ -17,6 +17,9 @@
selectInputClass?: string
class?: string
onClear?: () => void
/** Size the input to its content, like Path's folder picker. */
useContentEditable?: boolean
RightIcon?: any
}
let {
@@ -27,7 +30,9 @@
placeholder = undefined,
selectInputClass = '',
class: className = '',
onClear = undefined
onClear = undefined,
useContentEditable = false,
RightIcon = undefined
}: Props = $props()
let datatables = usePromise(() =>
@@ -47,6 +52,8 @@
{placeholder}
inputClass={selectInputClass}
{onClear}
{useContentEditable}
{RightIcon}
/>
{#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })}
<ExploreAssetButton class="mt-1 w-fit" asset={{ kind: 'datatable', path: value }} />
frontend/src/lib/components/DucklakePicker.svelte
+8 -1
View File
@@ -17,6 +17,9 @@
selectInputClass?: string
class?: string
onClear?: () => void
/** Size the input to its content, like Path's folder picker. */
useContentEditable?: boolean
RightIcon?: any
}
let {
@@ -27,7 +30,9 @@
placeholder = undefined,
selectInputClass = '',
class: className = '',
onClear = undefined
onClear = undefined,
useContentEditable = false,
RightIcon = undefined
}: Props = $props()
let ducklakes = usePromise(() =>
@@ -45,6 +50,8 @@
{placeholder}
inputClass={selectInputClass}
{onClear}
{useContentEditable}
{RightIcon}
/>
{#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'ducklake', path: value })}
<ExploreAssetButton class="mt-1 w-fit" asset={{ kind: 'ducklake', path: value }} />
frontend/src/lib/components/Editor.svelte
+3
View File
@@ -1523,6 +1523,9 @@
} catch (e) {
console.log('error initializing services', e)
}
// Unmounted while the services loaded: the container is detached, and
// creating the editor on it throws.
if (destroyed || !divEl?.isConnected) return
// vscode.languages.registerDefinitionProvider('*', {
// provideDefinition(document, position, token) {
frontend/src/lib/components/EditorBar.svelte
+1
View File
@@ -105,6 +105,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
frontend/src/lib/components/FlowStatusViewerInner.svelte
+5 -1
View File
@@ -69,6 +69,7 @@
retainLinkedToolsScope
} from './flows/linkedAgentToolsStore.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { AGENT_STEP_ID, agentStepLabel } from './flows/conversations/agentEditorChat'
const operatingWorkspace = useOperatingWorkspace()
@@ -1238,6 +1239,8 @@
id: string
type: FlowModuleValue['type']
suspend?: boolean
/** Shown in place of the id. */
label?: string
}
function allModulesForTimeline(
@@ -1250,7 +1253,8 @@
({
id: x.id,
type: x.value.type,
suspend: x.suspend != undefined
suspend: x.suspend != undefined,
label: x.id === AGENT_STEP_ID ? agentStepLabel(job?.script_path) : undefined
}) as FlowModuleForTimeline,
{
skipToolNodes: true
frontend/src/lib/components/FlowTimeline.svelte
+2 -2
View File
@@ -179,12 +179,12 @@
/>
</div>
{/if}
{#each flowModules as { id: k, type: typ, suspend: isSuspend } (k)}
{#each flowModules as { id: k, type: typ, suspend: isSuspend, label } (k)}
{@const subItems = items?.[k]?.filter((x) => x.created_at && x.started_at)}
<div class="relative px-3 py-1.5">
<div class="flex items-center justify-between mb-0.5">
<div class="text-xs font-medium"
>{k.startsWith('subflow:') ? k.substring(8) : k}
>{label ?? (k.startsWith('subflow:') ? k.substring(8) : k)}
{#if localModuleStates[k]?.selectedForloop && (typ == 'forloopflow' || typ == 'whileloopflow')}
<span class="text-xs font-mono font-medium inline-flex items-center -my-2">
<button onclick={(e) => e.stopPropagation()}>
frontend/src/lib/components/JobArgs.svelte
+2 -1
View File
@@ -15,6 +15,7 @@
import { isWindmillTooBigObject } from './job_args'
import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile'
import { appendViewToken } from '$lib/viewToken'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
interface Props {
id?: string | undefined
@@ -246,7 +247,7 @@ ${Object.entries(displayArgs)
If using the wmill client in your code, set the following env variables:
<pre
><code
>BASE_URL="{window.location.origin}"
>BASE_URL="{apiBaseUrl()}"
WM_TOKEN="{'<TOKEN>'}"</code
></pre
>
frontend/src/lib/components/NavBreadcrumb.svelte
+9
View File
@@ -396,6 +396,15 @@ not this component) — so the lead can run the whole line.
onPick={() => {}}
/>
{:else if section}
{#if section.parent}
<!-- Gives width back with the workspace part, before the section's own name does. -->
<span
class={twMerge(SEGMENT, LEAD, 'shrink-[999] hover:bg-transparent hover:text-secondary')}
>
<span class="truncate">{section.parent}</span>
</span>
{@render slash()}
{/if}
<!-- A section's own widget lays out as a row: its title, and whatever belongs with it. -->
<span
class={twMerge(
frontend/src/lib/components/PageHeaderBar.svelte
+14 -3
View File
@@ -32,6 +32,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
const item = $derived(content?.item)
const section = $derived(content?.section)
const hasSummary = $derived(!!item && !!(item.summaryContent || item.summary))
// Below this the trail holds a tighter cap on the path it draws. The names in it truncate on
// their own at any width (see NavBreadcrumb), so this is not a breakpoint the look changes at —
@@ -135,8 +136,18 @@ The row's height matches the sidebar's own header row, so the two read as one ba
where the user is.
An embed (`navHidden`) keeps the page's own name but drops the workspace part around it:
the trail would offer to navigate the host's workspace, while the name is what says which
page the controls beside it belong to. -->
<div class="flex shrink-[0.3] min-w-[5rem]">
page the controls beside it belong to.
The 0.3 only holds against a summary or an actions box that shrinks too: alone, a factor
sum below 1 makes flex hand out only that share of the overflow, and the bar overflows
instead. -->
<div
class={twMerge(
'flex shrink-[0.3] min-w-[5rem]',
!hasSummary &&
(actions.length === 0 || (phone && !content?.actionsFlexible && !content?.actionsFill)) &&
'shrink'
)}
>
<!-- Bridged like the actions below: what a page hangs off its own name renders here, out of
the tree that named it, and the rename editor in there asks that tree who the acting user
is before it offers to rename anything. -->
@@ -161,7 +172,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
</svelte:boundary>
</div>
{#if item && (item.summaryContent || item.summary)}
{#if item && hasSummary}
<!-- No kind icon: the page below is the item, and saying "this is a flow" above a flow
editor tells the reader what they can already see. -->
<div class={twMerge('flex items-center gap-1 min-w-0', phone && 'shrink')}>
frontend/src/lib/components/ScriptBuilder.svelte
+20 -2
View File
@@ -596,6 +596,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
@@ -1543,10 +1544,12 @@
(lang == script.language &&
template != 'bunnative' &&
template != 'docker' &&
template != 'claudesandbox') ||
template != 'claudesandbox' &&
template != 'pisandbox') ||
(template == 'bunnative' && lang == 'bunnative') ||
(template == 'docker' && lang == 'docker') ||
(template == 'claudesandbox' && lang == 'bun')}
((template == 'claudesandbox' || template == 'pisandbox') &&
lang == 'bun')}
<Popover
disablePopup={!enterpriseLangs.includes(lang) || !!$enterpriseLicense}
>
@@ -1597,6 +1600,21 @@
>
Claude Sandbox
</Button>
<Button
unifiedSize="xs"
variant="default"
startIcon={{
icon: LanguageIcon,
props: { lang: 'pisandbox', width: 16, height: 16 }
} as ButtonType.Icon}
onclick={() => {
template = 'pisandbox'
script.language = 'bun'
initContent('bun', script.kind, template)
}}
>
Pi Sandbox
</Button>
<span title="Workflow-as-Code">
<Button
size="xs2"
frontend/src/lib/components/ScriptEditor.svelte
+35 -37
View File
@@ -150,6 +150,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
@@ -2100,43 +2101,15 @@
</div>
{/if}
{#if customUi?.previewPanel?.hideArgs}
<!-- Compact preview layout used by the pipeline editor:
no args column (the script is known to take no
inputs), LogPanel takes the full width, with a
small Test/Cancel button at the top-left of the
preview band. The earlier `-translate-y-1/2`
"float onto the editor" version was clipped in
Firefox (transform + overflow: visible interaction
on the parent splitpane Pane), so the button is
now positioned inside the panel — `top-1` keeps
it visually pinned to the top edge without
relying on cross-browser overflow behaviour. -->
<div class="relative h-full pt-9 flex flex-col">
{#if testJob?.id && testJob.type === 'CompletedJob' && opWs}
<!-- Right-side affordances when we're displaying a *completed*
job (either the user just ran a test, or the on-mount
last-run loader populated the panel). The job-id link
opens the full run page in a new tab; the dispatch
button mounts a popover that shows what downstream
jobs this run triggered (rendered only when the run
actually dispatched anything). -->
<div class="absolute top-1 right-2 z-10 flex items-center gap-2">
<a
class="text-3xs text-blue-600 hover:underline font-mono"
href={`${base}/run/${testJob.id}?workspace=${opWs}`}
target="_blank"
rel="noopener noreferrer"
title="Open this run"
>
{testJob.id.slice(0, 8)}… ↗
</a>
<DispatchEventsButton
workspace={testJob.workspace_id ?? opWs}
jobId={testJob.id}
/>
</div>
{/if}
<div class="absolute top-1 left-2 z-10">
<!-- Compact preview layout used by the pipeline editor: no args
column (most pipeline scripts take no inputs) and a full-width
LogPanel. The controls sit in normal flow rather than floating
onto the editor band, which Firefox clipped (transform +
overflow: visible on the parent splitpane Pane). -->
<div class="h-full flex flex-col">
<!-- Test sits at the top right, the last run's link under it. -->
<div class="shrink-0 flex flex-col items-end gap-1 px-2 pt-2 pb-1">
<div>
{#if testIsLoading}
{@render cancelTestButton('sm', 'shadow-md')}
{:else if (customUi?.previewPanel?.downstreamSubscribers ?? 0) > 0 || customUi?.previewPanel?.onBoundedRun}
@@ -2270,6 +2243,31 @@
{@render runTestButton('sm', 'shadow-md')}
{/if}
</div>
{#if testJob?.id && testJob.type === 'CompletedJob' && opWs}
<!-- Right-side affordances when we're displaying a *completed*
job (either the user just ran a test, or the on-mount
last-run loader populated the panel). The job-id link
opens the full run page in a new tab; the dispatch
button mounts a popover that shows what downstream
jobs this run triggered (rendered only when the run
actually dispatched anything). -->
<div class="flex items-center gap-2">
<a
class="text-3xs text-blue-600 hover:underline font-mono"
href={`${base}/run/${testJob.id}?workspace=${opWs}`}
target="_blank"
rel="noopener noreferrer"
title="Open this run"
>
{testJob.id.slice(0, 8)}… ↗
</a>
<DispatchEventsButton
workspace={testJob.workspace_id ?? opWs}
jobId={testJob.id}
/>
</div>
{/if}
</div>
{#if customUi?.previewPanel?.argsAboveLogs && schema?.properties && Object.keys(schema.properties).length > 0}
<div
class="shrink-0 overflow-auto max-h-[45%] border-b border-gray-200 dark:border-gray-700 px-4 py-3"
frontend/src/lib/components/SummaryPathDisplay.svelte
+22 -1
View File
@@ -31,6 +31,13 @@
/** What is being renamed. An agent is a resource, so the path field is scoped to one and
* the write goes through the agent branch of the rename manager. */
kind?: 'flow' | 'script' | 'agent'
/** Saves instead of the API, for an item that exists only locally (a draft).
* Returns an error message to keep the popover open. */
saveOverride?: (next: {
path: string
summary: string
labels: string[] | undefined
}) => string | undefined
/** Header variant: no path line (the host shows it) and a lighter summary. */
compact?: boolean
/** Rendered inside the trigger in place of the summary, for a host that hangs a second copy
@@ -51,6 +58,7 @@
editable = false,
onSaved,
kind = 'flow',
saveOverride = undefined,
compact = false,
label,
triggerClass = 'block min-w-0 max-w-full px-1 py-0.5 rounded text-left cursor-pointer hover:bg-surface-hover transition-colors',
@@ -67,7 +75,9 @@
let popoverOpen = $state(false)
// Resolved before the popover opens, not on opening it: the path field only exists for an owner,
// and `openFocus` looks for it in the same flush the popover renders in.
const own = $derived(isOwner(path ?? '', actingUser, $operatingWorkspace))
const own = $derived(
saveOverride !== undefined || isOwner(path ?? '', actingUser, $operatingWorkspace)
)
let onBehalfOfEmail = $state<string | undefined>(undefined)
let summaryInput: ReturnType<typeof TextInput> | undefined = $state()
let labelsDirty = $state(false)
@@ -101,6 +111,17 @@
async function save(close: () => void) {
const initialPath = path ?? ''
const newPath = own ? editPath : initialPath
if (saveOverride) {
const error = saveOverride({ path: newPath, summary: editSummary, labels })
if (error) {
sendUserToast(error, true)
return
}
labelsDirty = false
close()
onSaved?.(newPath)
return
}
try {
await updateItemPathAndSummary({
frontend/src/lib/components/SuperadminSettingsInner.svelte
+1 -1
View File
@@ -372,7 +372,7 @@
</div>
{/if}
<Tabs bind:selected={usersSubTab} class="mb-4">
<Tabs bind:selected={usersSubTab} class="mb-4" wrapperClass="shrink-0">
<Tab value="users" label="Users" />
{#if extJwtTokens.length > 0}
<Tab value="ext_jwt" label="External JWTs" />
frontend/src/lib/components/WorkerGroup.svelte
+2 -1
View File
@@ -55,6 +55,7 @@
import TagList from './TagList.svelte'
import DedicatedWorkersSelector from './DedicatedWorkersSelector.svelte'
import { computeHashedTag } from './dedicated_worker'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
import ObjectStoreConfigSettings, {
type ObjectStoreConfig
} from './ObjectStoreConfigSettings.svelte'
@@ -1238,7 +1239,7 @@
<p class="text-xs text-tertiary">Requires a superadmin token.</p>
<pre
class="mt-1 p-2 bg-surface-secondary rounded text-xs overflow-x-auto whitespace-pre-wrap break-all"
>curl -X POST '{window.location.origin}/api/configs/update/worker__{name}' \
>curl -X POST '{apiBaseUrl()}/api/configs/update/worker__{name}' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer &lt;token&gt;' \
-d '{JSON.stringify(cleanWorkerGroupConfig(nconfig ?? {}), null, 2)}'</pre
frontend/src/lib/components/apps/editor/InWorkspaceAppViewer.svelte
+2 -2
View File
@@ -19,7 +19,7 @@
import type { Item } from '$lib/utils'
import { Button } from '$lib/components/common'
import { AppService, OpenAPI } from '$lib/gen'
import type { UserExt } from '$lib/stores'
import { userStore, type UserExt } from '$lib/stores'
import { canWrite } from '$lib/utils'
import { getUserExt } from '$lib/user'
import { ExternalLink, Pen } from 'lucide-svelte'
@@ -251,7 +251,7 @@
<PublicApp
{app}
{workspace}
user={appUser}
user={appUser ?? $userStore}
{notExists}
{noPermission}
jwtError={false}
frontend/src/lib/components/apps/editor/PublicApp.svelte
+59 -149
View File
@@ -1,28 +1,15 @@
<script lang="ts">
import { applyDarkModeVariant } from '$lib/darkModeVariant'
import { enterpriseLicense, userStore, type UserExt } from '$lib/stores'
import { base } from '$app/paths'
import { page } from '$app/state'
import Login from '$lib/components/Login.svelte'
import type { UserExt } from '$lib/stores'
import { isCloudHosted } from '$lib/cloud'
import Alert from '$lib/components/common/alert/Alert.svelte'
import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte'
import WindmillIcon from '$lib/components/icons/WindmillIcon.svelte'
import { getContext, setContext } from 'svelte'
import {
EMBED_NAV_CONTEXT_KEY,
IS_APP_PUBLIC_CONTEXT_KEY,
type EditorBreakpoint,
type EmbedNav
} from '../types'
import { UserService, type AppWithLastVersion, type GlobalWhoamiResponse } from '$lib/gen'
import { urlParamsToObject } from '$lib/utils'
import { goto } from '$app/navigation'
import { loadAppPreview } from './loadAppPreview'
import { setContext } from 'svelte'
import { IS_APP_PUBLIC_CONTEXT_KEY } from '../types'
import type { AppWithLastVersion } from '$lib/gen/types.gen'
import RawAppPreview from '$lib/components/raw_apps/RawAppPreview.svelte'
import type { Runnable } from '$lib/components/raw_apps/rawAppPolicy'
import { twMerge } from 'tailwind-merge'
import { writable } from 'svelte/store'
import { loadAppCss } from './publicAppApi'
let {
notExists,
@@ -33,6 +20,7 @@
app,
workspace,
user = undefined,
license = undefined,
inWorkspace = false,
hideRefreshBar = false,
syncHashToUrl = true
@@ -46,11 +34,13 @@
onLoginSuccess: () => void
app: (AppWithLastVersion & { value: any; workspace_id?: string }) | undefined
workspace: string | undefined
/** The viewer's membership in `workspace`, for the app's `ctx`. Without it `ctx` falls
* back to `userStore`, which describes the workspace the page is navigated to — the
* same one on every standalone viewer route, but not inside an AI session's preview
* tab, where the app belongs to the session's workspace. */
/** The viewer's membership in `workspace`, for the app's `ctx`. Callers pass it
* explicitly — `$userStore` describes the workspace the page is navigated to, which
* is not the app's inside an AI session's preview tab. */
user?: UserExt | undefined
/** The instance license, which fades the "Powered by" badge and lets a low-code app's
* custom CSS apply. */
license?: string | undefined
/**
* In-workspace rendering (`/apps/get`, `/app_embed`): keep exact parity
* with the pre-sandbox member viewer — no "Powered by Windmill" badge, no
@@ -65,23 +55,11 @@
// Use workspace from props or from app.workspace_id (for custom path responses)
let effectiveWorkspace = $derived(workspace ?? app?.workspace_id)
// `ctx.username` / `ctx.groups` are per-workspace, and they sit next to `ctx.workspace`
// in the same object, so they have to come from the same workspace it names.
let ctxUser = $derived(user ?? $userStore)
// On the public surfaces (untrusted distribution) runnable-authored html/svg needs
// the viewer's approval before it renders, unless the app sandbox isolates it. The
// in-workspace viewer renders it verbatim. See getAppMarkupTrust.
setContext(IS_APP_PUBLIC_CONTEXT_KEY, !inWorkspace)
// WIN-2006: inside the opaque viewer iframe, navigations to other routes
// (navbar "app" items) must happen on the TOP page — the iframe is cookieless,
// so navigating it would just show a login screen. PublicAppFrame provides the
// relay; outside the opaque viewer this is undefined and goto works directly.
const embedNav = getContext<EmbedNav | undefined>(EMBED_NAV_CONTEXT_KEY)
const breakpoint = writable<EditorBreakpoint>('lg')
const darkMode =
window.localStorage.getItem('dark-mode') ??
(window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light')
@@ -93,84 +71,46 @@
}
// This route bypasses the (root) layout, so restore the variant class too.
applyDarkModeVariant()
let globalUser = $state<GlobalWhoamiResponse | undefined>(undefined)
async function loadGlobalUser() {
try {
globalUser = await UserService.globalWhoami()
} catch (error) {
console.error(error)
}
// const user = await fetch('/api/global/user')
// console.log(user)
}
// Only the no-access page reads it, to tell a signed-in non-member which
// workspace the app belongs to.
$effect(() => {
if (noPermission && !guestAppPath && !$userStore && !globalUser) {
loadGlobalUser()
}
})
</script>
<!-- Only the raw-app branch is imported statically: everything else waits for app.css and
its own chunk, see publicAppApi.ts. -->
{#if !inWorkspace}
<div
class="z-50 text-xs fixed bottom-1 right-2 {$enterpriseLicense && !isCloudHosted()
? 'transition-opacity delay-1000 duration-1000 opacity-20 hover:delay-0 hover:opacity-100'
: ''}"
>
<a href="https://windmill.dev" class="whitespace-nowrap text-primary inline-flex items-center"
>Powered by &nbsp;<WindmillIcon />&nbsp;Windmill</a
{#await loadAppCss() then}
<div
class="z-50 text-xs fixed bottom-1 right-2 {license && !isCloudHosted()
? 'transition-opacity delay-1000 duration-1000 opacity-20 hover:delay-0 hover:opacity-100'
: ''}"
>
</div>
<a href="https://windmill.dev" class="whitespace-nowrap text-primary inline-flex items-center"
>Powered by &nbsp;<WindmillIcon />&nbsp;Windmill</a
>
</div>
{/await}
{/if}
{#if notExists}
<div class="px-4 mt-20"
><Alert type="error" title="Not found"
>There was an error loading the app, is the url correct? <a href={base}>Go to Windmill</a>
</Alert></div
>
{:else if noPermission}
{#if guestAppPath && !$userStore}
<div class="px-4 mt-20 w-full text-center font-bold text-xl"> Sign in to open this app </div>
<div class="text-center mt-8 text-sm text-primary">
You do not need a Windmill account. Signing in lets you open this app and nothing else.
</div>
{:else}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
This app requires read access
</div>
<div class="text-center mt-8 text-sm text-primary">
{#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace}
You are logged in but are not a member of the workspace <span class="text-xl font-bold"
>{effectiveWorkspace}</span
> this app is part of
{:else}You must be logged in and have read access to this app{/if}</div
>
{/if}
<div class="px-2 mx-auto mt-20 max-w-xl w-full">
{#if !jwtError}
<Login
{onLoginSuccess}
popup
guestApp={guestAppPath}
rd={page.url.pathname + page.url.search + page.url.hash}
/>
{/if}
</div>
{#if notExists || noPermission}
{#await Promise.all([loadAppCss(), import('./PublicAppAccess.svelte')]) then [, { default: PublicAppAccess }]}
<PublicAppAccess
{notExists}
{jwtError}
{guestAppPath}
{user}
workspace={effectiveWorkspace}
{onLoginSuccess}
/>
{/await}
{:else if app}
{#key app}
{#if app.raw_app && effectiveWorkspace}
<!-- The bundle's iframe is `h-full`, so every host has to give this box a height to
<!-- The bundle's iframe is full-height, so every host has to give this box a height to
resolve against: in the workspace the layout hands one down, and the share, embed
and custom-path routes floor themselves at the viewport. An iframe with no height
above it falls back to 150px. -->
<div class="h-full w-full">
above it falls back to 150px. Styled inline: app.css may not be loaded here. -->
<div style="height: 100%; width: 100%">
<RawAppPreview
workspace={effectiveWorkspace}
user={ctxUser}
{user}
secret={app.bundle_secret}
path={app.path}
runnables={(app.value?.runnables ?? {}) as Record<string, Runnable>}
@@ -178,58 +118,28 @@
/>
</div>
{:else if app.raw_app && !effectiveWorkspace}
<div class="px-4 mt-20">
<Alert type="error" title="Configuration error">
Unable to load raw app: workspace information is missing.
</Alert>
</div>
{#await Promise.all([loadAppCss(), import('$lib/components/common/alert/Alert.svelte')]) then [, { default: Alert }]}
<div class="px-4 mt-20">
<Alert type="error" title="Configuration error">
Unable to load raw app: workspace information is missing.
</Alert>
</div>
{/await}
{:else}
<div
class={twMerge(
// `flex-col` matches the pre-sandbox in-workspace viewer exactly;
// the public viewer always used a plain `flex` wrapper. `min-h-full`, not
// `h-full`: the box has to reach the bottom of what the host gives it so the
// app's own background covers the window, and still grow with a grid taller
// than that. A viewport floor instead would overhang the page box by the
// height of the page header band.
inWorkspace ? 'min-h-full w-full flex flex-col' : 'min-h-full w-full flex',
app?.value?.['css']?.['app']?.['viewer']?.class,
'wm-app-viewer'
)}
style={app?.value?.['css']?.['app']?.['viewer']?.style}
>
{#await loadAppPreview()}
<Skeleton layout={[[4], 0.5, [50]]} />
{:then Module}
<Module.default
noBackend={false}
{hideRefreshBar}
context={{
email: ctxUser?.email,
name: ctxUser?.name,
groups: ctxUser?.groups,
username: ctxUser?.username,
query: urlParamsToObject(page.url.searchParams, { stripReserved: true }),
hash: page.url.hash.substring(1)
}}
workspace={effectiveWorkspace}
summary={app.summary}
app={app.value}
appPath={app.path}
{breakpoint}
policy={app.policy}
isEditor={false}
replaceStateFn={(path) => goto(path)}
gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))}
/>
{:catch}
<div class="px-4 mt-20 w-full">
<Alert type="error" title="Could not load the app">Reload the page to try again.</Alert>
</div>
{/await}
</div>
{#await Promise.all([loadAppCss(), import('./PublicLowCodeApp.svelte')]) then [, { default: PublicLowCodeApp }]}
<PublicLowCodeApp
{app}
workspace={effectiveWorkspace}
{user}
{license}
{inWorkspace}
{hideRefreshBar}
/>
{/await}
{/if}
{/key}
{:else}
<Skeleton layout={[[4], 0.5, [50]]} />
{#await loadAppCss() then}
<Skeleton layout={[[4], 0.5, [50]]} />
{/await}
{/if}
frontend/src/lib/components/apps/editor/PublicAppAccess.svelte
+81
View File
@@ -0,0 +1,81 @@
<script lang="ts">
import { base } from '$app/paths'
import { page } from '$app/state'
import Login from '$lib/components/Login.svelte'
import Alert from '$lib/components/common/alert/Alert.svelte'
import { UserService, type GlobalWhoamiResponse } from '$lib/gen'
import type { UserExt } from '$lib/stores'
import { dropPrefetched } from './publicAppApi'
let {
notExists,
jwtError,
guestAppPath,
user,
workspace,
onLoginSuccess
}: {
notExists: boolean
jwtError: boolean
guestAppPath: string | undefined
user: UserExt | undefined
workspace: string | undefined
onLoginSuccess: () => void
} = $props()
let globalUser = $state<GlobalWhoamiResponse | undefined>(undefined)
async function loadGlobalUser() {
try {
globalUser = await UserService.globalWhoami()
} catch (error) {
console.error(error)
}
}
// Only the no-access page reads it, to tell a signed-in non-member which
// workspace the app belongs to.
$effect(() => {
if (!notExists && !guestAppPath && !user && !globalUser) {
loadGlobalUser()
}
})
</script>
{#if notExists}
<div class="px-4 mt-20"
><Alert type="error" title="Not found"
>There was an error loading the app, is the url correct? <a href={base}>Go to Windmill</a>
</Alert></div
>
{:else}
{#if guestAppPath && !user}
<div class="px-4 mt-20 w-full text-center font-bold text-xl"> Sign in to open this app </div>
<div class="text-center mt-8 text-sm text-primary">
You do not need a Windmill account. Signing in lets you open this app and nothing else.
</div>
{:else}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
This app requires read access
</div>
<div class="text-center mt-8 text-sm text-primary">
{#if user}You are logged in but have no read access to this app{:else if globalUser && workspace}
You are logged in but are not a member of the workspace <span class="text-xl font-bold"
>{workspace}</span
> this app is part of
{:else}You must be logged in and have read access to this app{/if}</div
>
{/if}
<div class="px-2 mx-auto mt-20 max-w-xl w-full">
{#if !jwtError}
<Login
onLoginSuccess={() => {
dropPrefetched()
onLoginSuccess()
}}
popup
guestApp={guestAppPath}
rd={page.url.pathname + page.url.search + page.url.hash}
/>
{/if}
</div>
{/if}
frontend/src/lib/components/apps/editor/PublicAppFrame.svelte
+148 -104
View File
@@ -23,24 +23,44 @@
* bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`.
*/
import { BROWSER } from 'esm-env'
import { OpenAPI, UserService } from '$lib/gen'
import { OpenAPI } from '$lib/gen/core/OpenAPI'
import { page } from '$app/state'
import { onDestroy, onMount, setContext, type Snippet } from 'svelte'
import Alert from '$lib/components/common/alert/Alert.svelte'
import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte'
import { base } from '$app/paths'
import { goto } from '$app/navigation'
import Login from '$lib/components/Login.svelte'
import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils'
import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils/queryParams'
import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types'
import { loadAppPreview } from './loadAppPreview'
import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte'
import { dropPrefetched, loadAppCss } from './publicAppApi'
import {
hasStoredSdkConsent,
sdkConsentCovers,
storeSdkConsent
} from '$lib/components/raw_apps/sdkScopes'
// Everything but a rendered app (sign-in, errors, the SDK consent prompt) loads on
// demand: a raw app that renders right away needs none of it. See publicAppApi.ts.
let frameUi:
| Promise<{
Alert: typeof import('$lib/components/common/alert/Alert.svelte').default
Login: typeof import('$lib/components/Login.svelte').default
RawAppSdkConsent: typeof import('$lib/components/raw_apps/RawAppSdkConsent.svelte').default
}>
| undefined
function loadFrameUi() {
return (frameUi ??= Promise.all([
loadAppCss(),
import('$lib/components/common/alert/Alert.svelte'),
import('$lib/components/Login.svelte'),
import('$lib/components/raw_apps/RawAppSdkConsent.svelte')
]).then(([, alert, login, consent]) => ({
Alert: alert.default,
Login: login.default,
RawAppSdkConsent: consent.default
})))
}
type EmbedToken = {
token?: string | null
raw_app?: boolean
@@ -235,7 +255,8 @@
if (offerSignIn && accountSession === 'unknown') {
// Workspace-less, so it answers for an account and never for a guest
// (pinned to its workspace) or for nobody.
UserService.getCurrentEmail()
import('$lib/gen')
.then(({ UserService }) => UserService.getCurrentEmail())
.then(() => (accountSession = 'held'))
.catch(() => (accountSession = 'none'))
}
@@ -249,8 +270,12 @@
let staleGuestLogoutFailed = $state(false)
$effect(() => {
if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) {
UserService.logout()
.then(() => (staleGuestCleared = true))
import('$lib/gen')
.then(({ UserService }) => UserService.logout())
.then(() => {
dropPrefetched()
staleGuestCleared = true
})
.catch(() => (staleGuestLogoutFailed = true))
}
})
@@ -427,7 +452,10 @@
// Render the app directly on this origin: same-origin when unsandboxed
// (the default), or a single opaque bundle iframe when it's a sandboxed
// raw app. A low-code app's runtime downloads alongside the app payload.
if (!isRaw) loadAppPreview().catch(() => {})
if (!isRaw) {
loadAppCss().catch(() => {})
loadAppPreview().catch(() => {})
}
onViewerReady?.(undefined, requestTokenRefresh)
} else {
// Sandboxed low-code: hand the scoped token to the opaque viewer iframe.
@@ -537,6 +565,7 @@
onMount(() => {
if (isViewer) {
// Only a sandboxed low-code app is ever framed as a viewer.
loadAppCss().catch(() => {})
loadAppPreview().catch(() => {})
window.addEventListener('message', handleViewerMessage)
installHashRelay()
@@ -569,105 +598,120 @@
{#if viewerReady}
{@render viewer()}
{:else if viewerOrphaned}
<div class="px-4 mt-20 max-w-xl mx-auto">
<Alert type="info" title="Open this app from Windmill">
This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your own
page, use the app's public URL without the <code>wm_embed</code> parameter.
</Alert>
</div>
{#await loadFrameUi() then { Alert }}
<div class="px-4 mt-20 max-w-xl mx-auto">
<Alert type="info" title="Open this app from Windmill">
This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your
own page, use the app's public URL without the <code>wm_embed</code> parameter.
</Alert>
</div>
{/await}
{:else}
<Skeleton layout={[[4], 0.5, [50]]} />
{#await loadAppCss() then}
<Skeleton layout={[[4], 0.5, [50]]} />
{/await}
{/if}
{:else if status === 'loading'}
<Skeleton layout={[[4], 0.5, [50]]} />
{:else if status === 'notExists' && !offerSignIn}
<div class="px-4 mt-20">
<Alert type="error" title="Not found">
There was an error loading the app, is the url correct?
<a href={base}>Go to Windmill</a>
</Alert>
</div>
{:else if status === 'sdkPrompt'}
<!-- Raw-app frontend SDK: ask before the app's code runs, so the viewer sees
what it will be able to do with their identity. -->
<RawAppSdkConsent
scopes={sdkScopes ?? []}
onContinue={onSdkConsentContinue}
onDecline={onSdkConsentDecline}
/>
{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))}
<Skeleton layout={[[4], 0.5, [50]]} />
{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)}
<div class="px-4 mt-20">
<Alert type="error" title="Could not check access">
The app could not be reached to find out who may open it. Reload to try again.
</Alert>
</div>
{:else if offerSignIn}
<!-- Login happens here, on the embedder (main) window, so the session cookie
is set on the main origin only and never reaches the opaque iframe. -->
{#if signInDidNotHelp}
<!-- Offering the same sign-in again would loop: they are signed in, and this
app still will not open for them. Say why and stop. -->
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
You are signed in, but this app is not open to you
</div>
<div class="text-center mt-8 text-sm text-primary">
It is open to the people it was shared with{guestAppPath
? ', and to guests who have no Windmill account'
: ''}. Ask the person who shared it to give your account access.
</div>
{#await loadAppCss() then}
<Skeleton layout={[[4], 0.5, [50]]} />
{/await}
{:else if status === 'ready'}
{#if unsandboxed}
<!-- Same-origin (full session): the app was not opted into sandbox isolation
(the default). Rendered directly here; RawAppPreview reads
IS_APP_UNSANDBOXED to drop the bundle's opaque sandbox. -->
{@render viewer()}
{:else if isRaw}
<!-- Variant A: sandboxed raw app rendered directly on the real origin. The
untrusted author bundle stays isolated in its own opaque iframe (inside
RawAppPreview); no opaque viewer and no embed token are needed. -->
{@render viewer()}
{:else}
{#if guestAppPath}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">Sign in to open this app</div>
<div class="text-center mt-8 text-sm text-primary">
You do not need a Windmill account. Signing in lets you open this app and nothing else.
</div>
{:else}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
This app requires read access
</div>
{/if}
{#if signInError}
<div class="px-2 mx-auto mt-8 max-w-xl w-full">
<Alert type="error" title="Could not sign you in">{signInError}</Alert>
</div>
{/if}
<div class="px-2 mx-auto mt-20 max-w-xl w-full">
<Login
onLoginSuccess={() => {
signInError = undefined
accountSession = 'unknown'
initEmbedder()
}}
onLoginError={(message) => (signInError = message)}
popup
guestApp={guestAppPath}
rd={page.url.pathname + page.url.search + page.url.hash}
/>
</div>
<!-- referrerpolicy: the embedder page URL can carry a viewer credential (the
JWT path segment of share links); without this, the same-origin iframe
navigation would expose it to app-authored code via document.referrer.
Styled inline: it can render before app.css has loaded. -->
<iframe
bind:this={iframeEl}
src={buildViewerUrl()}
title="App"
style="display: block; width: 100%; height: 100%; border: 0"
sandbox="allow-scripts allow-forms allow-popups allow-popups-to-escape-sandbox allow-downloads allow-modals allow-top-navigation"
allow="clipboard-read; clipboard-write; fullscreen"
referrerpolicy="no-referrer"
></iframe>
{/if}
{:else if unsandboxed}
<!-- Same-origin (full session): the app was not opted into sandbox isolation
(the default). Rendered directly here; RawAppPreview reads
IS_APP_UNSANDBOXED to drop the bundle's opaque sandbox. -->
{@render viewer()}
{:else if isRaw}
<!-- Variant A: sandboxed raw app rendered directly on the real origin. The
untrusted author bundle stays isolated in its own opaque iframe (inside
RawAppPreview); no opaque viewer and no embed token are needed. -->
{@render viewer()}
{:else}
<!-- referrerpolicy: the embedder page URL can carry a viewer credential (the
JWT path segment of share links); without this, the same-origin iframe
navigation would expose it to app-authored code via document.referrer. -->
<iframe
bind:this={iframeEl}
src={buildViewerUrl()}
title="App"
class="w-full h-full border-0 block"
sandbox="allow-scripts allow-forms allow-popups allow-popups-to-escape-sandbox allow-downloads allow-modals allow-top-navigation"
allow="clipboard-read; clipboard-write; fullscreen"
referrerpolicy="no-referrer"
></iframe>
{#await loadFrameUi() then { Alert, Login, RawAppSdkConsent }}
{#if status === 'notExists' && !offerSignIn}
<div class="px-4 mt-20">
<Alert type="error" title="Not found">
There was an error loading the app, is the url correct?
<a href={base}>Go to Windmill</a>
</Alert>
</div>
{:else if status === 'sdkPrompt'}
<!-- Raw-app frontend SDK: ask before the app's code runs, so the viewer sees
what it will be able to do with their identity. -->
<RawAppSdkConsent
scopes={sdkScopes ?? []}
onContinue={onSdkConsentContinue}
onDecline={onSdkConsentDecline}
/>
{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))}
<Skeleton layout={[[4], 0.5, [50]]} />
{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)}
<div class="px-4 mt-20">
<Alert type="error" title="Could not check access">
The app could not be reached to find out who may open it. Reload to try again.
</Alert>
</div>
{:else if offerSignIn}
<!-- Login happens here, on the embedder (main) window, so the session cookie
is set on the main origin only and never reaches the opaque iframe. -->
{#if signInDidNotHelp}
<!-- Offering the same sign-in again would loop: they are signed in, and this
app still will not open for them. Say why and stop. -->
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
You are signed in, but this app is not open to you
</div>
<div class="text-center mt-8 text-sm text-primary">
It is open to the people it was shared with{guestAppPath
? ', and to guests who have no Windmill account'
: ''}. Ask the person who shared it to give your account access.
</div>
{:else}
{#if guestAppPath}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">Sign in to open this app</div
>
<div class="text-center mt-8 text-sm text-primary">
You do not need a Windmill account. Signing in lets you open this app and nothing else.
</div>
{:else}
<div class="px-4 mt-20 w-full text-center font-bold text-xl">
This app requires read access
</div>
{/if}
{#if signInError}
<div class="px-2 mx-auto mt-8 max-w-xl w-full">
<Alert type="error" title="Could not sign you in">{signInError}</Alert>
</div>
{/if}
<div class="px-2 mx-auto mt-20 max-w-xl w-full">
<Login
onLoginSuccess={() => {
dropPrefetched()
signInError = undefined
accountSession = 'unknown'
initEmbedder()
}}
onLoginError={(message) => (signInError = message)}
popup
guestApp={guestAppPath}
rd={page.url.pathname + page.url.search + page.url.hash}
/>
</div>
{/if}
{/if}
{/await}
{/if}
frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte
+91
View File
@@ -0,0 +1,91 @@
<script lang="ts">
import { goto } from '$app/navigation'
import { page } from '$app/state'
import Alert from '$lib/components/common/alert/Alert.svelte'
import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte'
import type { AppWithLastVersion } from '$lib/gen'
import { enterpriseLicense, userStore, type UserExt } from '$lib/stores'
import { urlParamsToObject } from '$lib/utils'
import { getContext } from 'svelte'
import { writable } from 'svelte/store'
import { twMerge } from 'tailwind-merge'
import { EMBED_NAV_CONTEXT_KEY, type EditorBreakpoint, type EmbedNav } from '../types'
import { loadAppPreview } from './loadAppPreview'
let {
app,
workspace,
user,
license,
inWorkspace,
hideRefreshBar
}: {
app: AppWithLastVersion & { value: any }
workspace: string | undefined
user: UserExt | undefined
license: string | undefined
inWorkspace: boolean
hideRefreshBar: boolean
} = $props()
// The public routes hold the viewer and license as props, while low-code components
// read them from the stores (the license gates the app's custom CSS, for one).
$effect(() => {
if (inWorkspace) return
userStore.set(user)
if (license) enterpriseLicense.set(license)
})
// WIN-2006: inside the opaque viewer iframe, navigations to other routes
// (navbar "app" items) must happen on the TOP page — the iframe is cookieless,
// so navigating it would just show a login screen. PublicAppFrame provides the
// relay; outside the opaque viewer this is undefined and goto works directly.
const embedNav = getContext<EmbedNav | undefined>(EMBED_NAV_CONTEXT_KEY)
const breakpoint = writable<EditorBreakpoint>('lg')
</script>
<div
class={twMerge(
// `flex-col` matches the pre-sandbox in-workspace viewer exactly;
// the public viewer always used a plain `flex` wrapper. `min-h-full`, not
// `h-full`: the box has to reach the bottom of what the host gives it so the
// app's own background covers the window, and still grow with a grid taller
// than that. A viewport floor instead would overhang the page box by the
// height of the page header band.
inWorkspace ? 'min-h-full w-full flex flex-col' : 'min-h-full w-full flex',
app?.value?.['css']?.['app']?.['viewer']?.class,
'wm-app-viewer'
)}
style={app?.value?.['css']?.['app']?.['viewer']?.style}
>
{#await loadAppPreview()}
<Skeleton layout={[[4], 0.5, [50]]} />
{:then Module}
<Module.default
noBackend={false}
{hideRefreshBar}
context={{
email: user?.email,
name: user?.name,
groups: user?.groups,
username: user?.username,
query: urlParamsToObject(page.url.searchParams, { stripReserved: true }),
hash: page.url.hash.substring(1)
}}
{workspace}
summary={app.summary}
app={app.value}
appPath={app.path}
{breakpoint}
policy={app.policy}
isEditor={false}
replaceStateFn={(path) => goto(path)}
gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))}
/>
{:catch}
<div class="px-4 mt-20 w-full">
<Alert type="error" title="Could not load the app">Reload the page to try again.</Alert>
</div>
{/await}
</div>
frontend/src/lib/components/apps/editor/publicAppApi.ts
+82
View File
@@ -0,0 +1,82 @@
// The public app routes (`/public/...`, `/a/...`) render a raw app as little more than its
// bundle's iframe, so they must not statically import the generated client, `$lib/stores`
// or `$lib/utils`: each drags in most of the app shell. `vite.config.js`
// (`assertLeanPublicAppRoutes`) fails the build if they do. Anything heavier is imported
// lazily by the branch that needs it.
import { OpenAPI } from '$lib/gen/core/OpenAPI'
import type { AppWithLastVersion, User } from '$lib/gen/types.gen'
import type { UserExt } from '$lib/stores'
import { mapUserToUserExt } from '$lib/userExt'
export type PublicAppValue = AppWithLastVersion & { value: any; workspace_id?: string }
declare global {
interface Window {
/** Responses `app.html` requested before any module loaded, keyed by URL. */
__wmPublicAppPrefetch?: Record<string, Promise<Response>>
}
}
/** Carries `status` like the generated client's `ApiError`, which callers branch on. */
export class PublicApiError extends Error {
constructor(public status: number) {
super(`Request failed with status ${status}`)
}
}
/** Prefetched responses answer for the credential the page loaded with; call this when it
* changes (a sign-in or logout on the page) so later reads go to the network. */
export function dropPrefetched(): void {
delete window.__wmPublicAppPrefetch
}
function takePrefetched(url: string): Promise<Response> | undefined {
const prefetched = window.__wmPublicAppPrefetch
const res = prefetched?.[url]
if (res) delete prefetched![url]
return res
}
/** GET an API path with the page credential: the bearer token in `OpenAPI.TOKEN` when the
* share link carries one, the session cookie otherwise. A prefetched response is only
* taken in the cookie case, which is the only credential `app.html` sends. */
export async function publicGet<T>(path: string): Promise<T> {
const url = `${OpenAPI.BASE}${path}`
const token = typeof OpenAPI.TOKEN === 'string' && OpenAPI.TOKEN ? OpenAPI.TOKEN : undefined
const res = await ((!token && takePrefetched(url)) ||
fetch(url, {
// As the generated client does: the opaque viewer frame is cross-origin to the
// API, whose `Access-Control-Allow-Origin: *` refuses a credentialed request.
credentials: OpenAPI.WITH_CREDENTIALS ? OpenAPI.CREDENTIALS : undefined,
headers: token ? { Authorization: `Bearer ${token}` } : {}
}))
if (!res.ok) throw new PublicApiError(res.status)
return (res.headers.get('content-type')?.includes('json') ? res.json() : res.text()) as T
}
export async function getUserExtOrUndefined(workspace: string): Promise<UserExt | undefined> {
try {
return mapUserToUserExt(
await publicGet<User>(`/w/${encodeURI(workspace)}/users/whoami`),
workspace
)
} catch {
return undefined
}
}
/** Only fades the "Powered by" badge, so a failure is not worth surfacing. */
export async function getLicenseOrUndefined(): Promise<string | undefined> {
try {
return (await publicGet<string>('/ee_license')) || undefined
} catch {
return undefined
}
}
let appCss: Promise<unknown> | undefined
/** The app-wide stylesheet, which these routes need only off the raw-app path: sign-in,
* errors, skeletons and low-code apps. A raw app's bundle brings its own styles. */
export function loadAppCss(): Promise<unknown> {
return (appCss ??= import('$lib/assets/app.css'))
}
frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte
+150
View File
@@ -0,0 +1,150 @@
<!--
@component
The menu that adds a pipeline script started by a data source (a schedule, a
webhook, a queue…): the add node's, and the canvas's right-click one.
-->
<script lang="ts">
import type { Snippet } from 'svelte'
import PipelineInsertMenu, {
type PipelineInsertOptions,
type PipelineInsertPick
} from './PipelineInsertMenu.svelte'
import {
Clock,
Webhook,
Mail,
Zap,
Radio,
MessageSquare,
Database,
Send,
CloudCog,
Upload
} from 'lucide-svelte'
import type { ScriptLang } from '$lib/gen'
import type { NativeTriggerKind } from './types'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import type { PipelineOutputKind } from './pipelineTemplates'
interface Props {
onAddPipelineScript: (
language: ScriptLang,
path: string,
source: { kind: NativeTriggerKind; path: string | undefined },
outputKind: PipelineOutputKind,
aiPrompt?: string,
options?: PipelineInsertOptions
) => void
pathPrefix: string
defaultPathSuffix: string
trigger: Snippet<[{ open: boolean }]>
/** Opens the menu each time it changes. */
openSignal?: number
}
let {
onAddPipelineScript,
pathPrefix,
defaultPathSuffix,
trigger,
openSignal
}: Props = $props()
function handlePick(pick: PipelineInsertPick) {
if (!pick.language || !pick.path) return
// Native trigger annotation is marker-only — the binding lives on the
// trigger row's own `script_path`, which is created separately.
onAddPipelineScript(
pick.language as ScriptLang,
pick.path,
{ kind: pick.kindId as NativeTriggerKind, path: undefined },
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt,
{ schedule: pick.schedule, outputAsset: pick.outputAsset }
)
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'data_upload',
label: 'On data upload',
description: 'UI-first: run by uploading a file via the S3 picker',
icon: Upload
},
{
id: 'schedule',
label: 'On schedule',
description: 'Triggered by a schedule you create',
icon: Clock
},
{
id: 'webhook',
label: 'On webhook',
description: 'Triggered by an HTTP webhook',
icon: Webhook
},
{
id: 'email',
configuredAfterCreate: true,
label: 'On email',
description: 'Triggered by incoming email',
icon: Mail
},
{
id: 'kafka',
configuredAfterCreate: true,
label: 'On Kafka',
description: 'Triggered by a Kafka message',
icon: Zap
},
{
id: 'mqtt',
configuredAfterCreate: true,
label: 'On MQTT',
description: 'Triggered by an MQTT message',
icon: Radio
},
{
id: 'amqp',
configuredAfterCreate: true,
label: 'On AMQP',
description: 'Triggered by an AMQP (RabbitMQ) message',
icon: Radio
},
{
id: 'nats',
configuredAfterCreate: true,
label: 'On NATS',
description: 'Triggered by a NATS message',
icon: MessageSquare
},
{
id: 'postgres',
configuredAfterCreate: true,
label: 'On Postgres',
description: 'Triggered by a Postgres event',
icon: Database
},
{
id: 'sqs',
configuredAfterCreate: true,
label: 'On SQS',
description: 'Triggered by an SQS message',
icon: Send
},
{
id: 'gcp',
configuredAfterCreate: true,
label: 'On GCP Pub/Sub',
description: 'Triggered by a Pub/Sub message',
icon: CloudCog
}
]}
languages={PIPELINE_LANGUAGES as any}
{pathPrefix}
{defaultPathSuffix}
onPick={handlePick}
{openSignal}
{trigger}
/>
frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte
+73
View File
@@ -0,0 +1,73 @@
<!--
@component
The menu adding a pipeline script downstream of an asset: opened from the asset's
"+", or where that "+" was dropped on the blank canvas.
-->
<script lang="ts" module>
import type { ScriptLang } from '$lib/gen'
import type { AssetKind } from '$lib/components/assets/lib'
import type { PipelineInsertOptions } from './PipelineInsertMenu.svelte'
import type { PipelineOutputKind } from './pipelineTemplates'
export type AddScriptForAsset = (
asset: { kind: AssetKind; path: string },
language: ScriptLang,
scriptPath: string,
outputKind: PipelineOutputKind,
aiPrompt?: string,
options?: PipelineInsertOptions
) => void
</script>
<script lang="ts">
import { Code2 } from 'lucide-svelte'
import type { Snippet } from 'svelte'
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
let {
asset,
onAddScript,
pathPrefix = '',
defaultPathSuffix = '',
openSignal,
trigger
}: {
asset: { kind: AssetKind; path: string }
onAddScript: AddScriptForAsset
pathPrefix?: string
defaultPathSuffix?: string
openSignal?: number
trigger: Snippet<[{ open: boolean }]>
} = $props()
function handlePick(pick: PipelineInsertPick) {
if (pick.kindId === 'pipeline_script' && pick.language && pick.path) {
onAddScript(
asset,
pick.language as ScriptLang,
pick.path,
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt,
{ outputAsset: pick.outputAsset }
)
}
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'pipeline_script',
label: 'Add downstream pipeline script',
description: 'Triggered when this asset changes',
icon: Code2
}
]}
languages={PIPELINE_LANGUAGES as any}
{pathPrefix}
{defaultPathSuffix}
{openSignal}
onPick={handlePick}
{trigger}
/>
frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte
+23
View File
@@ -0,0 +1,23 @@
<!--
@component
The "+" on an asset's bottom edge: a circle at rest, widened into a labelled pill
while its menu is open or while it is being dragged. The label animates its
max-width (width can't transition to auto) and never wraps.
-->
<script lang="ts">
import { Plus } from 'lucide-svelte'
let { expanded }: { expanded: boolean } = $props()
</script>
<span
class="group/add h-6 min-w-6 rounded-full flex items-center justify-center bg-surface border border-gray-400 dark:border-gray-600 text-secondary hover:text-primary hover:border-gray-500 shadow-sm leading-none overflow-hidden transition-[padding] duration-200 ease-out data-[open=true]:pl-1"
data-open={expanded}
>
<Plus size={16} strokeWidth={2.5} class="shrink-0" />
<span
class="max-w-0 opacity-0 whitespace-nowrap text-3xs font-normal transition-[max-width,opacity,margin] duration-200 ease-out group-data-[open=true]/add:max-w-32 group-data-[open=true]/add:opacity-100 group-data-[open=true]/add:ml-1 group-data-[open=true]/add:mr-2"
>
Add downstream step
</span>
</span>
frontend/src/lib/components/assets/AssetGraph/AddNode.svelte
+29 -138
View File
@@ -1,145 +1,36 @@
<script lang="ts">
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import {
Plus,
Clock,
Webhook,
Mail,
Zap,
Radio,
MessageSquare,
Database,
Send,
CloudCog,
Upload
} from 'lucide-svelte'
import type { ScriptLang } from '$lib/gen'
import type { NativeTriggerKind } from './types'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import type { PipelineOutputKind } from './pipelineTemplates'
// Each left-column kind is just "pipeline script triggered by <trigger
// source>". id === the SCRIPT_TRIGGER_KIND value, so the handler can
// dispatch on it uniformly. Asset-triggered scripts are not in this
// menu; those live under the per-asset + inside the graph.
type KindId = NativeTriggerKind
import AddDataSourceMenu from './AddDataSourceMenu.svelte'
import { Plus } from 'lucide-svelte'
import { NODE } from '$lib/components/graph/util'
interface Props {
data: {
onAddPipelineScript: (
language: ScriptLang,
path: string,
source: { kind: NativeTriggerKind; path: string | undefined },
outputKind: PipelineOutputKind,
aiPrompt?: string
) => void
pathPrefix: string
defaultPathSuffix: string
}
data: Pick<
import('svelte').ComponentProps<typeof AddDataSourceMenu>,
'onAddPipelineScript' | 'pathPrefix' | 'defaultPathSuffix'
>
}
let { data }: Props = $props()
function handlePick(pick: PipelineInsertPick) {
if (!pick.language || !pick.path) return
const kindId = pick.kindId as KindId
const outputKind = (pick.outputKind ?? 'none') as PipelineOutputKind
// Native trigger annotation is marker-only — the binding lives on
// the trigger row's own `script_path`, which the user creates
// separately. Seed with `path: undefined`.
data.onAddPipelineScript(
pick.language as ScriptLang,
pick.path,
{ kind: kindId, path: undefined },
outputKind,
pick.aiPrompt
)
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'data_upload',
label: 'On data upload',
description: 'UI-first: run by uploading a file via the S3 picker',
icon: Upload
},
{
id: 'schedule',
label: 'On schedule',
description: 'Triggered by a schedule you create',
icon: Clock
},
{
id: 'webhook',
label: 'On webhook',
description: 'Triggered by an HTTP webhook',
icon: Webhook
},
{
id: 'email',
label: 'On email',
description: 'Triggered by incoming email',
icon: Mail
},
{
id: 'kafka',
label: 'On Kafka',
description: 'Triggered by a Kafka message',
icon: Zap
},
{
id: 'mqtt',
label: 'On MQTT',
description: 'Triggered by an MQTT message',
icon: Radio
},
{
id: 'amqp',
label: 'On AMQP',
description: 'Triggered by an AMQP (RabbitMQ) message',
icon: Radio
},
{
id: 'nats',
label: 'On NATS',
description: 'Triggered by a NATS message',
icon: MessageSquare
},
{
id: 'postgres',
label: 'On Postgres',
description: 'Triggered by a Postgres event',
icon: Database
},
{
id: 'sqs',
label: 'On SQS',
description: 'Triggered by an SQS message',
icon: Send
},
{
id: 'gcp',
label: 'On GCP Pub/Sub',
description: 'Triggered by a Pub/Sub message',
icon: CloudCog
}
]}
languages={PIPELINE_LANGUAGES as any}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
onPick={handlePick}
>
{#snippet trigger()}
<!-- Quiet insert affordance, mirroring the flow editor's inline +
buttons (bg-surface + gray border + secondary text) — a filled
accent circle outweighed every real node on the canvas. -->
<button
type="button"
class="w-8 h-8 rounded-full flex items-center justify-center bg-surface border border-gray-400 dark:border-gray-600 text-secondary shadow-sm hover:bg-surface-hover transition-colors cursor-pointer"
title="Add to pipeline"
>
<Plus size={16} />
</button>
{/snippet}
</PipelineInsertMenu>
<!-- The layout gives this node a full node-width slot; centering the pill in
it lines it up with the nodes below whatever its label's width. -->
<div class="flex justify-center" style="width: {NODE.width}px;">
<AddDataSourceMenu
onAddPipelineScript={data.onAddPipelineScript}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
>
{#snippet trigger()}
<!-- Quiet insert affordance, mirroring the flow editor's inline +
buttons (bg-surface + gray border + secondary text) — a filled
accent pill would outweigh every real node on the canvas. -->
<button
type="button"
class="h-8 px-3 rounded-full flex items-center gap-1.5 whitespace-nowrap bg-surface border border-gray-400 dark:border-gray-600 text-xs font-normal text-secondary shadow-sm hover:bg-surface-hover transition-colors cursor-pointer"
>
<Plus size={16} class="shrink-0" />
Add data source
</button>
{/snippet}
</AddDataSourceMenu>
</div>
frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte
+1156 -73
View File
File diff suppressed because it is too large Load Diff
frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte
+229 -158
View File
@@ -20,10 +20,17 @@
SquareFunction,
Trash2,
X,
Pencil
Pencil,
ChevronDown,
Zap
} from 'lucide-svelte'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import { tick } from 'svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import {
upstreamScriptItems,
upstreamTriggerItems,
type UpstreamScript,
type UpstreamTriggerAction
} from './upstreamScriptItems'
import { inferArgs } from '$lib/infer'
import { emptySchema, sendUserToast } from '$lib/utils'
import type { Schema } from '$lib/common'
@@ -41,6 +48,7 @@
import { extractDraftMacros } from './resolveGraph'
import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph'
import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte'
import Badge from '$lib/components/common/badge/Badge.svelte'
import S3FilePreview from '$lib/components/S3FilePreview.svelte'
import DataTablePreview from './DataTablePreview.svelte'
import DucklakeAssetPanel from './DucklakeAssetPanel.svelte'
@@ -117,7 +125,9 @@
// (`onDraftPersist`). `onDraftPersist` stays the authoritative commit on
// navigate-away; this is the continuous feed that keeps the draft's
// server copy current while the user is still typing.
onContentChange?: (scriptPath: string | undefined, content: string) => void
// `base`: the deployed copy being edited, so the parent can autosave unsaved
// edits to it before they are promoted to a draft. Unset for a draft.
onContentChange?: (scriptPath: string | undefined, content: string, base?: Script) => void
// Fires when the user navigates away from a draft (selects another
// node, closes the pane, etc.) — the parent must persist the new
// content + write outputs into its drafts Map. Without this,
@@ -133,7 +143,8 @@
writes: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Body-inferred reads, so the parent's draft keeps its input
// lineage while inactive (no live inference runs for it).
reads: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Undefined when nothing was inferred: keep what the draft has.
reads?: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Full edited script — set when the source is a persisted
// script (needed to seed a brand-new draft entry).
script?: Script
@@ -156,6 +167,12 @@
path: string
unsaved?: boolean
}>
/** The selected asset's producing scripts, for its "Edit script" action,
* which calls `onEditScript` with the one picked. */
producerScripts?: UpstreamScript[]
onEditScript?: (path: string) => void
/** The selected asset's upstream triggers that can be opened from here. */
producerTriggers?: UpstreamTriggerAction[]
// Pipeline-wide column-lineage graph (built by the parent page from the
// resolved graph). Drives the transitive column-lineage trace shown for a
// selected materialized asset.
@@ -215,15 +232,18 @@
// the edges. Counter rather than boolean so back-to-back runs
// re-fire the effect even if no other prop changed.
requestRunSignal?: number
// Folder-scoped non-editable prefix shown next to the suffix
// editor when the user renames a draft (e.g. `f/<folder>/`). The
// new path = pathPrefix + suffix.
pathPrefix?: string
// Called when the user renames a draft — the parent reseats the
// path key in its drafts map and updates activeDraftPath. Returns
// true on success; false on collision/validation failure so the
// popover can keep itself open and surface the error inline.
onDraftPathChange?: (oldPath: string, newPath: string) => boolean | string
onDraftMetaChange?: (
path: string,
meta: { summary: string; labels: string[] | undefined }
) => void
// The pipeline's folder prefix (`f/<folder>/`). A draft renamed outside it
// would be deployed out of the pipeline, so the rename is refused.
pathPrefix?: string
// Bumped by the parent (e.g. from the runnable-node action menu) to
// auto-open the archive/delete confirmation modal for the currently
// loaded persisted script. No-ops while the pane is showing a draft
@@ -280,6 +300,19 @@
// can persist a data-upload entry's staged input (drives node readiness).
// `isValid` is the full-schema validity, not just "file present".
onRunFormArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
// Show only the open script's inputs and runs (a data-upload click), not
// its editor. `onRunInputs` runs it with them, drafts included, and
// `onShowScript` leaves this view for the script.
inputsOnly?: boolean
/** `live` is the pane's buffer when it differs from what is stored: always
* for a draft, and for a deployed script edited before switching to the
* inputs view. A run with it previews that body. */
onRunInputs?: (
path: string,
args: Record<string, any>,
live?: { content: string; language: ScriptLang }
) => Promise<string | undefined>
onShowScript?: () => void
}
let {
selection,
@@ -301,6 +334,9 @@
onScriptRenamed,
onScriptRemoved,
selectionProducers = [],
producerScripts = [],
onEditScript,
producerTriggers = [],
selectionColumnGraph,
selectionColumnLoading = false,
selectionColumnTruncated = false,
@@ -315,8 +351,9 @@
activeRunnable,
onTestStateChange,
requestRunSignal,
pathPrefix = '',
onDraftPathChange,
onDraftMetaChange,
pathPrefix = '',
requestRemoveSignal,
downstreamSubscribers = 0,
onStartBoundedRun,
@@ -328,10 +365,15 @@
onRunByPath,
onRunCascadeByPath,
runFormInitialArgs,
onRunFormArgsChange
onRunFormArgsChange,
inputsOnly = false,
onRunInputs,
onShowScript
}: Props = $props()
let readOnly = $derived(mode !== 'edit')
// Someone who cannot switch the pipeline to edit mode only gets to read it.
let editScriptLabel = $derived(readOnly && !onRequestEdit ? 'Open script' : 'Edit script')
// Root element of the pane — used to scope the S3-input lookup for the
// data_upload focus effect.
@@ -408,7 +450,7 @@
// PipelineScriptView renders instead of the test panel.
let viewReadyForTarget = $derived.by(
() =>
readOnly &&
(readOnly || inputsOnly) &&
runTargetPath !== undefined &&
script?.path === runTargetPath &&
(isDraft || (!scriptRes.loading && scriptRes.current?.path === runTargetPath))
@@ -484,6 +526,12 @@
// alongside `liveBodyAssets` and forwarded so the live graph can show
// inferred column lineage on the edited script before it deploys.
let liveColumnLineage = $state<ColumnLineage[] | undefined>(undefined)
// The script whose editor last bound `liveBodyAssets` (undefined there means it
// inferred nothing). They stay bound after that editor unmounts, so a script
// that never had one (the inputs-only view) must not report or persist them as
// its lineage. Set by an effect declared after the save-on-switch one, whose
// cleanup still needs the previous owner when the selection moves.
let liveAssetsOwner: string | undefined = undefined
// Bumped when the runs panel reports a watched job has reached a
// terminal state. Drives S3FilePreview's refreshKey so the preview
@@ -549,7 +597,9 @@
let args = $state<Record<string, any>>({})
let argsSeedPath: string | undefined = undefined
$effect.pre(() => {
const p = script?.path
// Re-seeded on leaving the inputs-only view too, so the file picked there
// shows in the editor's test form.
const p = script?.path === undefined ? undefined : `${script.path}|${inputsOnly}`
if (p === argsSeedPath) return
argsSeedPath = p
args = runFormInitialArgs ? structuredClone($state.snapshot(runFormInitialArgs)) : {}
@@ -559,7 +609,8 @@
// required field (not just the S3 file) is satisfied.
let runFormIsValid = $state(true)
$effect(() => {
if (readOnly || !script) return
// Inputs-only mounts no ScriptEditor: PipelineScriptView reports its own args.
if (readOnly || inputsOnly || !script) return
onRunFormArgsChange?.(script.path, $state.snapshot(args), runFormIsValid)
})
@@ -596,18 +647,25 @@
b: Array<{ kind: string; path: string }>
) => a.length === b.length && a.every((x, i) => x.kind === b[i]?.kind && x.path === b[i]?.path)
return () => {
const writes = (liveBodyAssets ?? [])
// No editor inferred this script (the inputs-only view): keep its stored
// lineage rather than persisting it as empty.
const inferred = liveAssetsOwner === captured.path
const writes = !inferred
? (writesAtRegister ?? [])
: (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'w' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
const reads = (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'r' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
const reads = !inferred
? readsAtRegister
: (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'r' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
// Draft runs: emit only when content or lineage changed in THIS clone.
// An unconditional emit ping-pongs forever when the entry is rewritten
// externally while the pane stays mounted (rename rekey, AI edit):
@@ -617,6 +675,7 @@
(captured.content ?? '') === contentAtRegister &&
refsEq(writesAtRegister ?? [], writes) &&
readsAtRegister != undefined &&
reads != undefined &&
refsEq(readsAtRegister, reads)
)
return
@@ -651,12 +710,42 @@
}
})
$effect(() => {
if (script && !readOnly && !inputsOnly) liveAssetsOwner = script.path
})
let saving = $state(false)
// What this pane last deployed, so the persist-back cleanup can tell "the
// buffer equals the new deployed head" from real unsaved edits (plain
// variable: only read inside the untracked cleanup).
let deployedFromPane: { path: string; content: string } | undefined = undefined
let isDraft = $derived(draftScript != undefined)
function liveBody(): { content: string; language: ScriptLang } | undefined {
if (!script) return undefined
const live = { content: script.content ?? '', language: script.language as ScriptLang }
if (isDraft) return live
const orig = scriptRes.current
return orig?.path === script.path && (orig.content ?? '') !== live.content ? live : undefined
}
// A draft's stored schema is empty until an editor infers it, and the inputs
// view mounts none: infer from the body here so its fields (the file picker)
// show up.
const inputsSchema = resource(
[() => (inputsOnly && isDraft ? script?.content : undefined), () => script?.language],
async ([content, language]) => {
if (content === undefined || !language) return undefined
const schema = emptySchema()
try {
await inferArgs(language, content, schema as Schema)
} catch {
return undefined
}
return schema
}
)
let inputsScript = $derived(
script && inputsSchema.current ? { ...script, schema: inputsSchema.current } : undefined
)
// Single trash-bin button opens one modal that exposes both Archive
// (always available) and Delete permanently (admin-only). Archive is
@@ -763,12 +852,17 @@
onAnnotationsChange?.(script?.path, liveAnnotations)
})
$effect(() => {
if (readOnly) return
if (readOnly || (inputsOnly && liveAssetsOwner !== script?.path)) return
onAssetsChange?.(script?.path, liveBodyAssets ?? [], liveColumnLineage)
})
$effect(() => {
if (readOnly) return
onContentChange?.(script?.path, script?.content ?? '')
const base = draftScript ? undefined : scriptRes.current
onContentChange?.(
script?.path,
script?.content ?? '',
base && base.path === script?.path ? base : undefined
)
})
async function save() {
@@ -910,49 +1004,6 @@
JSON.stringify(script.schema ?? null) === JSON.stringify(orig.schema ?? null)
)
})
// Suffix editor for the draft-path popover. Seeded from the current
// path each time the popover opens so the user starts with what they
// see, not stale state from an earlier rename.
let draftPathSuffix = $state('')
let draftPathError = $state<string | undefined>(undefined)
let draftPathInput: HTMLInputElement | undefined = $state(undefined)
function suffixOf(fullPath: string): string {
return fullPath.startsWith(pathPrefix) ? fullPath.slice(pathPrefix.length) : fullPath
}
async function openDraftPathEditor() {
draftPathSuffix = script ? suffixOf(script.path) : ''
draftPathError = undefined
await tick()
draftPathInput?.focus()
draftPathInput?.select()
}
function confirmDraftPath(close: () => void) {
if (!script) return
const suffix = draftPathSuffix.trim()
if (!suffix) {
draftPathError = 'Path cannot be empty'
return
}
const newPath = pathPrefix + suffix
if (newPath === script.path) {
close()
return
}
const result = onDraftPathChange?.(script.path, newPath)
if (result === true || result === undefined) {
script.path = newPath
draftPathError = undefined
close()
} else if (typeof result === 'string') {
draftPathError = result
} else {
draftPathError = 'Path already in use'
}
}
</script>
<div class="flex flex-col h-full bg-surface" bind:this={paneEl}>
@@ -964,80 +1015,34 @@
{@const draftScriptPath = script.path}
<Code2 size={16} class="shrink-0 text-tertiary" />
{#if onDraftPathChange && !readOnly}
<!-- Inline rename popover for drafts. The persisted-script
branch uses SummaryPathDisplay which round-trips through
updateItemPathAndSummary; drafts have no server row yet,
so we just rekey the parent's drafts map locally. -->
<Popover
placement="bottom-start"
contentClasses="p-3"
usePointerDownOutside
on:openChange={(e) => {
if (e.detail) openDraftPathEditor()
<!-- Same summary/path popover as a deployed script, saved into the
drafts map instead of through the API: a draft has no row yet. -->
<SummaryPathDisplay
summary={script.summary ?? ''}
path={draftScriptPath}
labels={script.labels ?? []}
kind="script"
onSaved={() => {}}
saveOverride={({ path, summary, labels }) => {
if (pathPrefix && (!path.startsWith(pathPrefix) || path === pathPrefix)) {
return `A pipeline script stays in ${pathPrefix}: pick a name inside that folder`
}
// Before the rename, which carries the draft entry over as it is.
onDraftMetaChange?.(draftScriptPath, { summary, labels })
if (script) {
script.summary = summary
script.labels = labels
}
if (path === draftScriptPath) return undefined
const result = onDraftPathChange(draftScriptPath, path)
if (result !== true && result !== undefined) {
return typeof result === 'string' ? result : 'Path already in use'
}
if (script) script.path = path
return undefined
}}
>
{#snippet trigger()}
<button
type="button"
class="flex flex-col min-w-0 text-left px-2 py-1 rounded-md hover:bg-surface-hover transition-colors group"
title="Edit draft path"
>
<span
class="text-3xs uppercase tracking-wide text-tertiary flex items-center gap-1"
>
Draft pipeline script
<Pencil size={9} class="opacity-0 group-hover:opacity-60 transition-opacity" />
</span>
<span class="text-xs font-mono truncate" title={draftScriptPath}
>{draftScriptPath}</span
>
</button>
{/snippet}
{#snippet content({ close })}
<div class="flex flex-col gap-2 w-[420px]">
<span class="text-2xs font-normal text-secondary">Path</span>
<div
class="flex items-stretch border rounded-md bg-surface overflow-hidden focus-within:ring-2 focus-within:ring-blue-400"
>
{#if pathPrefix}
<span
class="flex items-center px-2 bg-surface-secondary text-tertiary text-sm font-mono border-r select-none"
>
{pathPrefix}
</span>
{/if}
<input
bind:this={draftPathInput}
bind:value={draftPathSuffix}
onkeydown={(e) => {
if (e.key === 'Enter') {
e.preventDefault()
confirmDraftPath(close)
} else if (e.key === 'Escape') {
e.preventDefault()
close()
}
}}
class="flex-1 min-w-0 px-2 py-1.5 text-sm font-mono bg-transparent focus:outline-none"
placeholder="my_script"
/>
</div>
{#if draftPathError}
<span class="text-2xs text-red-500">{draftPathError}</span>
{/if}
<div class="flex justify-end">
<Button
variant="accent"
unifiedSize="sm"
disabled={!draftPathSuffix.trim()}
onClick={() => confirmDraftPath(close)}
>
Rename
</Button>
</div>
</div>
{/snippet}
</Popover>
/>
<Badge color="gray">Draft</Badge>
{:else}
<div class="flex flex-col min-w-0">
<span class="text-3xs uppercase tracking-wide text-tertiary">
@@ -1097,6 +1102,66 @@
{/if}
</div>
<div class="flex items-center gap-1 shrink-0">
{#if selection?.kind === 'asset' && onEditScript && producerScripts.length === 1}
<Button
variant="default"
unifiedSize="sm"
startIcon={{ icon: Pencil }}
onclick={() => onEditScript(producerScripts[0].path)}
title={`Open ${producerScripts[0].path}, which builds this asset`}
>
{editScriptLabel}
</Button>
{:else if selection?.kind === 'asset' && onEditScript && producerScripts.length > 1}
<DropdownV2
items={upstreamScriptItems(producerScripts, onEditScript)}
placement="bottom-end"
enableFlyTransition
>
{#snippet buttonReplacement()}
<Button
nonCaptureEvent
variant="default"
unifiedSize="sm"
startIcon={{ icon: Pencil }}
endIcon={{ icon: ChevronDown }}
title={`${producerScripts.length} scripts build this asset`}
>
{editScriptLabel}
</Button>
{/snippet}
</DropdownV2>
{/if}
{#if selection?.kind === 'asset' && producerTriggers.length === 1}
<Button
variant="default"
unifiedSize="sm"
startIcon={{ icon: Zap }}
onclick={() => producerTriggers[0].onOpen()}
title={producerTriggers[0].detail}
>
{producerTriggers[0].label}
</Button>
{:else if selection?.kind === 'asset' && producerTriggers.length > 1}
<DropdownV2
items={upstreamTriggerItems(producerTriggers)}
placement="bottom-end"
enableFlyTransition
>
{#snippet buttonReplacement()}
<Button
nonCaptureEvent
variant="default"
unifiedSize="sm"
startIcon={{ icon: Zap }}
endIcon={{ icon: ChevronDown }}
title={`${producerTriggers.length} triggers start the scripts that build this asset`}
>
Triggers
</Button>
{/snippet}
</DropdownV2>
{/if}
{#if readOnly && onRequestEdit}
<Button
variant="accent"
@@ -1118,21 +1183,10 @@
title="Discard draft"
/>
{/if}
<!-- Action order, left → right: trash, external link, save, close.
Save sits closest to Close so the primary commit action is
anchored at the right edge of the bar; trash lives at the
far left so destructive ops are visually separated from
navigation/commit. Mirrors the draft Discard placement. -->
{#if !readOnly && !isDraft && isScriptView && script?.hash}
<Button
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: Trash2 }}
onclick={() => (removeOpen = true)}
iconOnly
title="Archive or delete"
/>
{/if}
<!-- Action order, left → right: external link, save, close. Save sits
closest to Close so the primary commit action is anchored at the
right edge of the bar. Removing a script goes through its node's
menu. -->
{#if !readOnly && !isDraft && selection?.kind === 'runnable'}
<Button
variant="subtle"
@@ -1338,14 +1392,31 @@
{/if}
</div>
{:else if !isDraft && scriptRes.loading && !script}
<div class="absolute inset-0 flex items-center justify-center gap-2 text-tertiary">
<Loader2 size={16} class="animate-spin" />
<span class="text-xs">Loading script…</span>
</div>
<!-- Blank while the script loads: the fetch is short, and a spinner only flashes. -->
{:else if !isDraft && scriptRes.error}
<div class="p-3 text-xs text-red-500">
Failed to load: {scriptRes.error.message}
</div>
{:else if script && inputsOnly}
{#key script.path}
<PipelineScriptView
{isDraft}
canRun
runsDrafts
inputsOnly
{onShowScript}
onRun={onRunInputs && ((path, args) => onRunInputs(path, args, liveBody()))}
script={inputsScript ?? script}
{runsRefreshKey}
{runsPendingJobId}
initialArgs={runFormInitialArgs}
onArgsChange={onRunFormArgsChange}
onRunCompleted={() => {
previewRefreshKey += 1
onRunCompleted?.()
}}
/>
{/key}
{:else if script && readOnly}
<!-- Read-only modes: no Monaco/ScriptEditor (operators are
backend-blocked from previews anyway) — highlighted source,
frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte
+32 -22
View File
@@ -1,6 +1,7 @@
<script lang="ts">
import { BaseEdge, getBezierPath, type EdgeProps } from '@xyflow/svelte'
import { NODE } from '$lib/components/graph/util'
import { PIPELINE_NODE_HEIGHT } from './assetGraphLayout'
import { FlaskConical, Columns3, SquareFunction, BellOff } from 'lucide-svelte'
import type { ColumnLineage, DataTest } from './parsePipelineAnnotations'
@@ -103,33 +104,42 @@
// straight under the nodes in between — the child-and-grandchild-of-the-
// same-node case, where the tidy-tree layout puts all three on one
// column. Detour those through the gutter beside the column.
const SKIP_DY = NODE.gap.vertical * 2 + NODE.height
const SKIP_DY = NODE.gap.vertical * 2 + PIPELINE_NODE_HEIGHT
const NEAR_VERTICAL_DX = NODE.width / 4
// Lane offset: half a node plus a margin that stays inside the
// inter-column gutter (gap.horizontal / 2).
const LANE = NODE.width / 2 + NODE.gap.horizontal / 2
const CORNER = 14
// Rounded-orthogonal detour: drop out of the source, swing into the
// gutter lane at `midX`, run it down past the intermediate rows, swing
// back into the target.
// Detour through the gutter lane at `midX`, drawn with the same curves as
// every other edge: curve out of the source into the lane within the gap
// below it, run down the lane past the rows in between, curve into the
// target within the gap above it.
function gutterPath(midX: number): string {
const yTop = sourceY + 24
const yBot = targetY - 24
const outDir = midX > sourceX ? 1 : -1
const inDir = targetX > midX ? 1 : -1
return [
`M ${sourceX} ${sourceY}`,
`L ${sourceX} ${yTop - CORNER}`,
`Q ${sourceX} ${yTop} ${sourceX + outDir * CORNER} ${yTop}`,
`L ${midX - outDir * CORNER} ${yTop}`,
`Q ${midX} ${yTop} ${midX} ${yTop + CORNER}`,
`L ${midX} ${yBot - CORNER}`,
`Q ${midX} ${yBot} ${midX + inDir * CORNER} ${yBot}`,
`L ${targetX - inDir * CORNER} ${yBot}`,
`Q ${targetX} ${yBot} ${targetX} ${yBot + CORNER}`,
`L ${targetX} ${targetY}`
].join(' ')
const bend = NODE.gap.vertical
const yTop = sourceY + bend
// Clamp only downward edges: an upward (feedback) edge must still run up the
// lane, or its last curve cuts across the node the detour was clearing.
const yBot = targetY > sourceY ? Math.max(yTop, targetY - bend) : targetY - bend
const [out] = getBezierPath({
sourceX,
sourceY,
sourcePosition,
targetX: midX,
targetY: yTop,
targetPosition,
curvature: 0.25
})
const [into] = getBezierPath({
sourceX: midX,
sourceY: yBot,
sourcePosition,
targetX,
targetY,
targetPosition,
curvature: 0.25
})
// Continue from the lane rather than lifting the pen.
return `${out} L${midX},${yBot} ${into.replace(/^M/, 'L')}`
}
// Long edges bend out of the source within the layer gap and descend in
@@ -178,7 +188,7 @@
path={edgePath}
{markerEnd}
{style}
interactionWidth={0}
interactionWidth={12}
label={undefined}
labelStyle={undefined}
/>
frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte
+568 -169
View File
@@ -2,9 +2,14 @@
import { Handle, Position } from '@xyflow/svelte'
import { twMerge } from 'tailwind-merge'
import AssetGenericIcon from '$lib/components/icons/AssetGenericIcon.svelte'
import { formatShortAssetPath, type AssetKind } from '$lib/components/assets/lib'
import { NODE } from '$lib/components/graph/util'
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import { formatAssetKind, formatShortAssetPath, type AssetKind } from '$lib/components/assets/lib'
import PipelineNodeCard from './PipelineNodeCard.svelte'
import NodeActionsMenu from './NodeActionsMenu.svelte'
import NodeFixButton, { type NodeFix } from './NodeFixButton.svelte'
import NodeOnDot from './NodeOnDot.svelte'
import { PIPELINE_NODE_HEADER } from './assetGraphLayout'
import AddDownstreamMenu from './AddDownstreamMenu.svelte'
import AddDownstreamPill from './AddDownstreamPill.svelte'
import {
ArrowUpRight,
Code2,
@@ -12,19 +17,28 @@
History,
Play,
Loader2,
Plus,
ShieldCheck,
ShieldAlert,
CheckCircle2,
ChevronDown,
Pencil,
CircleSlash,
Trash2,
Upload,
XCircle
} from 'lucide-svelte'
import { Button } from '$lib/components/common'
import type { ScriptLang } from '$lib/gen'
import { enterpriseLicense } from '$lib/stores'
import { sendUserToast } from '$lib/utils'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import { sendUserToast, type Item } from '$lib/utils'
import type { PipelineOutputKind } from './pipelineTemplates'
import type { DbtAssetProvenance } from './types'
import DbtIcon from '$lib/components/icons/DbtIcon.svelte'
import LanguageIcon from '$lib/components/common/languageIcons/LanguageIcon.svelte'
import RunStateChip from './RunStateChip.svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import { upstreamScriptItems } from './upstreamScriptItems'
import type { RunnableRunState } from './activeRunnables.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
@@ -33,6 +47,49 @@
// `content` / `language` so the page-level run handler can dispatch to
// `runScriptPreview` instead of `runScriptByPath` (which 404s for
// non-deployed scripts).
/** What starts a producing script, as its chip shows it. */
export type TriggerChipData = {
label: string
/** The native trigger kind; a data upload renders as a call to action. */
kind?: string
/** No chip: the node's incoming edges already show what starts it. */
hidden?: boolean
missing?: boolean
draft?: boolean
/** Opens the trigger's editor; unset for what has none (manual, asset change). */
onOpen?: () => void
selected?: boolean
}
export type AssetUpstreamChips =
| { none: true }
| {
multiple: true
/** The producing scripts; one when only its triggers are ambiguous. */
scripts: Array<{
runnableId: string
path: string
summary?: string
language?: ScriptLang
unsaved?: boolean
onOpen: () => void
}>
/** The scripts' own triggers, each named with its script. */
triggers: Array<TriggerChipData & { script: string }>
selected?: boolean
}
| {
runnableId: string
path: string
summary?: string
language?: ScriptLang
unsaved?: boolean
runState?: RunnableRunState
trigger: TriggerChipData
onOpen: () => void
selected?: boolean
}
export type AssetProducer = {
kind: 'script' | 'flow'
path: string
@@ -51,6 +108,8 @@
// "current view of <dim>" marker so it reads as a derived node, not an
// unrelated table.
derived_from?: string
// Why the node is misconfigured, when it is: renders it red.
error?: string
// dbt provenance when this warehouse table is a dbt node: which model
// it is, how dbt materializes it, its tags and its generic tests.
dbt?: DbtAssetProvenance
@@ -65,10 +124,14 @@
language: ScriptLang,
scriptPath: string,
outputKind: PipelineOutputKind,
aiPrompt?: string
aiPrompt?: string,
options?: import('./PipelineInsertMenu.svelte').PipelineInsertOptions
) => void
pathPrefix?: string
defaultPathSuffix?: string
/** Starts dragging the "+" onto a script (subscribing it to this asset)
* or onto the blank canvas (opening the add menu there). */
onStartFeedDrag?: (e: PointerEvent) => void
// Producer scripts/flows that write to this asset, supplied by
// the canvas from the graph's write/rw edges. Drives the on-hover
// "Run" button. Includes unsaved/draft producers so the button
@@ -99,6 +162,41 @@
// instead of only settling once the job ends.
runStatus?: 'running' | 'materialized' | 'failed'
runRowCount?: number | null
/** Set in the assets-only view, where the producing script and its
* trigger are folded into the asset: `none` for an asset nothing in the
* pipeline writes, `multiple` when the producer or its trigger is
* ambiguous. */
upstream?: AssetUpstreamChips
/** What is wrong with the node, and how to fix it: replaces the "+". */
fix?: NodeFix
/** Assets-only: errors of the scripts and triggers folded into this node. */
foldedFixes?: NodeFix[]
/** Assets-only: starts dragging the top dot onto an asset, subscribing the
* scripts that build this one to it. */
onStartOnDrag?: (e: PointerEvent) => void
/** The assets-only node for the scripts that build no asset. */
noAsset?: boolean
/** The No-asset node's script, by its file name. */
noAssetName?: string
/** Assets-only: the node is only its script's banner. */
bannerOnly?: boolean
/** Assets-only: a script folded into the node has an error. */
scriptError?: boolean
/** No script in the workspace writes it; a flow's write fires no asset trigger. */
neverWritten?: boolean
/** Card width in the assets-only view, sized to its text. */
width?: number
/** Assets-only view: the producing script and trigger no other asset
* shares, which the menu offers to delete one by one or together. */
upstreamDelete?: import('./assetsOnlyView').AssetUpstreamDelete
/** When no combined delete is possible: the folded scripts, each deletable
* on its own. */
scriptDeletes?: Array<{ path: string; unsaved: boolean }>
triggerDeletes?: Array<NonNullable<import('./assetsOnlyView').AssetUpstreamDelete['trigger']>>
onDeleteUpstream?: (target: import('./assetsOnlyView').AssetUpstreamDelete) => void
deleteVerb?: 'Delete' | 'Archive'
/** Why nothing can be deleted, when the producer is shared. */
deleteBlocked?: string
}
// SvelteFlow injects this on the node component when the user clicks
// the node. Combined with our own `hovered` state to drive the
@@ -140,19 +238,94 @@
}
}
function handlePick(pick: PipelineInsertPick) {
if (pick.kindId === 'pipeline_script' && pick.language && pick.path) {
data.onAddScript?.(
{ kind: data.asset_kind, path: data.path },
pick.language as ScriptLang,
pick.path,
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt
)
}
}
let showAdd = $derived(data.onAddScript != undefined)
// A red asset needs fixing where its error comes from, not a downstream step;
// and a step downstream of what nothing writes would never run on its writes.
let showAdd = $derived(data.onAddScript != undefined && !data.error && !data.neverWritten)
let menuItems: Item[] = $derived(
data.onDeleteUpstream && data.upstreamDelete
? deleteItems(data.upstreamDelete, data.onDeleteUpstream)
: data.onDeleteUpstream && (data.scriptDeletes?.length || data.triggerDeletes?.length)
? [
...(data.scriptDeletes ?? []).map((script) => ({ script })),
...(data.triggerDeletes ?? []).map((trigger) => ({ trigger }))
].flatMap((t) => deleteItems(t, data.onDeleteUpstream!))
: data.deleteBlocked
? [
{
displayName: `Can't delete: ${data.deleteBlocked}`,
icon: Trash2,
disabled: true
}
]
: []
)
const lastSegment = (p: string) => p.split('/').pop() ?? p
// A script is archived rather than deleted when the user may not delete it;
// a trigger is always deleted.
function deleteItems(
t: import('./assetsOnlyView').AssetUpstreamDelete,
run: (target: import('./assetsOnlyView').AssetUpstreamDelete) => void
): Item[] {
const items: Item[] = []
if (t.script) {
const script = t.script
items.push({
displayName: `${data.deleteVerb ?? 'Delete'} script ${lastSegment(script.path)}`,
icon: Trash2,
type: 'delete' as const,
action: () => run({ script })
})
}
if (t.trigger) {
const trigger = t.trigger
items.push({
displayName: `Delete ${trigger.kind} trigger ${lastSegment(trigger.path)}`,
icon: Trash2,
type: 'delete' as const,
action: () => run({ trigger })
})
}
if (t.script && t.trigger) {
items.push({
displayName: `${data.deleteVerb === 'Archive' ? 'Remove' : 'Delete'} all 2 items`,
icon: Trash2,
type: 'delete' as const,
action: () => run(t)
})
}
return items
}
let upstream = $derived(data.upstream)
let HEADER_SELECTED = $derived(
data.scriptError
? 'bg-red-200 dark:bg-red-600 hover:bg-red-200 dark:hover:bg-red-600'
: 'bg-surface-accent-selected hover:bg-surface-accent-selected text-accent'
)
const CHIP_SELECTED =
'bg-surface-accent-selected border-border-selected hover:bg-surface-accent-selected text-accent'
let producer = $derived(upstream && 'path' in upstream ? upstream : undefined)
let coProducers = $derived(upstream && 'scripts' in upstream ? upstream.scripts : [])
let hasHeader = $derived(!!producer || coProducers.length > 0)
// The corner controls sit on the card's corner, below the header tab:
// -top-2 plus PIPELINE_NODE_HEADER (22px).
const CARD_CORNER_TOP = 'top-[14px]'
let coSelected = $derived(!!(upstream && 'scripts' in upstream && upstream.selected))
let coProducerItems = $derived(
upstreamScriptItems(coProducers, (path) => coProducers.find((s) => s.path === path)?.onOpen())
)
let scriptMenuOpen = $state(false)
let coTriggers = $derived(upstream && 'triggers' in upstream ? upstream.triggers : [])
let coTriggerItems: Item[] = $derived(
coTriggers.map((t) => ({
displayName: `${t.label}${t.draft ? ' · draft' : ''}`,
description: coProducers.length > 1 ? t.script : undefined,
disabled: !t.onOpen,
action: () => t.onOpen?.()
}))
)
let triggerMenuOpen = $state(false)
// dbt badge. `materialized` is dbt's own word rather than the Windmill
// strategy because `view` and `ephemeral` have no strategy, and showing the
@@ -220,6 +393,185 @@
)
</script>
{#snippet scriptHeader()}
<!-- The script building this asset, folded into it: a strip across the top
naming it, which opens it. Several scripts open a menu to pick one. -->
{@const one = producer ?? (coProducers.length === 1 ? coProducers[0] : undefined)}
{@const headerClass = twMerge(
'flex items-center gap-1.5 w-full min-w-0 px-2 text-3xs font-normal leading-none text-secondary hover:bg-surface-hover hover:text-primary',
data.scriptError &&
'bg-red-100 dark:bg-red-700 text-red-700 dark:text-red-200 hover:bg-red-200 dark:hover:bg-red-600 hover:text-red-800'
)}
{#if one}
{@const isSelected = producer ? !!producer.selected : coSelected}
<button
type="button"
class={twMerge(headerClass, isSelected && HEADER_SELECTED)}
title={`Open ${one.summary ? `${one.summary} (${one.path})` : one.path}${one.unsaved ? ' · draft' : ''}`}
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
onclick={(e) => {
e.stopPropagation()
one.onOpen()
}}
>
{#if one.language}
<LanguageIcon lang={one.language} width={11} height={11} />
{:else}
<Code2 size={11} class="shrink-0" />
{/if}
<span class="truncate">{one.summary || one.path}</span>
{#if one.unsaved}<span class="shrink-0 text-tertiary">draft</span>{/if}
{#if producer?.runState}
<RunStateChip runState={producer.runState} class="ml-auto h-4 px-1 rounded" />
{/if}
<Pencil size={10} class={twMerge('shrink-0 opacity-70', !producer?.runState && 'ml-auto')} />
</button>
{:else if coProducers.length > 1}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="w-full min-w-0 flex"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<DropdownV2
items={coProducerItems}
placement="bottom-start"
bind:open={scriptMenuOpen}
fixedHeight={false}
usePointerDownOutside
enableFlyTransition
class="w-full"
>
{#snippet buttonReplacement()}
<span
class={twMerge(headerClass, 'h-full', (coSelected || scriptMenuOpen) && HEADER_SELECTED)}
title={`Built by ${coProducers.length} scripts: pick one to open`}
>
{#each coProducers.slice(0, 2) as sc (sc.path)}
{#if sc.language}
<LanguageIcon lang={sc.language} width={11} height={11} />
{:else}
<Code2 size={11} class="shrink-0" />
{/if}
{/each}
<span class="truncate">{coProducers.length} scripts</span>
<ChevronDown size={11} class="shrink-0 ml-auto" />
</span>
{/snippet}
</DropdownV2>
</div>
{/if}
{/snippet}
{#snippet upstreamRow()}
{#if data.foldedFixes?.length}
<!-- A folded script or trigger is misconfigured: its Fix, one at a time. -->
<NodeFixButton
chip
fix={data.foldedFixes[0]}
label={`${data.foldedFixes.length} error${data.foldedFixes.length === 1 ? '' : 's'} · Fix`}
/>
{:else if upstream && coTriggers.length > 1 && !coTriggers.some((t) => t.onOpen)}
<!-- Several triggers, none editable here (View mode): a plain label. -->
<span
class="truncate rounded-md border px-1.5 py-1 text-3xs leading-none font-normal border-gray-300 dark:border-gray-600 text-secondary"
>Multiple triggers</span
>
{:else if upstream && coTriggers.length > 1}
<!-- Several triggers start what builds this asset: one chip, a menu to pick
the one to edit. -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="min-w-0 flex"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<DropdownV2
items={coTriggerItems}
placement="bottom-start"
bind:open={triggerMenuOpen}
fixedHeight={false}
usePointerDownOutside
enableFlyTransition
>
{#snippet buttonReplacement()}
<span
class={twMerge(
'flex items-center gap-1 min-w-0 rounded-md border px-1.5 py-1 text-3xs leading-none font-normal border-gray-300 dark:border-gray-600 text-secondary bg-surface hover:bg-surface-hover cursor-pointer',
(triggerMenuOpen || coTriggers.some((t) => t.selected)) && CHIP_SELECTED
)}
title="Started by several triggers: pick one to edit"
>
<span class="truncate">Multiple triggers</span>
<ChevronDown size={9} class="shrink-0 opacity-70" />
</span>
{/snippet}
</DropdownV2>
</div>
{:else if upstream}
{@const trigger = producer?.trigger ?? coTriggers[0]}
{@const triggerScript = producer?.path ?? coTriggers[0]?.script}
{@const chipClass = twMerge(
'truncate rounded-md border px-1.5 py-1 text-3xs leading-none font-normal',
trigger?.missing
? 'border-red-300 dark:border-red-600 text-red-700 dark:text-red-300'
: 'border-gray-300 dark:border-gray-600 text-secondary',
trigger?.draft && 'border-dashed',
trigger?.onOpen && 'bg-surface hover:bg-surface-hover cursor-pointer',
trigger?.selected && CHIP_SELECTED
)}
{#if trigger?.kind === 'data_upload' && trigger.onOpen}
<!-- The one trigger that needs the user: nothing runs until a file is
provided, so it reads as the node's main action rather than a chip. -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<span
class="flex min-w-0"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<Button
variant="accent"
unifiedSize="sm"
startIcon={{ icon: Upload }}
title={`Upload the data ${triggerScript} runs on`}
onClick={(e) => {
e?.stopPropagation()
trigger.onOpen?.()
}}
>
Upload data
</Button>
</span>
{:else if trigger?.onOpen}
<button
type="button"
class={twMerge(chipClass, 'flex items-center gap-1 min-w-0')}
title={trigger.missing
? `No ${trigger.label} trigger targets ${triggerScript} yet: click to create one`
: `Edit the ${trigger.label} trigger`}
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
onclick={(e) => {
e.stopPropagation()
trigger.onOpen?.()
}}
>
<span class="truncate">{trigger.label}{trigger.draft ? ' · draft' : ''}</span>
{#if !trigger.missing}<Pencil size={9} class="shrink-0 opacity-70" />{/if}
</button>
{:else}
<span class={chipClass}>
{#if 'none' in upstream}
External source
{:else if trigger}
{trigger.label}{trigger.draft ? ' · draft' : ''}
{/if}
</span>
{/if}
{/if}
{/snippet}
<!-- onmouseenter/leave on the wrapper (not the inner card) so the run
button — which floats outside the card — keeps the hover state alive
when the cursor moves between the card and the button. -->
@@ -236,123 +588,169 @@
<div class="wm-recompute-flash pointer-events-none absolute inset-0 z-10 rounded-md"></div>
{/key}
{/if}
<!-- Mirrors the flow editor's asset pill: quiet surface + gray border at
rest, accent reserved for the selected state. -->
<div
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden border',
'bg-surface-secondary border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500 transition-colors',
selected && 'bg-surface-accent-selected border-border-selected'
<NodeActionsMenu
items={menuItems}
hover={hovered}
kebabClass={twMerge(
showGuardBadge && '-right-8',
hasHeader && !data.bannerOnly && CARD_CORNER_TOP
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={data.path}
>
<!-- Data identity carries the accent (luminance blue), pairing with
the blue write edges that produce these assets — scripts stay
neutral, so script vs data reads at a glance. -->
<AssetGenericIcon
assetKind={data.asset_kind}
class={`shrink-0 ml-2 mr-2 ${selected ? 'text-accent' : 'text-blue-600 dark:text-blue-400'}`}
size="14px"
/>
{#if data.runStatus}
<!-- The run in view, per relation: a spinner while its producer is
building it, then its outcome. Left of the name so the eye finds
the moving nodes first on a wide graph. -->
<span
class="shrink-0 mr-1 {data.runStatus === 'failed'
? 'text-red-600 dark:text-red-400'
: data.runStatus === 'materialized'
? 'text-green-600 dark:text-green-400'
: 'text-blue-600 dark:text-blue-400'}"
title={data.runStatus}
{#if data.bannerOnly}
<!-- A script that builds nothing and runs only on writes to what it reads:
the incoming edges say how it starts, so its banner is the whole node. -->
<div
class={twMerge(
'flex items-stretch min-w-0 rounded-md overflow-hidden bg-surface border border-gray-200 dark:border-gray-700 drop-shadow-sm',
(producer ? producer.selected : coSelected) &&
'border-border-selected dark:border-border-selected'
)}
style="width: {data.width ?? 160}px; height: {PIPELINE_NODE_HEADER}px;"
>
{#if data.runStatus === 'running'}
<Loader2 size={11} class="animate-spin" />
{:else if data.runStatus === 'failed'}
<XCircle size={11} />
{@render scriptHeader()}
</div>
{:else}
<PipelineNodeCard
kindLabel={data.noAsset ? 'No asset' : formatAssetKind(asset)}
title={data.noAsset ? (data.noAssetName ?? '') : formatShortAssetPath(asset)}
tooltip={data.noAsset
? 'This script builds no asset'
: data.error
? `${data.path}: ${data.error}`
: data.path}
{selected}
tone={data.error ? 'error' : undefined}
subtitle={data.foldedFixes?.length ||
(upstream &&
!producer?.trigger?.hidden &&
!('triggers' in upstream && upstream.triggers.length === 0))
? upstreamRow
: undefined}
header={hasHeader ? scriptHeader : undefined}
headerTone={data.scriptError ? 'error' : undefined}
headerSelected={producer ? !!producer.selected : coSelected}
width={data.width}
surface={upstream ? 'primary' : 'secondary'}
>
{#snippet icon()}
<!-- Data identity carries the accent (luminance blue), pairing with
the blue write edges that produce these assets — scripts stay
neutral, so script vs data reads at a glance. -->
{#if data.noAsset}
<CircleSlash size={14} class="text-tertiary" />
{:else}
<CheckCircle2 size={11} />
<AssetGenericIcon
assetKind={data.asset_kind}
class={data.error
? 'text-red-600 dark:text-red-200'
: selected
? 'text-accent'
: 'text-blue-600 dark:text-blue-400'}
size="14px"
/>
{/if}
</span>
<!-- Rows the run wrote. Recorded per relation already, and the cheapest
answer to "did this model actually produce anything" — a model that
built green but emitted 0 rows is the failure that looks like a
success. Only once settled: mid-build the number is not yet real. -->
{#if data.runStatus === 'materialized' && data.runRowCount != undefined}
<span
class="shrink-0 mr-1 text-3xs tabular-nums text-tertiary"
title="{data.runRowCount} rows written by this run"
>
{Intl.NumberFormat().format(data.runRowCount)}
</span>
{/if}
{/if}
<span class="flex-1 min-w-0 pr-1 py-0.5 text-2xs font-mono text-emphasis truncate">
{formatShortAssetPath(asset)}
</span>
<!-- Fork data-environment chip: in a fork every asset shares its parent's
name, so the env it resolves to must read at a glance. Labeled + tinted
(amber "parent" = deferred read of the parent's current table via a
view; emerald "fork" = the fork's own materialized copy) rather than a
bare icon, which was too easy to miss. The title carries the detail. -->
{#if data.fork_materialization === 'deferred'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300 border border-amber-300 dark:border-amber-700"
title="Deferred to parent workspace: reads the parent's current data. Materialize it in this fork to iterate on it."
>
<ArrowUpRight size={10} />
parent
</span>
{:else if data.fork_materialization === 'fork'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300 border border-emerald-300 dark:border-emerald-700"
title="Materialized in this fork: reads and writes use the fork's isolated copy."
>
<GitFork size={10} />
fork
</span>
{/if}
<!-- dbt chip: names the materialization dbt declares, plus a test count
when the model carries generic tests. Orange keeps it visually
separate from the fork/SCD2 chips, which describe Windmill state.
Where the project node is on the graph it also stands in for the edge
to it: hovering lights that node up, clicking selects it. Where it is
not — the run page, and a pipeline holding a dbt project — there is
nothing to point at, so the chip renders inert. -->
{#if data.dbt}
<button
type="button"
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold tracking-wide bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300 border border-orange-300 dark:border-orange-700 {data.onDbtSelect
? 'hover:brightness-95 cursor-pointer'
: 'cursor-default'}"
title={dbtTitle}
onmouseenter={() => data.onDbtHover?.(true)}
onmouseleave={() => data.onDbtHover?.(false)}
onclick={(e) => {
e.stopPropagation()
data.onDbtSelect?.()
}}
>
<DbtIcon width={9} height={9} />
{dbtLabel}
{#if data.dbt.data_tests?.length}
<span class="opacity-70">&middot; {data.dbt.data_tests.length}T</span>
{/snippet}
{#snippet leading()}
{#if data.runStatus}
<!-- The run in view, per relation: a spinner while its producer is
building it, then its outcome. Left of the name so the eye finds
the moving nodes first on a wide graph. -->
<span
class="shrink-0 mr-1 {data.runStatus === 'failed'
? 'text-red-600 dark:text-red-400'
: data.runStatus === 'materialized'
? 'text-green-600 dark:text-green-400'
: 'text-blue-600 dark:text-blue-400'}"
title={data.runStatus}
>
{#if data.runStatus === 'running'}
<Loader2 size={11} class="animate-spin" />
{:else if data.runStatus === 'failed'}
<XCircle size={11} />
{:else}
<CheckCircle2 size={11} />
{/if}
</span>
<!-- Rows the run wrote. Recorded per relation already, and the cheapest
answer to "did this model actually produce anything" — a model that
built green but emitted 0 rows is the failure that looks like a
success. Only once settled: mid-build the number is not yet real. -->
{#if data.runStatus === 'materialized' && data.runRowCount != undefined}
<span
class="shrink-0 mr-1 text-3xs tabular-nums text-tertiary"
title="{data.runRowCount} rows written by this run"
>
{Intl.NumberFormat().format(data.runRowCount)}
</span>
{/if}
{/if}
</button>
{/snippet}
{#snippet trailing()}
<!-- Fork data-environment chip: in a fork every asset shares its parent's
name, so the env it resolves to must read at a glance. Labeled + tinted
(amber "parent" = deferred read of the parent's current table via a
view; emerald "fork" = the fork's own materialized copy) rather than a
bare icon, which was too easy to miss. The title carries the detail. -->
{#if data.fork_materialization === 'deferred'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300 border border-amber-300 dark:border-amber-700"
title="Deferred to parent workspace: reads the parent's current data. Materialize it in this fork to iterate on it."
>
<ArrowUpRight size={10} />
parent
</span>
{:else if data.fork_materialization === 'fork'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300 border border-emerald-300 dark:border-emerald-700"
title="Materialized in this fork: reads and writes use the fork's isolated copy."
>
<GitFork size={10} />
fork
</span>
{/if}
<!-- dbt chip: names the materialization dbt declares, plus a test count
when the model carries generic tests. Orange keeps it visually
separate from the fork/SCD2 chips, which describe Windmill state.
Where the project node is on the graph it also stands in for the edge
to it: hovering lights that node up, clicking selects it. Where it is
not — the run page, and a pipeline holding a dbt project — there is
nothing to point at, so the chip renders inert. -->
{#if data.dbt}
<button
type="button"
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold tracking-wide bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300 border border-orange-300 dark:border-orange-700 {data.onDbtSelect
? 'hover:brightness-95 cursor-pointer'
: 'cursor-default'}"
title={dbtTitle}
onmouseenter={() => data.onDbtHover?.(true)}
onmouseleave={() => data.onDbtHover?.(false)}
onclick={(e) => {
e.stopPropagation()
data.onDbtSelect?.()
}}
>
<DbtIcon width={9} height={9} />
{dbtLabel}
{#if data.dbt.data_tests?.length}
<span class="opacity-70">&middot; {data.dbt.data_tests.length}T</span>
{/if}
</button>
{/if}
<!-- SCD2 companion marker: this node is the `<dim>_current` "latest row
per key" view its producer maintains alongside the base dimension.
Icon-only (the pill already truncates); the title names the base. -->
{#if data.derived_from}
<span
class="shrink-0 mr-1.5 text-violet-600 dark:text-violet-400"
title={`SCD2 current view of ${data.derived_from}: latest row per key, maintained by the same producer as the base dimension.`}
>
<History size={12} />
</span>
{/if}
{/snippet}
</PipelineNodeCard>
{/if}
<!-- SCD2 companion marker: this node is the `<dim>_current` "latest row
per key" view its producer maintains alongside the base dimension.
Icon-only (the pill already truncates); the title names the base. -->
{#if data.derived_from}
<span
class="shrink-0 mr-1.5 text-violet-600 dark:text-violet-400"
title={`SCD2 current view of ${data.derived_from}: latest row per key, maintained by the same producer as the base dimension.`}
>
<History size={12} />
</span>
{/if}
</div>
</NodeActionsMenu>
{#if showGuardBadge}
<!-- Data-test outcome badge. Floats off the TOP-RIGHT corner (opposite the
left-edge run button and the bottom + inserter) so it never collides
@@ -361,6 +759,7 @@
<div
class={twMerge(
'absolute -top-2 -right-2 z-10 rounded-full w-5 h-5 grid place-items-center border shadow-sm',
hasHeader && CARD_CORNER_TOP,
guardClass
)}
title={guardTitle}
@@ -376,7 +775,13 @@
cards. Drafts are runnable too via runScriptPreview, so no
greyed-out state — the page-supplied callback handles the
dispatch. -->
<div class="absolute -left-3 top-1/2 -translate-y-1/2 z-10">
<!-- Centred on the card, below the header tab when there is one. -->
<div
class={twMerge(
'absolute -left-3 top-1/2 -translate-y-1/2 z-10',
hasHeader && !data.bannerOnly && 'top-[calc(50%+11px)]'
)}
>
<button
type="button"
onclick={runProducers}
@@ -398,53 +803,47 @@
</button>
</div>
{/if}
{#if showAdd}
<!-- Always-visible + below the asset for downstream pipeline-script
creation. Half-overlapping the bottom edge so it visually attaches
to the node like the flow editor's between-step inserter. -->
{#if data.onStartOnDrag}
<NodeOnDot onStart={data.onStartOnDrag} visible={hovered} />
{/if}
{#if data.fix}
<!-- A red node's way out replaces its "+": adding downstream of a broken
node is not what it needs. -->
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<PipelineInsertMenu
kinds={[
{
id: 'pipeline_script',
label: 'Add downstream pipeline script',
description: 'Triggered when this asset changes',
icon: Code2
}
]}
languages={PIPELINE_LANGUAGES as any}
pathPrefix={data.pathPrefix ?? ''}
defaultPathSuffix={data.defaultPathSuffix ?? ''}
onPick={handlePick}
<NodeFixButton fix={data.fix} />
</div>
{:else if showAdd}
<!-- Always-visible + on the asset's bottom edge for downstream pipeline-
script creation, half-overlapping it like the flow editor's
between-step inserter. -->
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<AddDownstreamMenu
asset={{ kind: data.asset_kind, path: data.path }}
onAddScript={data.onAddScript!}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
>
{#snippet trigger()}
<!--
Sizing notes for the round + button:
- w-6/h-6 (24px) chosen so that with border-2 (2px each
side) the inner area is exactly 20px — divisible by
the 16px icon to leave a 2px gap on every side. At
20px / 12px before, the gap was 4px-each but the
odd rounding interacted badly with svelte-flow's
fractional zoom transforms and the icon drifted
half a pixel off-center on certain zoom levels.
- `grid place-items-center` instead of flex centering:
flex's baseline alignment introduces a sub-pixel
nudge on small elements that grid avoids.
- `leading-none` strips the default line-height
contribution that the SVG inherits via the parent's
text rendering, otherwise the icon is shifted
downward by ~0.5px at fractional zooms.
-->
{#snippet trigger({ open })}
<!-- A click opens the menu; a drag goes to the canvas, which
tells a real drag from a click by how far the pointer moves. -->
<button
type="button"
class="nopan nodrag block"
onclick={(e) => e.stopPropagation()}
class="bg-surface border border-gray-400 dark:border-gray-600 text-secondary hover:bg-surface-hover rounded-full w-6 h-6 grid place-items-center shadow-sm leading-none"
title="Add downstream pipeline script"
onpointerdown={(e) => {
if (e.button !== 0) return
e.preventDefault()
data.onStartFeedDrag?.(e)
}}
aria-label="Add downstream step"
title={data.onStartFeedDrag
? 'Click to add a downstream step, or drag onto a script to run it after each write'
: undefined}
>
<Plus size={16} strokeWidth={2.5} />
<AddDownstreamPill expanded={open} />
</button>
{/snippet}
</PipelineInsertMenu>
</AddDownstreamMenu>
</div>
{/if}
</div>
frontend/src/lib/components/assets/AssetGraph/DucklakeAssetPanel.svelte
+24 -20
View File
@@ -11,7 +11,8 @@
// can't be read), so a selectable version always exists in the table.
import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
import { Table2, History, Columns3 } from 'lucide-svelte'
import { Table2, History, Columns3, Database } from 'lucide-svelte'
import DBManagerContent from '$lib/components/DBManagerContent.svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { resource } from 'runed'
import { fetchDucklakeSnapshots } from '$lib/components/dbOps'
@@ -31,7 +32,7 @@
}
let { path, workspace, schemaCanEvolve = true }: Props = $props()
let tab = $state<'partitions' | 'history' | 'schema'>('partitions')
let tab = $state<'data' | 'partitions' | 'history' | 'schema'>('data')
// The user's explicit snapshot pick (undefined until they click a row).
let selectedVersion = $state<number | undefined>(undefined)
@@ -72,24 +73,28 @@
</script>
<div class="flex flex-col h-full">
{#if !qualifiedTable}
<!-- Catalog-level ducklake node (no table segment, e.g. `ducklake://main`):
snapshot history / time-travel are per-table, so only the partition
grid applies here. -->
<PartitionStatusGrid {path} {workspace} />
{:else}
<div class="flex items-center gap-2 px-3 py-2 border-b shrink-0">
<ToggleButtonGroup selected={tab} on:selected={(e) => (tab = e.detail)}>
{#snippet children({ item })}
<ToggleButton size="sm" value="partitions" label="Partitions" icon={Table2} {item} />
<div class="flex items-center gap-2 px-3 py-2 border-b shrink-0">
<ToggleButtonGroup selected={tab} on:selected={(e) => (tab = e.detail)}>
{#snippet children({ item })}
<ToggleButton size="sm" value="data" label="Data" icon={Database} {item} />
<ToggleButton size="sm" value="partitions" label="Partitions" icon={Table2} {item} />
<!-- Snapshot history and time-travel are per table: a catalog-level node
(`ducklake://main`, no table segment) has only its data and partitions. -->
{#if qualifiedTable}
<ToggleButton size="sm" value="schema" label="Schema" icon={Columns3} {item} />
<ToggleButton size="sm" value="history" label="History" icon={History} {item} />
{/snippet}
</ToggleButtonGroup>
</div>
{/if}
{/snippet}
</ToggleButtonGroup>
</div>
<div class="flex-1 min-h-0">
{#if tab === 'partitions'}
<div class="flex-1 min-h-0">
{#if tab === 'data' && parsed}
<!-- Keyed on the asset: the manager opens its table once, on mount. -->
{#key path}
<DBManagerContent input={parsed} {workspace} />
{/key}
{:else if tab === 'partitions' || !qualifiedTable}
<PartitionStatusGrid {path} {workspace} />
{:else if tab === 'schema'}
<SchemaHistoryPanel {path} {workspace} canEvolve={schemaCanEvolve} />
@@ -117,7 +122,6 @@
</Pane>
</Splitpanes>
</div>
{/if}
</div>
{/if}
{/if}
</div>
</div>
frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte
+3 -1
View File
@@ -25,6 +25,8 @@
if (!nodesInitialized.current) return
if (!nodes.some((n) => n.type !== 'add')) return
fittedFor = fitKey
void fitView({ padding: 0.1, maxZoom: 1 })
// A re-fit over nodes that keep their ids finds them already measured, one
// render before their new positions reach the flow.
requestAnimationFrame(() => void fitView({ padding: 0.1, maxZoom: 1 }))
})
</script>

Some files were not shown because too many files have changed in this diff Show More