starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-12 08:02:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'oidc-discovery-signing-alg' into oidc-external-signer

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
This commit is contained in:
Ruben Fiszel
2026-10-09 11:46:47 +02:00
parent 53449be8c7 7bed8a67e4
commit 7bfd7a266c
251 changed files with 13239 additions and 3326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217"
}
backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b"
}
backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json
Generated
+45
View File
@@ -0,0 +1,45 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e"
}
backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13"
}
backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json
Generated
-31
View File
@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d"
}
backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5"
}
backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d"
}
backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665"
}
backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc"
}
backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json
Generated
+43
View File
@@ -0,0 +1,43 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8"
}
backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd"
}
backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET draining = true WHERE worker = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919"
}
backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json
Generated
+34
View File
@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b"
}
backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323"
}
backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed"
}
backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json → backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -107,6 +107,11 @@
"ordinal": 20,
"name": "native_mode",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
@@ -138,8 +143,9 @@
true,
true,
true,
false,
false
]
},
"hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca"
"hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c"
}
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
2f901d28bb1923ac91a04f71d3fdcd8abf28c6db
f94aa566e18aacab3b012d6909b3e0d6d6996a9a
backend/migrations/20261008070335_worker_ping_draining.down.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining;
backend/migrations/20261008070335_worker_ping_draining.up.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE;
backend/summarized_schema.txt
+1 -1
View File
@@ -219,7 +219,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool)
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
backend/tests/error_handler.rs
+67
View File
@@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its failure must reach the workspace error handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_error_handler_triggered_on_hub_script_failure(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000002";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/fail"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(!completed_job.success, "Hub script should have failed");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace error handler should handle the failed hub script"
);
Ok(())
}
backend/tests/otel.rs
+26 -1
View File
@@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() {
otel_set_worker_uptime("w-uptime", 3600.0);
otel_set_health_status_phase("healthy");
otel_set_health_db_unresponsive(true);
otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024));
otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35);
otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25);
otel_record_job_memory_peak("python3", 100 * 1024 * 1024);
// ── Histograms ──────────────────────────────────────────────────
@@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() {
let metrics = flush_and_get_metrics(&state);
let names = metric_names(&metrics);
// ── Verify all 21 metric names are present ──────────────────────
// ── Verify all metric names are present ─────────────────────────
let expected = [
"windmill.queue.push_count",
@@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() {
"windmill.worker.uptime",
"windmill.health.status",
"windmill.health.db_unresponsive",
"windmill.worker.memory_usage",
"windmill.worker.windmill_memory_usage",
"windmill.worker.memory_limit",
"windmill.worker.cpu_usage",
"windmill.worker.cpu_periods",
"windmill.worker.cpu_throttled_periods",
"windmill.worker.cpu_throttled_time",
"windmill.worker.occupancy_rate",
"windmill.worker.job_memory_peak",
];
for name in expected {
assert!(
@@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() {
.expect("worker.busy data point with worker=worker-test-1 not found");
assert_eq!(dp.1, 1);
let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap();
let values = gauge_i64_values(m);
let dp = values
.iter()
.find(|(attrs, _)| {
attrs
.iter()
.any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu")
})
.expect("worker.memory_limit data point with worker_group=gpu not found");
assert_eq!(dp.1, 1024);
let m = find_metric(&metrics, "windmill.db.pool.active").unwrap();
assert_eq!(gauge_i64_values(m)[0].1, 5);
let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap();
backend/tests/success_handler.rs
+67
View File
@@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow:
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its success must reach the workspace success handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_triggered_on_hub_script_success(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000003";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/ok"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(completed_job.success, "Hub script should have succeeded");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace success handler should handle the successful hub script"
);
Ok(())
}
backend/tests/wac_suspend_started_at.rs
+88
View File
@@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool<Postgres>) -> anyhow::Res
Ok(())
}
/// Each park clears `started_at`, so the completion would otherwise store only the last
/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that
/// started an hour late. The completed row spans the workflow from its first start, and
/// keeps the last segment aside: that is what is metered and what worker-time readers sum.
#[sqlx::test]
async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let job_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \
permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \
VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \
'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)",
)
.bind(job_id)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \
VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')",
)
.bind(job_id)
.execute(&db)
.await?;
// Two parks, each followed by the pull re-stamping `started_at`: the second park must
// not replace the first start with its own segment's.
for resumed_ago in ["10 minutes", "3 seconds"] {
let mut tx = db.begin().await?;
suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?;
tx.commit().await?;
sqlx::query(
"UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \
WHERE id = $1",
)
.bind(job_id)
.bind(resumed_ago)
.execute(&db)
.await?;
}
let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db)
.await?
.unwrap();
let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?;
let (_, span) = windmill_queue::add_completed_job(
&db,
&job,
true,
false,
sqlx::types::Json(&result),
None,
0,
None,
false,
Some(3000),
false,
)
.await?;
let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option<i64>) = sqlx::query_as(
"SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \
(extras->>'wac_last_segment_ms')::bigint \
FROM v2_job_completed WHERE id = $1",
)
.bind(job_id)
.fetch_one(&db)
.await?;
assert!(
(started_ago_s - 3600.0).abs() < 60.0,
"started_at must be the first start, got {started_ago_s}s ago"
);
assert!(
(duration_ms - 3_600_000).abs() < 60_000,
"duration_ms must be the wall time since the first start, got {duration_ms}"
);
assert_eq!(last_segment_ms, Some(3000));
assert_eq!(
span.duration_ms, duration_ms,
"a flow records the step with the span the row holds"
);
assert!(span.outlasts_run(job.started_at));
assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter");
Ok(())
}
backend/tests/worker_ping_ip.rs
+31 -1
View File
@@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres};
use uuid::Uuid;
use windmill_common::{
external_ip::UNKNOWN_IP,
worker::{insert_ping_query, update_worker_ping_main_loop_query},
worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query},
};
async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> anyhow::Result<()> {
register(db, worker, ip, None).await
}
async fn register(
db: &Pool<Postgres>,
worker: &str,
ip: Option<&str>,
draining: Option<bool>,
) -> anyhow::Result<()> {
insert_ping_query(
"test-instance",
worker,
@@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool<Postgres>, worker: &str, ip: Option<&str>) -> any
None,
None,
false,
draining,
db,
)
.await?;
Ok(())
}
/// A worker re-registers mid-life once its external IP resolves. That registration must not
/// clear a draining flag the same process already published; only a worker starting up under
/// a reclaimed name resets it.
#[sqlx::test]
async fn only_a_starting_worker_resets_draining(db: Pool<Postgres>) -> anyhow::Result<()> {
let draining = || async {
sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'")
.fetch_one(&db)
.await
};
register(&db, "wk-drain", None, Some(false)).await?;
set_worker_draining_query("wk-drain", &db).await?;
register(&db, "wk-drain", Some("1.2.3.4"), None).await?;
assert!(draining().await?);
register(&db, "wk-drain", None, Some(false)).await?;
assert!(!draining().await?);
Ok(())
}
/// The external IP resolves in the background, so the initial ping often has none yet. That must
/// not blank the address a previous process wrote to the row this one reclaims — worker names are
/// stable across restarts under EXIT_AFTER_N_JOBS.
backend/windmill-api-auth/src/scopes.rs
+1
View File
@@ -996,6 +996,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui
["job_metrics", "set_progress", ..] => (http_method == "POST", 2),
["job_metrics", "get_progress", ..] => (http_method == "GET", 2),
["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2),
["jobs", "worker_is_draining", ..] => (http_method == "GET", 2),
["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3),
["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3),
_ => return false,
backend/windmill-api-integration-tests/tests/jobs_authed.rs
+48
View File
@@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue(
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_worker_is_draining_follows_the_job_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job = Uuid::new_v4();
sqlx::query(
"INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args)
VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)",
)
.bind(job)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker)
VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')",
)
.bind(job)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version)
VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')",
)
.execute(&db)
.await?;
let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}");
let draining = || async {
let resp = authed(client().get(&url)).send().await?;
anyhow::Ok(resp.json::<bool>().await?)
};
assert!(!draining().await?);
windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?;
assert!(draining().await?);
// A parked job keeps its last worker on the queue row without running on it.
sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1")
.bind(job)
.execute(&db)
.await?;
assert!(!draining().await?);
Ok(())
}
backend/windmill-api-settings/src/lib.rs
+17 -17
View File
@@ -58,10 +58,10 @@ use windmill_common::{
error::{self, pg_error_message, JsonResult, Result},
get_database_url,
global_settings::{
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING,
AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXTERNAL_INSTANCE_PG_SETTING,
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING,
APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -1238,26 +1238,22 @@ async fn run_setting_pre_write_hook(
}
}
}
GITHUB_APP_WEBHOOK_BASE_URL_SETTING => {
// A bad value here yields a webhook GitHub can never deliver to, and the
GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => {
// A bad webhook base yields a webhook GitHub can never deliver to, and the
// failure only shows up much later as "falling back to polling" on a
// repository — so reject it at the boundary instead.
// repository; a bad API base puts an unreachable URL in front of every user.
// So reject both at the boundary instead.
match value {
// Clearing (delete row) is handled by the caller; allow it through.
serde_json::Value::Null => {}
serde_json::Value::String(s) if s.trim().is_empty() => {}
serde_json::Value::String(s) => {
windmill_common::global_settings::validate_webhook_base_url(s).map_err(
|e| {
error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}"
))
},
)?;
windmill_common::global_settings::validate_webhook_base_url(s)
.map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?;
}
_ => {
return Err(error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string"
"{key} must be a URL string"
)));
}
}
@@ -1394,8 +1390,12 @@ async fn set_instance_config(
// Applied and audited one setting at a time: the writes are not transactional, so a
// batch that fails midway must not leave the settings it did write unrecorded.
let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } =
settings_diff;
let instance_config::SettingsDiff {
upserts,
deletes,
mut previous_values,
unchanged_count,
} = settings_diff;
let steps = upserts
.into_iter()
.map(|(k, v)| (k, Some(v)))
backend/windmill-api-users/src/users.rs
+1 -1
View File
@@ -490,7 +490,7 @@ async fn list_user_usage(
"
SELECT usr.email, usage.executions
FROM usr, LATERAL (
SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions
SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions
FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.workspace_id = $1
AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')
backend/windmill-api-workers/src/lib.rs
+2 -1
View File
@@ -86,6 +86,7 @@ struct WorkerPing {
job_isolation: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
native_mode: Option<bool>,
draining: bool,
}
// #[derive(Serialize, Deserialize)]
@@ -118,7 +119,7 @@ async fn list_worker_pings(
WorkerPing,
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,
custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode
custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining
FROM worker_ping
WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
ORDER BY ping_at desc LIMIT $2 OFFSET $3",
backend/windmill-api/openapi.yaml
+27 -2
View File
@@ -1919,7 +1919,7 @@ paths:
/settings/instance_ui:
get:
summary: get the instance-wide announcement banner and accent color
summary: get the instance-wide announcement banner, accent color and API base url
description: |
Readable by any authenticated user; every page load reads it. Each value
is the raw stored setting, or null when unset.
@@ -1928,7 +1928,7 @@ paths:
- setting
responses:
"200":
description: the instance_banner and accent_color settings
description: the instance_banner, accent_color and api_base_url settings
content:
application/json:
schema:
@@ -1936,6 +1936,7 @@ paths:
properties:
instance_banner: {}
accent_color: {}
api_base_url: {}
/settings_u/ruff_config:
get:
@@ -17382,6 +17383,27 @@ paths:
schema:
type: string
/w/{workspace}/jobs/worker_is_draining/{id}:
get:
summary: whether the worker running this job is draining
description: |
True once the worker running the job has received its shutdown signal. The worker
still waits for the job to finish, so a long-running job can poll this to exit early.
False when the job is not currently running on a worker.
operationId: workerIsDraining
tags:
- job
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/JobId"
responses:
"200":
description: whether the worker running this job is draining
content:
application/json:
schema:
type: boolean
/w/{workspace}/jobs_u/get_logs/{id}:
get:
summary: get job logs
@@ -35961,6 +35983,9 @@ components:
type: string
native_mode:
type: boolean
draining:
type: boolean
description: the worker received its shutdown signal and exits once its current job is done
required:
- worker
- worker_instance
backend/windmill-api/src/jobs.rs
+51
View File
@@ -378,6 +378,10 @@ pub fn workspaced_service() -> Router {
"/wac_approval_urls/{job_id}/{step_key}",
get(get_wac_approval_urls).layer(cors.clone()),
)
.route(
"/worker_is_draining/{id}",
get(worker_is_draining).layer(cors.clone()),
)
.route(
"/result_by_id/{job_id}/{node_id}",
get(get_result_by_id).layer(cors.clone()),
@@ -613,6 +617,53 @@ async fn get_root_job(
Ok(Json(res))
}
/// Whether the worker running this job has received its shutdown signal. A draining worker
/// never interrupts its job, so a long-running script polls this to exit on its own terms.
async fn worker_is_draining(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> windmill_common::error::JsonResult<bool> {
// A job always may ask about itself. Anyone else must be able to see the job: the
// caller's row policies on v2_job decide, narrowed by the token's tag filter.
let draining = if authed.job_id == Some(id) {
sqlx::query_scalar!(
"SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker
WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
id,
w_id
)
.fetch_optional(&db)
.await?
} else {
let tags = get_scope_tags(&authed)
.map(|tags| tags.into_iter().map(str::to_string).collect::<Vec<_>>());
let mut tx = user_db.begin(&authed).await?;
let draining = sqlx::query_scalar!(
"SELECT wp.draining FROM v2_job j
JOIN v2_job_queue q ON q.id = j.id
JOIN worker_ping wp ON wp.worker = q.worker
WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL
AND ($3::text[] IS NULL OR j.tag = ANY($3))",
id,
w_id,
tags.as_deref()
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
draining
}
.unwrap_or(false);
windmill_common::feature_usage::log_feature_usage(
"worker_draining",
"check",
if draining { "draining" } else { "not_draining" },
);
Ok(Json(draining))
}
async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result<String> {
let root_job = sqlx::query_scalar!(
r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#,
backend/windmill-common/src/global_settings.rs
+34 -7
View File
@@ -181,6 +181,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app";
/// `base_url` when unset; set it when the browser-facing URL is not reachable
/// from GitHub and a separate ingress fronts the API for inbound webhooks.
pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url";
/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP
/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on.
/// For deployments that serve the API on a separate domain or behind a gateway.
/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]:
/// every user is shown URLs built from it.
pub const API_BASE_URL_SETTING: &str = "api_base_url";
/// Instance-wide announcement rendered above every page of the app (maintenance
/// windows, incidents). Readable by any authenticated user, unlike most settings:
/// the banner exists to be shown to everyone, so it must never hold anything the
@@ -313,12 +319,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> {
pub struct InstanceUi {
pub instance_banner: Option<serde_json::Value>,
pub accent_color: Option<serde_json::Value>,
pub api_base_url: Option<serde_json::Value>,
}
pub async fn get_instance_ui(db: &Pool<Postgres>) -> error::Result<InstanceUi> {
let rows = sqlx::query!(
"SELECT name, value FROM global_settings WHERE name = ANY($1)",
&[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str]
&[
INSTANCE_BANNER_SETTING,
ACCENT_COLOR_SETTING,
API_BASE_URL_SETTING
] as &[&str]
)
.fetch_all(db)
.await?;
@@ -327,17 +338,22 @@ pub async fn get_instance_ui(db: &Pool<Postgres>) -> error::Result<InstanceUi> {
match row.name.as_str() {
INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value),
ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value),
API_BASE_URL_SETTING => ui.api_base_url = Some(row.value),
_ => {}
}
}
Ok(ui)
}
/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value.
const SHELL_ACTIVE_CHARS: [char; 16] = [
'$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!',
];
/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value.
///
/// The receiver path is appended to it verbatim, so anything that doesn't
/// concatenate into a URL GitHub can POST to must be rejected at write time
/// rather than silently producing an unreachable hook: a wrong scheme
/// A path is appended to it verbatim, so anything that doesn't concatenate into a
/// URL an external caller can reach must be rejected at write time rather than
/// silently producing an unreachable hook or endpoint: a wrong scheme
/// (`httpss://`), a missing host, embedded whitespace, or a query/fragment
/// (appending a path after `?`/`#` keeps it inside the query/fragment).
///
@@ -353,7 +369,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> {
url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?;
if !url.username().is_empty() || url.password().is_some() {
return Err(
"must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(),
"must not embed a username or password: the URL is readable by users who are not instance admins".to_string(),
);
}
if !matches!(url.scheme(), "http" | "https") {
@@ -366,13 +382,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> {
}
if url.query().is_some() || url.fragment().is_some() {
return Err(
"must not include a query string or fragment, since the webhook path is appended to it"
"must not include a query string or fragment, since a path is appended to it"
.to_string(),
);
}
if value.chars().any(char::is_whitespace) {
return Err("must not contain whitespace".to_string());
}
// The UI pastes the URL into shell commands users copy, where double quotes
// do not stop `$(...)` or backticks from running.
if value.contains(SHELL_ACTIVE_CHARS) {
return Err(
"must not contain shell metacharacters, since it is shown inside commands users copy"
.to_string(),
);
}
// Matches the sibling `base_url` / `hub_base_url` convention. The receiver
// builder trims it anyway, so this is about keeping the stored value canonical
// rather than about reachability.
@@ -1101,6 +1125,9 @@ mod tests {
"https://hooks.example.com#",
"https://user:password@hooks.example.com",
"https://user@hooks.example.com",
"https://hooks.example.com/$(uname)",
"https://hooks.example.com/`uname`",
"https://hooks.example.com/a;b",
];
for v in accept {
assert!(
backend/windmill-common/src/instance_config.rs
+33 -25
View File
@@ -281,6 +281,8 @@ pub struct GlobalSettings {
#[serde(skip_serializing_if = "Option::is_none")]
pub github_app_webhook_base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub api_base_url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub email_domain: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hub_base_url: Option<String>,
@@ -1379,8 +1381,8 @@ pub fn diff_worker_configs(
}
/// Declaratively replace the global settings, rejecting a
/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the
/// API would reject.
/// `github_app_webhook_base_url`, `api_base_url` or
/// `http_route_default_allowed_origins` the API would reject.
///
/// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's
/// ConfigMap sync) MUST go through this rather than calling
@@ -1403,29 +1405,35 @@ pub async fn sync_global_settings_declarative(
current: &BTreeMap<String, serde_json::Value>,
desired: &BTreeMap<String, serde_json::Value>,
) -> anyhow::Result<()> {
let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING;
// Non-string shapes are rejected rather than ignored: `as_str()` alone would let a
// bool/number/object through as if the key were absent, and the diff below would
// then persist it — where the HTTP path answers "must be a URL string".
match desired.get(webhook_key) {
None | Some(serde_json::Value::Null) => {}
Some(serde_json::Value::String(s)) if s.trim().is_empty() => {}
Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s)
.map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?,
// Names the JSON kind rather than printing it: this is the last message on
// this path that could report submitted content, and an object or array could
// carry a secret into `sync-config` output and operator logs.
Some(other) => {
let kind = match other {
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
_ => "a non-string value",
};
return Err(anyhow::anyhow!(
"{webhook_key} must be a URL string, got {kind}"
));
for url_key in [
crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING,
crate::global_settings::API_BASE_URL_SETTING,
] {
// Non-string shapes are rejected rather than ignored: `as_str()` alone would let a
// bool/number/object through as if the key were absent, and the diff below would
// then persist it — where the HTTP path answers "must be a URL string".
match desired.get(url_key) {
None | Some(serde_json::Value::Null) => {}
Some(serde_json::Value::String(s)) if s.trim().is_empty() => {}
Some(serde_json::Value::String(s)) => {
crate::global_settings::validate_webhook_base_url(s)
.map_err(|e| anyhow::anyhow!("{url_key}: {e}"))?
}
// Names the JSON kind rather than printing it: this is the last message on
// this path that could report submitted content, and an object or array could
// carry a secret into `sync-config` output and operator logs.
Some(other) => {
let kind = match other {
serde_json::Value::Bool(_) => "a boolean",
serde_json::Value::Number(_) => "a number",
serde_json::Value::Array(_) => "an array",
serde_json::Value::Object(_) => "an object",
_ => "a non-string value",
};
return Err(anyhow::anyhow!(
"{url_key} must be a URL string, got {kind}"
));
}
}
}
backend/windmill-common/src/otel_oss.rs
+33
View File
@@ -132,3 +132,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {}
#[cfg(not(feature = "private"))]
pub fn otel_set_worker_memory(
_worker: &str,
_worker_group: &str,
_usage: Option<i64>,
_windmill_usage: Option<i64>,
_limit: Option<i64>,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_add_worker_cpu(
_worker: &str,
_worker_group: &str,
_usage_secs: f64,
_periods: u64,
_throttled_periods: u64,
_throttled_secs: f64,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_set_worker_occupancy_rate(
_worker: &str,
_worker_group: &str,
_window: &'static str,
_rate: f64,
) {
}
#[cfg(not(feature = "private"))]
pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {}
backend/windmill-common/src/variables.rs
+34 -8
View File
@@ -14,7 +14,7 @@ use crate::utils::WarnAfterExt;
use crate::worker::Connection;
use crate::{worker::WORKER_GROUP, BASE_URL, DB};
use chrono::{SecondsFormat, Utc};
use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait};
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
@@ -375,13 +375,15 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String {
mc.encrypt_str_to_base64(value)
}
const DECRYPT_FAILED: &str =
"Could not decrypt value. The value may have been encrypted with a different key.";
/// Every failure returns the same message so responses carry no signal about why decryption
/// failed (padding, base64, UTF-8); the cause is only logged.
pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result<String> {
mc.decrypt_base64_to_string(value).map_err(|e| match e {
MagicCryptError::DecryptError(_) => error::Error::internal_err(
"Could not decrypt value. The value may have been encrypted with a different key."
.to_string(),
),
_ => error::Error::internal_err(e.to_string()),
mc.decrypt_base64_to_string(value).map_err(|e| {
tracing::warn!("decryption failed: {e}");
error::Error::internal_err(DECRYPT_FAILED.to_string())
})
}
@@ -746,7 +748,31 @@ pub async fn get_variable_or_self_as<T: Authable + Sync>(
#[cfg(test)]
mod tests {
use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier};
use super::{
can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier,
DECRYPT_FAILED,
};
use crate::error::Error;
use magic_crypt::MagicCryptTrait;
#[test]
fn decrypt_failures_all_return_the_same_message() {
let mc = magic_crypt::new_magic_crypt!("workspace-key", 256);
let other = magic_crypt::new_magic_crypt!("another-key", 256);
let failures = [
"not base64 at all!".to_string(),
mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]),
other.encrypt_str_to_base64("secret"),
];
for value in failures {
match decrypt(&mc, value.clone()) {
Err(Error::InternalErrLoc { message, .. }) => {
assert_eq!(message, DECRYPT_FAILED, "for {value:?}")
}
other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"),
}
}
}
#[test]
fn is_valid_js_identifier_gates_prologue_injection() {
backend/windmill-common/src/worker.rs
+115 -2
View File
@@ -1783,6 +1783,73 @@ pub fn get_cpu_period() -> Option<i64> {
Some(100000)
}
/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job
/// peak-memory histograms so the two exports stay comparable.
pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = {
let mut buckets = [0.0; 12];
let mut i = 0;
while i < 12 {
buckets[i] = (16u64 << (20 + i)) as f64;
i += 1;
}
buckets
};
/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0
/// when the cgroup has no CPU controller, which is also what an unthrottled one reports.
#[derive(Debug, Clone, Copy, Default, PartialEq)]
pub struct CpuStat {
pub usage_usec: u64,
pub nr_periods: u64,
pub nr_throttled: u64,
pub throttled_usec: u64,
}
fn cpu_stat_field(stat: &str, key: &str) -> Option<u64> {
stat.lines().find_map(|line| {
let (k, v) = line.split_once(' ')?;
(k == key).then(|| v.trim().parse().ok()).flatten()
})
}
fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option<CpuStat> {
Some(CpuStat {
usage_usec: cpu_stat_field(stat, "usage_usec")?,
nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0),
nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0),
throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0),
})
}
/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds.
fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option<CpuStat> {
Some(CpuStat {
usage_usec: cpuacct_usage.trim().parse::<u64>().ok()? / 1000,
nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0),
nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0),
throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000,
})
}
#[cfg(not(windows))]
pub fn get_cpu_stat() -> Option<CpuStat> {
const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage";
if Path::new(V1_USAGE).exists() {
parse_cgroup_v1_cpu_stat(
&std::fs::read_to_string(V1_USAGE).ok()?,
&std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(),
)
} else {
let cgroup_path = get_cgroupv2_path()?;
parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?)
}
}
#[cfg(windows)]
pub fn get_cpu_stat() -> Option<CpuStat> {
None
}
/// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count
/// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds
/// the quota in, so a fraction of a CPU makes it report a single-core machine.
@@ -1952,6 +2019,7 @@ pub enum PingType {
MainLoop,
Job,
InitScript,
Draining,
}
#[derive(Serialize, Deserialize)]
pub struct Ping {
@@ -1974,6 +2042,8 @@ pub struct Ping {
pub occupancy_rate_30m: Option<f32>,
pub job_isolation: Option<String>,
pub native_mode: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub draining: Option<bool>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
@@ -2026,6 +2096,7 @@ pub async fn update_ping_http(
insert_ping.memory,
insert_ping.job_isolation,
insert_ping.native_mode.unwrap_or(false),
insert_ping.draining,
db,
)
.await?;
@@ -2054,10 +2125,21 @@ pub async fn update_ping_http(
)
.await?
}
PingType::Draining => set_worker_draining_query(worker_name, db).await?,
}
Ok(())
}
pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET draining = true WHERE worker = $1",
worker_name
)
.execute(db)
.await?;
Ok(())
}
#[derive(Debug, Serialize, Deserialize)]
pub struct JobCancelled {
pub canceled_by: String,
@@ -2156,6 +2238,9 @@ pub async fn fetch_raw_script_from_app_query(
/// `started_at` and `jobs_executed` are the only two columns carried over, being the
/// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP
/// lookup has not resolved yet and the predecessor's address is still the best guess.
///
/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering
/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime.
pub async fn insert_ping_query(
worker_instance: &str,
worker_name: &str,
@@ -2169,6 +2254,7 @@ pub async fn insert_ping_query(
memory: Option<i64>,
job_isolation: Option<String>,
native_mode: bool,
draining: Option<bool>,
db: &DB,
) -> anyhow::Result<i32> {
// A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and
@@ -2176,8 +2262,8 @@ pub async fn insert_ping_query(
// lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The
// literal below must stay equal to `external_ip::UNKNOWN_IP`.
let previous_jobs_executed = sqlx::query_scalar!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)
DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)
DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)
RETURNING jobs_executed",
worker_instance,
worker_name,
@@ -2191,6 +2277,7 @@ pub async fn insert_ping_query(
memory,
job_isolation.as_deref(),
native_mode,
draining,
)
.fetch_one(db)
.await?;
@@ -2899,6 +2986,32 @@ mod tests {
use super::*;
use std::collections::HashMap;
#[test]
fn cgroup_cpu_stat_is_normalized_to_microseconds() {
let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\
nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n";
let expected = CpuStat {
usage_usec: 2_500_000,
nr_periods: 40,
nr_throttled: 7,
throttled_usec: 350_000,
};
assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected));
let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n";
assert_eq!(
parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat),
Some(expected)
);
// No CPU controller: usage is still reported, throttling reads as zero.
assert_eq!(
parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"),
Some(CpuStat { usage_usec: 12, ..Default::default() })
);
assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None);
}
#[test]
fn datatable_role_is_read_from_the_leading_comment_block() {
let role = |code| SqlAnnotations::datatable_role(code);
backend/windmill-queue/src/jobs.rs
+100 -32
View File
@@ -1221,7 +1221,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
worker_name: &str,
flow_is_done: bool,
duration: Option<i64>,
) -> Result<(), Error> {
) -> Result<CompletedSpan, Error> {
record_failure_metrics(completed_job, worker_name).await;
tracing::error!(
@@ -1229,7 +1229,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
completed_job.id,
completed_job.workspace_id,
);
let _ = add_completed_job(
let (_, span) = add_completed_job(
db,
completed_job,
false,
@@ -1244,7 +1244,7 @@ pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + V
)
.warn_after_seconds(10)
.await?;
Ok(())
Ok(span)
}
pub async fn add_completed_job_error(
@@ -1257,6 +1257,31 @@ pub async fn add_completed_job_error(
flow_is_done: bool,
duration: Option<i64>,
) -> Result<WrappedError, Error> {
add_completed_job_error_with_span(
db,
completed_job,
mem_peak,
canceled_by,
e,
worker_name,
flow_is_done,
duration,
)
.await
.map(|(wrapped, _)| wrapped)
}
/// `add_completed_job_error`, also handing back what the completed row holds.
pub async fn add_completed_job_error_with_span(
db: &Pool<Postgres>,
completed_job: &MiniCompletedJob,
mem_peak: i32,
canceled_by: Option<CanceledBy>,
e: serde_json::Value,
worker_name: &str,
flow_is_done: bool,
duration: Option<i64>,
) -> Result<(WrappedError, CompletedSpan), Error> {
record_failure_metrics(completed_job, worker_name).await;
let result = WrappedError { error: e };
@@ -1266,7 +1291,7 @@ pub async fn add_completed_job_error(
completed_job.workspace_id,
serde_json::to_string(&result).unwrap_or_else(|_| "".to_string())
);
let _ = add_completed_job(
let (_, span) = add_completed_job(
db,
&completed_job,
false,
@@ -1281,7 +1306,7 @@ pub async fn add_completed_job_error(
)
.warn_after_seconds(10)
.await?;
Ok(result)
Ok((result, span))
}
lazy_static::lazy_static! {
@@ -1320,7 +1345,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
flow_is_done: bool,
duration: Option<i64>,
from_cache: bool,
) -> Result<(Uuid, i64), Error> {
) -> Result<(Uuid, CompletedSpan), Error> {
// tracing::error!("Start");
// let start = tokio::time::Instant::now();
@@ -1354,7 +1379,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
};
let result_columns = result_columns.as_ref();
let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| {
let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| {
commit_completed_job(
db,
completed_job,
@@ -1395,7 +1420,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
if let Some(job_id) = opt_uuid {
return Ok((job_id, duration));
return Ok((job_id, span));
}
// Auto-resolve a retry chain that ultimately worked, from whichever of the two
@@ -1436,7 +1461,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
}
#[cfg(feature = "cloud")]
apply_completed_job_cloud_usage(db, completed_job, duration);
apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms);
#[cfg(all(feature = "enterprise", feature = "private"))]
crate::jobs_ee::apply_completed_job_error_handlers(
@@ -1453,7 +1478,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// tracing::error!("4 {:?}", start.elapsed());
Ok((completed_job.id, duration))
Ok((completed_job.id, span))
}
async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
@@ -1471,7 +1496,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// True when a native script retry was enqueued for this failed attempt, i.e.
// this is not the terminal attempt — schedule completion handlers must wait.
retry_pending: bool,
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool, bool)> {
) -> windmill_common::error::Result<(Option<Uuid>, CompletedSpan, bool, bool)> {
// let start = std::time::Instant::now();
let job_id = completed_job.id;
@@ -1538,11 +1563,11 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
&& monitor_parent.is_none()
&& !is_scheduled
{
let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else {
let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else {
return Err(not_in_queue_error(db, job_id).await);
};
log_completed_job(completed_job, duration, success);
return Ok((None, duration, false, false));
log_completed_job(completed_job, completed.duration_ms, success);
return Ok((None, completed, false, false));
}
let mut tx = db.begin().warn_after_seconds(10).await?;
@@ -1550,7 +1575,8 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// The parent's rows are locked ahead of the child's own queue row (see
// `record_child_completion` for the order this must keep), so the duration it stamps is read
// before the completion: the one the completed row will hold. `now()` is fixed for the
// transaction, and a completed row already there keeps its own duration.
// transaction, and a completed row already there keeps its own duration. For a child that
// itself parked it is the last run's, matching the last start its timeline entry carries.
let mut wac_parent_ready = false;
if let Some(parent_job) = wac_parent {
let Some(duration) = sqlx::query_scalar!(
@@ -1577,7 +1603,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.await?;
}
let Some(duration) = completion.execute(&mut *tx).await? else {
let Some(completed) = completion.execute(&mut *tx).await? else {
return Err(not_in_queue_error(&mut *tx, job_id).await);
};
@@ -1787,16 +1813,35 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
tx.commit().warn_after_seconds(10).await?;
log_completed_job(completed_job, duration, success);
log_completed_job(completed_job, completed.duration_ms, success);
// tracing::info!("completed job: {:?}", start.elapsed().as_micros());
Ok((
None,
duration,
completed,
_skip_downstream_error_handlers,
wac_parent_ready,
))
}
/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's.
/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered;
/// it differs from `duration_ms` only for a job that parked.
#[derive(Debug, Clone, Copy)]
pub struct CompletedSpan {
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
pub duration_ms: i64,
pub last_run_ms: i64,
}
impl CompletedSpan {
/// Whether the row spans more than the run that started at `last_started_at`, i.e. the
/// job parked and resumed. A flow records such a step with the row's span, so the step
/// reads like the job does.
pub fn outlasts_run(&self, last_started_at: Option<chrono::DateTime<chrono::Utc>>) -> bool {
matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last)
}
}
/// What a job's completion writes, for `Completion::execute`.
struct Completion<'a> {
completed_job: &'a MiniCompletedJob,
@@ -1813,6 +1858,11 @@ impl Completion<'_> {
/// Moves the job from the queue to the completed jobs and refreshes a flow step's parent
/// ping, as one statement. Returns `None` when the job was no longer in the queue.
///
/// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's
/// `started_at` and `duration_ms` span the whole workflow from its first start, and the
/// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from
/// completed rows.
///
/// The completion takes its cancellation from the queue row it deletes, not only from
/// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still
/// being written, so the deleted row is the final word on whether the job was canceled.
@@ -1820,7 +1870,7 @@ impl Completion<'_> {
/// It locks the queue row before the completed row's key. Any other writer completing a job
/// (the monitor's zombie fallback, debounce) must take them in the same order, or the two
/// deadlock.
async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result<Option<i64>> {
async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result<Option<CompletedSpan>> {
let Completion {
completed_job,
success,
@@ -1851,16 +1901,20 @@ impl Completion<'_> {
.then_some(completed_job.parent_job)
.flatten()
else {
return sqlx::query_scalar!(
return sqlx::query_as!(
CompletedSpan,
"WITH deleted AS (
DELETE FROM v2_job_queue WHERE id = $1
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
(extras->>'wac_first_started_at')::timestamptz AS first_started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
), completed AS (
INSERT INTO v2_job_completed AS cj
( workspace_id
, id
, started_at
, duration_ms
, extras
, result
, result_columns
, canceled_by
@@ -1871,8 +1925,11 @@ impl Completion<'_> {
, status
, worker
)
SELECT d.workspace_id, d.id, d.started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),
SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
CASE WHEN d.first_started_at IS NULL THEN d.run_ms
ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
CASE WHEN d.first_started_at IS NOT NULL
THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
$3::text::jsonb, $10,
CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
@@ -1888,9 +1945,11 @@ impl Completion<'_> {
THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
RETURNING duration_ms
RETURNING cj.started_at, cj.duration_ms,
COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
)
SELECT duration_ms AS \"duration_ms!\" FROM completed",
SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
last_run_ms AS \"last_run_ms!\" FROM completed",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result,
@@ -1910,16 +1969,20 @@ impl Completion<'_> {
// A canceled flow is pinged too: it is completed by its next transition like any other
// flow, and the zombie flow monitor needs the ping to finish the cancel if that
// transition is lost.
sqlx::query_scalar!(
sqlx::query_as!(
CompletedSpan,
"WITH deleted AS (
DELETE FROM v2_job_queue WHERE id = $1
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason
RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
(extras->>'wac_first_started_at')::timestamptz AS first_started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
), completed AS (
INSERT INTO v2_job_completed AS cj
( workspace_id
, id
, started_at
, duration_ms
, extras
, result
, result_columns
, canceled_by
@@ -1930,8 +1993,11 @@ impl Completion<'_> {
, status
, worker
)
SELECT d.workspace_id, d.id, d.started_at,
COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),
SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
CASE WHEN d.first_started_at IS NULL THEN d.run_ms
ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
CASE WHEN d.first_started_at IS NOT NULL
THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
$3::text::jsonb, $10,
CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
@@ -1947,14 +2013,16 @@ impl Completion<'_> {
THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
RETURNING duration_ms
RETURNING cj.started_at, cj.duration_ms,
COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
), parent_ping AS (
UPDATE v2_job_runtime r SET ping = now()
FROM v2_job_queue q
WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12
AND EXISTS (SELECT 1 FROM completed)
)
SELECT duration_ms AS \"duration_ms!\" FROM completed",
SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
last_run_ms AS \"last_run_ms!\" FROM completed",
/* $1 */ completed_job.id,
/* $2 */ success,
/* $3 */ result,
@@ -2021,7 +2089,7 @@ async fn check_result_size<T: ValidableJson>(
db: &Pool<Postgres>,
queued_job: &MiniCompletedJob,
result: Json<&T>,
) -> Option<Result<(Option<Uuid>, i64, bool, bool), Error>> {
) -> Option<Result<(Option<Uuid>, CompletedSpan, bool, bool), Error>> {
let result_size = result.size() / 1024 / 1024;
if result_size > 2 {
if result_size > *MAX_RESULT_SIZE_MB {
backend/windmill-worker-volumes/src/lib.rs
+34
View File
@@ -54,11 +54,45 @@ pub struct VolumeMount {
pub target: String,
}
/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while
/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A
/// write within the filesystem's timestamp granularity of the hand-off is not seen.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FileStamp {
size: u64,
changed: Option<(i64, i64)>,
}
impl FileStamp {
/// Whether the file is as it was at `earlier`. An unreadable change time never matches.
pub fn unchanged_since(&self, earlier: &FileStamp) -> bool {
self.changed.is_some() && self == earlier
}
pub fn of(meta: &std::fs::Metadata) -> Self {
// ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it.
#[cfg(unix)]
let changed = {
use std::os::unix::fs::MetadataExt;
Some((meta.ctime(), meta.ctime_nsec()))
};
#[cfg(not(unix))]
let changed = meta
.modified()
.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64));
Self { size: meta.len(), changed }
}
}
pub struct VolumeState {
pub mount: VolumeMount,
pub local_dir: PathBuf,
pub manifest: HashMap<String, FileEntry>,
pub symlinks: HashMap<String, String>,
/// Stamp of each pulled file as the job received it, keyed like `manifest`.
pub stamps: HashMap<String, FileStamp>,
}
pub struct DownloadStats {
backend/windmill-worker/src/bun_executor.rs
+4 -1
View File
@@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output(
// status row, the order every child completion takes.
let segment_ms = sqlx::query_scalar!(
"WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)
UPDATE v2_job_queue q SET running = false, started_at = null
UPDATE v2_job_queue q SET running = false, started_at = null,
extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras
ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))
END
FROM prev WHERE q.id = $1
RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
job.id,
backend/windmill-worker/src/common.rs
+1
View File
@@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script(
wm_memory_usage: None,
job_isolation: None,
native_mode: None,
draining: None,
ping_type: PingType::InitScript,
},
)
backend/windmill-worker/src/handle_child.rs
+1
View File
@@ -1049,6 +1049,7 @@ where
let wm_memory_usage = get_windmill_memory_usage();
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref());
if job_id != Uuid::nil() {
if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await {
tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
backend/windmill-worker/src/lib.rs
+1
View File
@@ -81,6 +81,7 @@ mod prepare_deps;
mod python_executor;
#[cfg(feature = "python")]
mod python_versions;
mod resource_metrics;
pub mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
backend/windmill-worker/src/resource_metrics.rs
+235
View File
@@ -0,0 +1,235 @@
use std::sync::{atomic::Ordering, Mutex, OnceLock};
use windmill_common::{
otel_oss::{
otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory,
otel_set_worker_occupancy_rate,
},
worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP},
OTEL_METRICS_ENABLED,
};
use crate::common::OccupancyResult;
#[cfg(feature = "prometheus")]
struct PromMetrics {
memory_usage: prometheus::IntGaugeVec,
windmill_memory_usage: prometheus::IntGaugeVec,
memory_limit: prometheus::IntGaugeVec,
cpu_usage: prometheus::CounterVec,
cpu_periods: prometheus::IntCounterVec,
cpu_throttled_periods: prometheus::IntCounterVec,
cpu_throttled_seconds: prometheus::CounterVec,
occupancy_rate: prometheus::GaugeVec,
job_memory_peak: prometheus::HistogramVec,
}
#[cfg(feature = "prometheus")]
lazy_static::lazy_static! {
static ref PROM_METRICS: Option<PromMetrics> =
if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) {
const WORKER: &[&str] = &["name", "worker_group"];
Some(PromMetrics {
memory_usage: prometheus::register_int_gauge_vec!(
"worker_memory_usage_bytes",
"Memory used by the worker's container, page cache excluded",
WORKER
)
.unwrap(),
windmill_memory_usage: prometheus::register_int_gauge_vec!(
"worker_windmill_memory_usage_bytes",
"Resident memory of the worker process itself",
WORKER
)
.unwrap(),
memory_limit: prometheus::register_int_gauge_vec!(
"worker_memory_limit_bytes",
"Memory limit of the worker's container",
WORKER
)
.unwrap(),
cpu_usage: prometheus::register_counter_vec!(
"worker_cpu_usage_seconds_total",
"CPU time consumed by the worker's container",
WORKER
)
.unwrap(),
cpu_periods: prometheus::register_int_counter_vec!(
"worker_cpu_periods_total",
"CPU enforcement periods elapsed for the worker's container",
WORKER
)
.unwrap(),
cpu_throttled_periods: prometheus::register_int_counter_vec!(
"worker_cpu_throttled_periods_total",
"CPU enforcement periods in which the worker's container was throttled",
WORKER
)
.unwrap(),
cpu_throttled_seconds: prometheus::register_counter_vec!(
"worker_cpu_throttled_seconds_total",
"Time the worker's container spent throttled by its CPU limit",
WORKER
)
.unwrap(),
occupancy_rate: prometheus::register_gauge_vec!(
"worker_occupancy_rate",
"Share of time the worker spent executing jobs, over the window",
&["name", "worker_group", "window"]
)
.unwrap(),
job_memory_peak: prometheus::register_histogram_vec!(
"worker_job_memory_peak_bytes",
"Peak memory of a job's main process",
&["tag"],
windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec()
)
.unwrap(),
})
} else {
None
};
}
fn metrics_enabled() -> bool {
#[cfg(feature = "prometheus")]
if PROM_METRICS.is_some() {
return true;
}
OTEL_METRICS_ENABLED.load(Ordering::Relaxed)
}
// Memory and CPU are read from the cgroup, which every worker of a process shares
// (native mode runs several). Only the first worker to report exports them, so that
// summing the series over workers does not count one container several times.
static CONTAINER_SERIES_OWNER: OnceLock<String> = OnceLock::new();
static MEMORY_LIMIT: Mutex<Option<Option<i64>>> = Mutex::new(None);
/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits.
pub(crate) fn set_memory_limit(limit: Option<i64>) {
*MEMORY_LIMIT.lock().unwrap() = Some(limit);
}
static LAST_CPU_STAT: Mutex<CpuStat> =
Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 });
/// Exports the worker's resource readings. Meant for the ping ticks, which have already
/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read.
pub(crate) fn record_worker_resources(
worker_name: &str,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy: Option<&OccupancyResult>,
) {
if !metrics_enabled() {
return;
}
let worker_group = WORKER_GROUP.as_str();
if let Some(occupancy) = occupancy {
let windows = [
("total", Some(occupancy.occupancy_rate)),
("15s", occupancy.occupancy_rate_15s),
("5m", occupancy.occupancy_rate_5m),
("30m", occupancy.occupancy_rate_30m),
];
for (window, rate) in windows {
let Some(rate) = rate.filter(|r| r.is_finite()) else {
continue;
};
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
m.occupancy_rate
.with_label_values(&[worker_name, worker_group, window])
.set(rate as f64);
}
otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64);
}
}
if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name {
return;
}
let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory);
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
let labels = &[worker_name, worker_group];
if let Some(v) = memory_usage {
m.memory_usage.with_label_values(labels).set(v);
}
if let Some(v) = wm_memory_usage {
m.windmill_memory_usage.with_label_values(labels).set(v);
}
if let Some(v) = memory_limit {
m.memory_limit.with_label_values(labels).set(v);
}
}
otel_set_worker_memory(
worker_name,
worker_group,
memory_usage,
wm_memory_usage,
memory_limit,
);
let Some(stat) = get_cpu_stat() else {
return;
};
let delta = {
let mut last = LAST_CPU_STAT.lock().unwrap();
// A reading below the previous one means the cgroup changed under the process:
// restart from it rather than exporting a negative increase.
let delta = if stat.usage_usec < last.usage_usec {
CpuStat::default()
} else {
CpuStat {
usage_usec: stat.usage_usec - last.usage_usec,
nr_periods: stat.nr_periods.saturating_sub(last.nr_periods),
nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled),
throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec),
}
};
*last = stat;
delta
};
let usage_secs = delta.usage_usec as f64 / 1e6;
let throttled_secs = delta.throttled_usec as f64 / 1e6;
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
let labels = &[worker_name, worker_group];
m.cpu_usage.with_label_values(labels).inc_by(usage_secs);
m.cpu_periods
.with_label_values(labels)
.inc_by(delta.nr_periods);
m.cpu_throttled_periods
.with_label_values(labels)
.inc_by(delta.nr_throttled);
m.cpu_throttled_seconds
.with_label_values(labels)
.inc_by(throttled_secs);
}
otel_add_worker_cpu(
worker_name,
worker_group,
usage_secs,
delta.nr_periods,
delta.nr_throttled,
throttled_secs,
);
}
/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process
/// to sample or the reading failed, and those are not observations.
pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) {
if mem_peak_kb <= 0 {
return;
}
let bytes = mem_peak_kb as u64 * 1024;
#[cfg(feature = "prometheus")]
if let Some(m) = PROM_METRICS.as_ref() {
m.job_memory_peak
.with_label_values(&[tag])
.observe(bytes as f64);
}
otel_record_job_memory_peak(tag, bytes);
}
backend/windmill-worker/src/result_processor.rs
+27 -13
View File
@@ -1776,7 +1776,7 @@ pub async fn process_completed_job(
add_time!(bench, "pre add_completed_job");
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&job,
true,
@@ -1835,7 +1835,10 @@ pub async fn process_completed_job(
true,
canceled_by,
result,
started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }),
span.started_at.or(started_at).map(|x| FlowJobDuration {
started_at: x,
duration_ms: span.duration_ms,
}),
StepFailureKind::Normal,
&same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(),
&worker_dir,
@@ -1867,9 +1870,10 @@ pub async fn process_completed_job(
// `wm_failure` field) so a real runtime failure whose raw
// result happens to contain a `wm_failure` field still goes
// through the standard `WrappedError { error: ... }` wrap path.
let downstream_result: Arc<Box<RawValue>> = if is_pre_shaped_wm_failure_result(result.get())
{
windmill_queue::add_completed_job_pre_shaped_failure(
let (downstream_result, span): (Arc<Box<RawValue>>, _) = if is_pre_shaped_wm_failure_result(
result.get(),
) {
let span = windmill_queue::add_completed_job_pre_shaped_failure(
db,
&job,
mem_peak.to_owned(),
@@ -1880,9 +1884,9 @@ pub async fn process_completed_job(
None,
)
.await?;
result.clone()
(result.clone(), span)
} else {
let wrapped = add_completed_job_error(
let (wrapped, span) = windmill_queue::add_completed_job_error_with_span(
db,
&job,
mem_peak.to_owned(),
@@ -1895,7 +1899,10 @@ pub async fn process_completed_job(
None,
)
.await?;
Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap())
(
Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()),
span,
)
};
#[cfg(all(feature = "enterprise", feature = "private"))]
if job.kind == JobKind::DeploymentCallback {
@@ -1927,12 +1934,19 @@ pub async fn process_completed_job(
false,
canceled_by,
downstream_result,
duration.and_then(|d| {
job.started_at.map(|started_at| FlowJobDuration {
started_at: started_at,
duration_ms: d,
if span.outlasts_run(job.started_at) {
span.started_at.map(|started_at| FlowJobDuration {
started_at,
duration_ms: span.duration_ms,
})
}),
} else {
duration.and_then(|d| {
job.started_at.map(|started_at| FlowJobDuration {
started_at: started_at,
duration_ms: d,
})
})
},
StepFailureKind::Normal,
&same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(),
&worker_dir,
backend/windmill-worker/src/wac_executor.rs
+9 -1
View File
@@ -113,6 +113,11 @@ pub enum WacPark {
/// handler — falls back to `now() - started_at`. Left pointing at the first segment,
/// that fallback reports the whole sleep or approval wait as execution time.
///
/// The first park keeps that first start in `extras.wac_first_started_at`, so the
/// completed row can still say when the workflow began and how long it took overall
/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline
/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way.
///
/// Call it before any write to the parent's `v2_job_status` row in the same
/// transaction: a child's completion locks the queue row and then the status row
/// (`record_child_completion`), and taking them the other way round here can
@@ -156,7 +161,10 @@ pub async fn suspend_wac_parent(
sqlx::query!(
"UPDATE v2_job_queue
SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null
SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,
extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras
ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))
END
WHERE id = $1 AND workspace_id = $2",
job_id,
w_id,
backend/windmill-worker/src/worker.rs
+29 -3
View File
@@ -155,7 +155,7 @@ use crate::{
handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
tally_unfinished_dependency_deploy,
},
worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full},
worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full},
};
#[cfg(feature = "rust")]
@@ -2713,7 +2713,7 @@ pub async fn run_worker(
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
let mut reported_ip = cached_ip();
let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn)
let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn)
.await
.expect("initial ping could be sent");
@@ -3160,6 +3160,31 @@ pub async fn run_worker(
let mut killpill_rx2 = killpill_rx.resubscribe();
// The loop below only sees the killpill once the running job is done, which is too late
// for that job to learn its worker is going away.
{
let mut killpill_rx = killpill_rx.resubscribe();
let conn = conn.clone();
let worker_name = worker_name.clone();
tokio::spawn(async move {
if matches!(
killpill_rx.recv().await,
Err(broadcast::error::RecvError::Closed)
) {
return;
}
for attempt in 1..=3 {
match mark_worker_draining(&conn, &worker_name).await {
Ok(()) => return,
Err(e) => {
tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}");
tokio::time::sleep(Duration::from_secs(2)).await;
}
}
}
});
}
loop {
let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst);
if last_processing_duration_secs > 5 {
@@ -3235,7 +3260,7 @@ pub async fn run_worker(
// which costs at most the last job's id here: no job of this worker is in flight at this
// point in the loop, and the next one refills them.
if ip_just_resolved && conn.as_sql().is_none() {
if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await {
if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await {
tracing::warn!(
worker = %worker_name, hostname = %hostname,
"failed to re-register with the resolved external IP: {e}"
@@ -5105,6 +5130,7 @@ pub async fn handle_queued_job(
// WAC v2 job suspended while waiting for child jobs — don't complete it
return Ok(JobOutcome::Completed);
}
crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak);
process_result(
cjob,
result.map(|x| Arc::new(x)),
backend/windmill-worker/src/worker_flow.rs
+4 -4
View File
@@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal(
)
.await?;
let duration = if success {
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&cflow_job,
true,
@@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal(
false,
)
.await?;
duration
span.duration_ms
} else {
let (_, duration) = add_completed_job(
let (_, span) = add_completed_job(
db,
&cflow_job,
false,
@@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal(
false,
)
.await?;
duration
span.duration_ms
};
flow_job_duration = flow_job
.started_at
backend/windmill-worker/src/worker_utils.rs
+58 -5
View File
@@ -7,9 +7,9 @@ use windmill_common::{
external_ip::UNKNOWN_IP,
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED,
WORKER_CONFIG, WORKER_GROUP,
insert_ping_query, set_worker_draining_query, update_job_ping_query,
update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping,
PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP,
},
KillpillSender, DB,
};
@@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full(
let wm_memory_usage = get_windmill_memory_usage();
let (vcpus, memory) = if read_cgroups {
(get_vcpus(), get_memory())
let memory = get_memory();
crate::resource_metrics::set_memory_limit(memory);
(get_vcpus(), memory)
} else {
(None, None)
};
let occupancy = occupancy_metrics.update_occupancy_metrics();
crate::resource_metrics::record_worker_resources(
worker_name,
memory_usage,
wm_memory_usage,
Some(&occupancy),
);
let OccupancyResult {
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
} = occupancy_metrics.update_occupancy_metrics();
} = occupancy;
let ping_start = std::time::Instant::now();
if let Err(e) = (|| {
@@ -164,6 +173,7 @@ async fn update_worker_ping_full_inner(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation: None,
native_mode: Some(native_mode),
draining: None,
ping_type: PingType::MainLoop,
},
)
@@ -179,6 +189,7 @@ pub async fn insert_ping(
worker_instance: &str,
worker_name: &str,
ip: Option<&str>,
reset_draining: bool,
db: &Connection,
) -> anyhow::Result<i32> {
let (tags, dw, dws, native_mode) = {
@@ -224,6 +235,7 @@ pub async fn insert_ping(
memory,
job_isolation,
native_mode,
reset_draining.then_some(false),
db,
)
.await;
@@ -256,6 +268,7 @@ pub async fn insert_ping(
wm_memory_usage: get_windmill_memory_usage(),
job_isolation,
native_mode: Some(native_mode),
draining: reset_draining.then_some(false),
ping_type: PingType::Initial,
},
)
@@ -314,6 +327,7 @@ pub async fn update_worker_ping_from_job(
&Ping {
last_job_executed: Some(job_id.clone()),
last_job_workspace_id: Some(w_id.to_string()),
draining: None,
ping_type: PingType::Job,
worker_instance: None,
ip: None,
@@ -342,6 +356,45 @@ pub async fn update_worker_ping_from_job(
Ok(())
}
/// Publishes on the worker's ping row that it has received its killpill, so the job it is
/// still running can find out and exit early.
pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> {
match conn {
Connection::Sql(db) => set_worker_draining_query(worker_name, db).await,
Connection::Http(client) => {
client
.post::<Ping, ()>(
UPDATE_PING_URL,
None,
&Ping {
last_job_executed: None,
last_job_workspace_id: None,
draining: None,
ping_type: PingType::Draining,
worker_instance: None,
ip: None,
tags: None,
dw: None,
dws: None,
version: None,
vcpus: None,
memory: None,
memory_usage: None,
wm_memory_usage: None,
jobs_executed: None,
occupancy_rate: None,
occupancy_rate_15s: None,
occupancy_rate_5m: None,
occupancy_rate_30m: None,
job_isolation: None,
native_mode: None,
},
)
.await
}
}
}
pub async fn ping_job_status(
conn: &Connection,
job_id: &Uuid,
cli/src/guidance/skills.gen.ts
Generated
+76 -2
View File
@@ -839,6 +839,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -1674,6 +1682,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -2615,6 +2631,14 @@ async setProgress(percent: number, jobId?: any): Promise<void>
*/
async getProgress(jobId?: any): Promise<number | null>
/**
* Whether the worker running this job has received its shutdown signal.
* A draining worker waits for its current job to finish before exiting, so a
* long-running script can poll this and return early to let the worker be replaced.
* @param jobId? Job to check, defaults to the current job
*/
async workerIsDraining(jobId?: string): Promise<boolean>
/**
* Set a flow user state
* @param key key of the state
@@ -4620,6 +4644,18 @@ def set_progress(value: int, job_id: Optional[str] = None)
# Progress value (0-100) or None if not set
def get_progress(job_id: Optional[str] = None) -> Any
# Whether the worker running the job has received its shutdown signal.
#
# A draining worker waits for its current job to finish before exiting. A long-running
# script can poll this and return early so the worker can be replaced.
#
# Args:
# job_id: Job ID (defaults to current WM_JOB_ID)
#
# Returns:
# True if the worker running the job is draining
def worker_is_draining(job_id: Optional[str] = None) -> bool
# Set the user state of a flow at a given key
def set_flow_user_state(key: str, value: Any) -> None
@@ -7879,10 +7915,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step
A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake://<name>/<table>\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else:
- \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live).
- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store.
- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store.
Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal.
## Ingesting from Python / TypeScript
A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table.
Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema:
\`\`\`python
# pipeline
# on schedule
import wmill
def main():
rows = fetch_charges() # list[dict]
lake = wmill.ducklake()
lake.query("""
CREATE OR REPLACE TABLE stripe_charges AS
SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)
""", rows=rows).fetch()
\`\`\`
\`\`\`ts
// pipeline
// on schedule
import * as wmill from "windmill-client"
export async function main() {
const rows = await fetchCharges()
const lake = wmill.ducklake()
await lake\`CREATE OR REPLACE TABLE stripe_charges AS
SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch()
}
\`\`\`
- Use \`INSERT INTO <table> SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first.
- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/<table>\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms.
- \`wmill.ducklake("<name>")\` targets a non-default catalog; \`wmill.datatable("<name>")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above).
- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`.
## Storage prerequisites
A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build.
@@ -7926,7 +8000,7 @@ The key must be a **string literal** — the graph parser is static and cannot f
## Materialize (the managed output)
> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake://<name>/<table>\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table.
> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake://<name>/<table>\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table.
>
> The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt://<warehouse>/<schema>/<name>\`, where \`<warehouse>\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt://<warehouse>/<schema>/<name>\` fires when the ingestion node completes.
frontend/src/lib/apiBaseUrl.svelte.ts
+22
View File
@@ -0,0 +1,22 @@
import { fromStore } from 'svelte/store'
import { base } from '$lib/base'
import { instanceUi } from '$lib/instanceUi'
const ui = fromStore(instanceUi)
/**
* Base of every URL shown for an external client to call (webhooks, HTTP routes, push
* trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the
* origin being browsed. Append `/api/...` to it.
*
* Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links
* to pages): those must stay on the browsing origin, where the session cookie lives.
*
* Reactive: the setting lands after page load, so read it inside a `$derived` or the
* template rather than caching the result.
*/
export function apiBaseUrl(): string {
const setting = ui.current?.api_base_url
const configured = typeof setting === 'string' ? setting.trim() : ''
return configured || `${window.location.origin}${base}`
}
frontend/src/lib/components/AutosaveIndicator.svelte
+7 -1
View File
@@ -142,7 +142,13 @@
let hintTimer: ReturnType<typeof setTimeout> | undefined
const kindLabel = $derived(
itemKind === 'flow' ? 'flow' : itemKind === 'app' || itemKind === 'raw_app' ? 'app' : 'script'
itemKind === 'flow'
? 'flow'
: itemKind === 'app' || itemKind === 'raw_app'
? 'app'
: itemKind === 'data_pipeline'
? 'pipeline'
: 'script'
)
// Each false → true transition fires a hint (others > loaded). Seeded to
frontend/src/lib/components/CliHelpBox.svelte
+2 -1
View File
@@ -2,10 +2,11 @@
import ClipboardPanel from './details/ClipboardPanel.svelte'
import Section from './Section.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
const operatingWorkspace = useOperatingWorkspace()
let url = $derived(`${window.location.protocol}//${window.location.hostname}/`)
let url = $derived(`${apiBaseUrl()}/`)
</script>
<Section label="CLI quick setup">
frontend/src/lib/components/CronBuilder.svelte
+1 -1
View File
@@ -15,7 +15,7 @@
contentClasses="p-4"
>
{#snippet trigger()}
<Button variant="accent" size="xs" nonCaptureEvent={true} startIcon={{ icon: Clock }}>
<Button variant="default" unifiedSize="sm" nonCaptureEvent={true} startIcon={{ icon: Clock }}>
Cron builder
</Button>
{/snippet}
frontend/src/lib/components/DatatablePicker.svelte
+8 -1
View File
@@ -17,6 +17,9 @@
selectInputClass?: string
class?: string
onClear?: () => void
/** Size the input to its content, like Path's folder picker. */
useContentEditable?: boolean
RightIcon?: any
}
let {
@@ -27,7 +30,9 @@
placeholder = undefined,
selectInputClass = '',
class: className = '',
onClear = undefined
onClear = undefined,
useContentEditable = false,
RightIcon = undefined
}: Props = $props()
let datatables = usePromise(() =>
@@ -47,6 +52,8 @@
{placeholder}
inputClass={selectInputClass}
{onClear}
{useContentEditable}
{RightIcon}
/>
{#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })}
<ExploreAssetButton class="mt-1 w-fit" asset={{ kind: 'datatable', path: value }} />
frontend/src/lib/components/DucklakePicker.svelte
+8 -1
View File
@@ -17,6 +17,9 @@
selectInputClass?: string
class?: string
onClear?: () => void
/** Size the input to its content, like Path's folder picker. */
useContentEditable?: boolean
RightIcon?: any
}
let {
@@ -27,7 +30,9 @@
placeholder = undefined,
selectInputClass = '',
class: className = '',
onClear = undefined
onClear = undefined,
useContentEditable = false,
RightIcon = undefined
}: Props = $props()
let ducklakes = usePromise(() =>
@@ -45,6 +50,8 @@
{placeholder}
inputClass={selectInputClass}
{onClear}
{useContentEditable}
{RightIcon}
/>
{#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'ducklake', path: value })}
<ExploreAssetButton class="mt-1 w-fit" asset={{ kind: 'ducklake', path: value }} />
frontend/src/lib/components/Editor.svelte
+3
View File
@@ -1523,6 +1523,9 @@
} catch (e) {
console.log('error initializing services', e)
}
// Unmounted while the services loaded: the container is detached, and
// creating the editor on it throws.
if (destroyed || !divEl?.isConnected) return
// vscode.languages.registerDefinitionProvider('*', {
// provideDefinition(document, position, token) {
frontend/src/lib/components/EditorBar.svelte
+1
View File
@@ -105,6 +105,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
frontend/src/lib/components/FlowStatusViewerInner.svelte
+5 -1
View File
@@ -69,6 +69,7 @@
retainLinkedToolsScope
} from './flows/linkedAgentToolsStore.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { AGENT_STEP_ID, agentStepLabel } from './flows/conversations/agentEditorChat'
const operatingWorkspace = useOperatingWorkspace()
@@ -1238,6 +1239,8 @@
id: string
type: FlowModuleValue['type']
suspend?: boolean
/** Shown in place of the id. */
label?: string
}
function allModulesForTimeline(
@@ -1250,7 +1253,8 @@
({
id: x.id,
type: x.value.type,
suspend: x.suspend != undefined
suspend: x.suspend != undefined,
label: x.id === AGENT_STEP_ID ? agentStepLabel(job?.script_path) : undefined
}) as FlowModuleForTimeline,
{
skipToolNodes: true
frontend/src/lib/components/FlowTimeline.svelte
+2 -2
View File
@@ -179,12 +179,12 @@
/>
</div>
{/if}
{#each flowModules as { id: k, type: typ, suspend: isSuspend } (k)}
{#each flowModules as { id: k, type: typ, suspend: isSuspend, label } (k)}
{@const subItems = items?.[k]?.filter((x) => x.created_at && x.started_at)}
<div class="relative px-3 py-1.5">
<div class="flex items-center justify-between mb-0.5">
<div class="text-xs font-medium"
>{k.startsWith('subflow:') ? k.substring(8) : k}
>{label ?? (k.startsWith('subflow:') ? k.substring(8) : k)}
{#if localModuleStates[k]?.selectedForloop && (typ == 'forloopflow' || typ == 'whileloopflow')}
<span class="text-xs font-mono font-medium inline-flex items-center -my-2">
<button onclick={(e) => e.stopPropagation()}>
frontend/src/lib/components/JobArgs.svelte
+2 -1
View File
@@ -15,6 +15,7 @@
import { isWindmillTooBigObject } from './job_args'
import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile'
import { appendViewToken } from '$lib/viewToken'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
interface Props {
id?: string | undefined
@@ -246,7 +247,7 @@ ${Object.entries(displayArgs)
If using the wmill client in your code, set the following env variables:
<pre
><code
>BASE_URL="{window.location.origin}"
>BASE_URL="{apiBaseUrl()}"
WM_TOKEN="{'<TOKEN>'}"</code
></pre
>
frontend/src/lib/components/NavBreadcrumb.svelte
+9
View File
@@ -396,6 +396,15 @@ not this component) — so the lead can run the whole line.
onPick={() => {}}
/>
{:else if section}
{#if section.parent}
<!-- Gives width back with the workspace part, before the section's own name does. -->
<span
class={twMerge(SEGMENT, LEAD, 'shrink-[999] hover:bg-transparent hover:text-secondary')}
>
<span class="truncate">{section.parent}</span>
</span>
{@render slash()}
{/if}
<!-- A section's own widget lays out as a row: its title, and whatever belongs with it. -->
<span
class={twMerge(
frontend/src/lib/components/PageHeaderBar.svelte
+14 -3
View File
@@ -32,6 +32,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
const item = $derived(content?.item)
const section = $derived(content?.section)
const hasSummary = $derived(!!item && !!(item.summaryContent || item.summary))
// Below this the trail holds a tighter cap on the path it draws. The names in it truncate on
// their own at any width (see NavBreadcrumb), so this is not a breakpoint the look changes at —
@@ -135,8 +136,18 @@ The row's height matches the sidebar's own header row, so the two read as one ba
where the user is.
An embed (`navHidden`) keeps the page's own name but drops the workspace part around it:
the trail would offer to navigate the host's workspace, while the name is what says which
page the controls beside it belong to. -->
<div class="flex shrink-[0.3] min-w-[5rem]">
page the controls beside it belong to.
The 0.3 only holds against a summary or an actions box that shrinks too: alone, a factor
sum below 1 makes flex hand out only that share of the overflow, and the bar overflows
instead. -->
<div
class={twMerge(
'flex shrink-[0.3] min-w-[5rem]',
!hasSummary &&
(actions.length === 0 || (phone && !content?.actionsFlexible && !content?.actionsFill)) &&
'shrink'
)}
>
<!-- Bridged like the actions below: what a page hangs off its own name renders here, out of
the tree that named it, and the rename editor in there asks that tree who the acting user
is before it offers to rename anything. -->
@@ -161,7 +172,7 @@ The row's height matches the sidebar's own header row, so the two read as one ba
</svelte:boundary>
</div>
{#if item && (item.summaryContent || item.summary)}
{#if item && hasSummary}
<!-- No kind icon: the page below is the item, and saying "this is a flow" above a flow
editor tells the reader what they can already see. -->
<div class={twMerge('flex items-center gap-1 min-w-0', phone && 'shrink')}>
frontend/src/lib/components/ScriptBuilder.svelte
+20 -2
View File
@@ -596,6 +596,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
@@ -1543,10 +1544,12 @@
(lang == script.language &&
template != 'bunnative' &&
template != 'docker' &&
template != 'claudesandbox') ||
template != 'claudesandbox' &&
template != 'pisandbox') ||
(template == 'bunnative' && lang == 'bunnative') ||
(template == 'docker' && lang == 'docker') ||
(template == 'claudesandbox' && lang == 'bun')}
((template == 'claudesandbox' || template == 'pisandbox') &&
lang == 'bun')}
<Popover
disablePopup={!enterpriseLangs.includes(lang) || !!$enterpriseLicense}
>
@@ -1597,6 +1600,21 @@
>
Claude Sandbox
</Button>
<Button
unifiedSize="xs"
variant="default"
startIcon={{
icon: LanguageIcon,
props: { lang: 'pisandbox', width: 16, height: 16 }
} as ButtonType.Icon}
onclick={() => {
template = 'pisandbox'
script.language = 'bun'
initContent('bun', script.kind, template)
}}
>
Pi Sandbox
</Button>
<span title="Workflow-as-Code">
<Button
size="xs2"
frontend/src/lib/components/ScriptEditor.svelte
+35 -37
View File
@@ -150,6 +150,7 @@
| 'powershell'
| 'bunnative'
| 'claudesandbox'
| 'pisandbox'
| 'wac_python'
| 'wac_typescript'
| 'ci_test_bun'
@@ -2100,43 +2101,15 @@
</div>
{/if}
{#if customUi?.previewPanel?.hideArgs}
<!-- Compact preview layout used by the pipeline editor:
no args column (the script is known to take no
inputs), LogPanel takes the full width, with a
small Test/Cancel button at the top-left of the
preview band. The earlier `-translate-y-1/2`
"float onto the editor" version was clipped in
Firefox (transform + overflow: visible interaction
on the parent splitpane Pane), so the button is
now positioned inside the panel — `top-1` keeps
it visually pinned to the top edge without
relying on cross-browser overflow behaviour. -->
<div class="relative h-full pt-9 flex flex-col">
{#if testJob?.id && testJob.type === 'CompletedJob' && opWs}
<!-- Right-side affordances when we're displaying a *completed*
job (either the user just ran a test, or the on-mount
last-run loader populated the panel). The job-id link
opens the full run page in a new tab; the dispatch
button mounts a popover that shows what downstream
jobs this run triggered (rendered only when the run
actually dispatched anything). -->
<div class="absolute top-1 right-2 z-10 flex items-center gap-2">
<a
class="text-3xs text-blue-600 hover:underline font-mono"
href={`${base}/run/${testJob.id}?workspace=${opWs}`}
target="_blank"
rel="noopener noreferrer"
title="Open this run"
>
{testJob.id.slice(0, 8)}… ↗
</a>
<DispatchEventsButton
workspace={testJob.workspace_id ?? opWs}
jobId={testJob.id}
/>
</div>
{/if}
<div class="absolute top-1 left-2 z-10">
<!-- Compact preview layout used by the pipeline editor: no args
column (most pipeline scripts take no inputs) and a full-width
LogPanel. The controls sit in normal flow rather than floating
onto the editor band, which Firefox clipped (transform +
overflow: visible on the parent splitpane Pane). -->
<div class="h-full flex flex-col">
<!-- Test sits at the top right, the last run's link under it. -->
<div class="shrink-0 flex flex-col items-end gap-1 px-2 pt-2 pb-1">
<div>
{#if testIsLoading}
{@render cancelTestButton('sm', 'shadow-md')}
{:else if (customUi?.previewPanel?.downstreamSubscribers ?? 0) > 0 || customUi?.previewPanel?.onBoundedRun}
@@ -2270,6 +2243,31 @@
{@render runTestButton('sm', 'shadow-md')}
{/if}
</div>
{#if testJob?.id && testJob.type === 'CompletedJob' && opWs}
<!-- Right-side affordances when we're displaying a *completed*
job (either the user just ran a test, or the on-mount
last-run loader populated the panel). The job-id link
opens the full run page in a new tab; the dispatch
button mounts a popover that shows what downstream
jobs this run triggered (rendered only when the run
actually dispatched anything). -->
<div class="flex items-center gap-2">
<a
class="text-3xs text-blue-600 hover:underline font-mono"
href={`${base}/run/${testJob.id}?workspace=${opWs}`}
target="_blank"
rel="noopener noreferrer"
title="Open this run"
>
{testJob.id.slice(0, 8)}… ↗
</a>
<DispatchEventsButton
workspace={testJob.workspace_id ?? opWs}
jobId={testJob.id}
/>
</div>
{/if}
</div>
{#if customUi?.previewPanel?.argsAboveLogs && schema?.properties && Object.keys(schema.properties).length > 0}
<div
class="shrink-0 overflow-auto max-h-[45%] border-b border-gray-200 dark:border-gray-700 px-4 py-3"
frontend/src/lib/components/SummaryPathDisplay.svelte
+22 -1
View File
@@ -31,6 +31,13 @@
/** What is being renamed. An agent is a resource, so the path field is scoped to one and
* the write goes through the agent branch of the rename manager. */
kind?: 'flow' | 'script' | 'agent'
/** Saves instead of the API, for an item that exists only locally (a draft).
* Returns an error message to keep the popover open. */
saveOverride?: (next: {
path: string
summary: string
labels: string[] | undefined
}) => string | undefined
/** Header variant: no path line (the host shows it) and a lighter summary. */
compact?: boolean
/** Rendered inside the trigger in place of the summary, for a host that hangs a second copy
@@ -51,6 +58,7 @@
editable = false,
onSaved,
kind = 'flow',
saveOverride = undefined,
compact = false,
label,
triggerClass = 'block min-w-0 max-w-full px-1 py-0.5 rounded text-left cursor-pointer hover:bg-surface-hover transition-colors',
@@ -67,7 +75,9 @@
let popoverOpen = $state(false)
// Resolved before the popover opens, not on opening it: the path field only exists for an owner,
// and `openFocus` looks for it in the same flush the popover renders in.
const own = $derived(isOwner(path ?? '', actingUser, $operatingWorkspace))
const own = $derived(
saveOverride !== undefined || isOwner(path ?? '', actingUser, $operatingWorkspace)
)
let onBehalfOfEmail = $state<string | undefined>(undefined)
let summaryInput: ReturnType<typeof TextInput> | undefined = $state()
let labelsDirty = $state(false)
@@ -101,6 +111,17 @@
async function save(close: () => void) {
const initialPath = path ?? ''
const newPath = own ? editPath : initialPath
if (saveOverride) {
const error = saveOverride({ path: newPath, summary: editSummary, labels })
if (error) {
sendUserToast(error, true)
return
}
labelsDirty = false
close()
onSaved?.(newPath)
return
}
try {
await updateItemPathAndSummary({
frontend/src/lib/components/SuperadminSettingsInner.svelte
+1 -1
View File
@@ -372,7 +372,7 @@
</div>
{/if}
<Tabs bind:selected={usersSubTab} class="mb-4">
<Tabs bind:selected={usersSubTab} class="mb-4" wrapperClass="shrink-0">
<Tab value="users" label="Users" />
{#if extJwtTokens.length > 0}
<Tab value="ext_jwt" label="External JWTs" />
frontend/src/lib/components/WorkerGroup.svelte
+2 -1
View File
@@ -55,6 +55,7 @@
import TagList from './TagList.svelte'
import DedicatedWorkersSelector from './DedicatedWorkersSelector.svelte'
import { computeHashedTag } from './dedicated_worker'
import { apiBaseUrl } from '$lib/apiBaseUrl.svelte'
import ObjectStoreConfigSettings, {
type ObjectStoreConfig
} from './ObjectStoreConfigSettings.svelte'
@@ -1238,7 +1239,7 @@
<p class="text-xs text-tertiary">Requires a superadmin token.</p>
<pre
class="mt-1 p-2 bg-surface-secondary rounded text-xs overflow-x-auto whitespace-pre-wrap break-all"
>curl -X POST '{window.location.origin}/api/configs/update/worker__{name}' \
>curl -X POST '{apiBaseUrl()}/api/configs/update/worker__{name}' \
-H 'Content-Type: application/json' \
-H 'Authorization: Bearer &lt;token&gt;' \
-d '{JSON.stringify(cleanWorkerGroupConfig(nconfig ?? {}), null, 2)}'</pre
frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte
+150
View File
@@ -0,0 +1,150 @@
<!--
@component
The menu that adds a pipeline script started by a data source (a schedule, a
webhook, a queue…): the add node's, and the canvas's right-click one.
-->
<script lang="ts">
import type { Snippet } from 'svelte'
import PipelineInsertMenu, {
type PipelineInsertOptions,
type PipelineInsertPick
} from './PipelineInsertMenu.svelte'
import {
Clock,
Webhook,
Mail,
Zap,
Radio,
MessageSquare,
Database,
Send,
CloudCog,
Upload
} from 'lucide-svelte'
import type { ScriptLang } from '$lib/gen'
import type { NativeTriggerKind } from './types'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import type { PipelineOutputKind } from './pipelineTemplates'
interface Props {
onAddPipelineScript: (
language: ScriptLang,
path: string,
source: { kind: NativeTriggerKind; path: string | undefined },
outputKind: PipelineOutputKind,
aiPrompt?: string,
options?: PipelineInsertOptions
) => void
pathPrefix: string
defaultPathSuffix: string
trigger: Snippet<[{ open: boolean }]>
/** Opens the menu each time it changes. */
openSignal?: number
}
let {
onAddPipelineScript,
pathPrefix,
defaultPathSuffix,
trigger,
openSignal
}: Props = $props()
function handlePick(pick: PipelineInsertPick) {
if (!pick.language || !pick.path) return
// Native trigger annotation is marker-only — the binding lives on the
// trigger row's own `script_path`, which is created separately.
onAddPipelineScript(
pick.language as ScriptLang,
pick.path,
{ kind: pick.kindId as NativeTriggerKind, path: undefined },
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt,
{ schedule: pick.schedule, outputAsset: pick.outputAsset }
)
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'data_upload',
label: 'On data upload',
description: 'UI-first: run by uploading a file via the S3 picker',
icon: Upload
},
{
id: 'schedule',
label: 'On schedule',
description: 'Triggered by a schedule you create',
icon: Clock
},
{
id: 'webhook',
label: 'On webhook',
description: 'Triggered by an HTTP webhook',
icon: Webhook
},
{
id: 'email',
configuredAfterCreate: true,
label: 'On email',
description: 'Triggered by incoming email',
icon: Mail
},
{
id: 'kafka',
configuredAfterCreate: true,
label: 'On Kafka',
description: 'Triggered by a Kafka message',
icon: Zap
},
{
id: 'mqtt',
configuredAfterCreate: true,
label: 'On MQTT',
description: 'Triggered by an MQTT message',
icon: Radio
},
{
id: 'amqp',
configuredAfterCreate: true,
label: 'On AMQP',
description: 'Triggered by an AMQP (RabbitMQ) message',
icon: Radio
},
{
id: 'nats',
configuredAfterCreate: true,
label: 'On NATS',
description: 'Triggered by a NATS message',
icon: MessageSquare
},
{
id: 'postgres',
configuredAfterCreate: true,
label: 'On Postgres',
description: 'Triggered by a Postgres event',
icon: Database
},
{
id: 'sqs',
configuredAfterCreate: true,
label: 'On SQS',
description: 'Triggered by an SQS message',
icon: Send
},
{
id: 'gcp',
configuredAfterCreate: true,
label: 'On GCP Pub/Sub',
description: 'Triggered by a Pub/Sub message',
icon: CloudCog
}
]}
languages={PIPELINE_LANGUAGES as any}
{pathPrefix}
{defaultPathSuffix}
onPick={handlePick}
{openSignal}
{trigger}
/>
frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte
+73
View File
@@ -0,0 +1,73 @@
<!--
@component
The menu adding a pipeline script downstream of an asset: opened from the asset's
"+", or where that "+" was dropped on the blank canvas.
-->
<script lang="ts" module>
import type { ScriptLang } from '$lib/gen'
import type { AssetKind } from '$lib/components/assets/lib'
import type { PipelineInsertOptions } from './PipelineInsertMenu.svelte'
import type { PipelineOutputKind } from './pipelineTemplates'
export type AddScriptForAsset = (
asset: { kind: AssetKind; path: string },
language: ScriptLang,
scriptPath: string,
outputKind: PipelineOutputKind,
aiPrompt?: string,
options?: PipelineInsertOptions
) => void
</script>
<script lang="ts">
import { Code2 } from 'lucide-svelte'
import type { Snippet } from 'svelte'
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
let {
asset,
onAddScript,
pathPrefix = '',
defaultPathSuffix = '',
openSignal,
trigger
}: {
asset: { kind: AssetKind; path: string }
onAddScript: AddScriptForAsset
pathPrefix?: string
defaultPathSuffix?: string
openSignal?: number
trigger: Snippet<[{ open: boolean }]>
} = $props()
function handlePick(pick: PipelineInsertPick) {
if (pick.kindId === 'pipeline_script' && pick.language && pick.path) {
onAddScript(
asset,
pick.language as ScriptLang,
pick.path,
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt,
{ outputAsset: pick.outputAsset }
)
}
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'pipeline_script',
label: 'Add downstream pipeline script',
description: 'Triggered when this asset changes',
icon: Code2
}
]}
languages={PIPELINE_LANGUAGES as any}
{pathPrefix}
{defaultPathSuffix}
{openSignal}
onPick={handlePick}
{trigger}
/>
frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte
+23
View File
@@ -0,0 +1,23 @@
<!--
@component
The "+" on an asset's bottom edge: a circle at rest, widened into a labelled pill
while its menu is open or while it is being dragged. The label animates its
max-width (width can't transition to auto) and never wraps.
-->
<script lang="ts">
import { Plus } from 'lucide-svelte'
let { expanded }: { expanded: boolean } = $props()
</script>
<span
class="group/add h-6 min-w-6 rounded-full flex items-center justify-center bg-surface border border-gray-400 dark:border-gray-600 text-secondary hover:text-primary hover:border-gray-500 shadow-sm leading-none overflow-hidden transition-[padding] duration-200 ease-out data-[open=true]:pl-1"
data-open={expanded}
>
<Plus size={16} strokeWidth={2.5} class="shrink-0" />
<span
class="max-w-0 opacity-0 whitespace-nowrap text-3xs font-normal transition-[max-width,opacity,margin] duration-200 ease-out group-data-[open=true]/add:max-w-32 group-data-[open=true]/add:opacity-100 group-data-[open=true]/add:ml-1 group-data-[open=true]/add:mr-2"
>
Add downstream step
</span>
</span>
frontend/src/lib/components/assets/AssetGraph/AddNode.svelte
+29 -138
View File
@@ -1,145 +1,36 @@
<script lang="ts">
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import {
Plus,
Clock,
Webhook,
Mail,
Zap,
Radio,
MessageSquare,
Database,
Send,
CloudCog,
Upload
} from 'lucide-svelte'
import type { ScriptLang } from '$lib/gen'
import type { NativeTriggerKind } from './types'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import type { PipelineOutputKind } from './pipelineTemplates'
// Each left-column kind is just "pipeline script triggered by <trigger
// source>". id === the SCRIPT_TRIGGER_KIND value, so the handler can
// dispatch on it uniformly. Asset-triggered scripts are not in this
// menu; those live under the per-asset + inside the graph.
type KindId = NativeTriggerKind
import AddDataSourceMenu from './AddDataSourceMenu.svelte'
import { Plus } from 'lucide-svelte'
import { NODE } from '$lib/components/graph/util'
interface Props {
data: {
onAddPipelineScript: (
language: ScriptLang,
path: string,
source: { kind: NativeTriggerKind; path: string | undefined },
outputKind: PipelineOutputKind,
aiPrompt?: string
) => void
pathPrefix: string
defaultPathSuffix: string
}
data: Pick<
import('svelte').ComponentProps<typeof AddDataSourceMenu>,
'onAddPipelineScript' | 'pathPrefix' | 'defaultPathSuffix'
>
}
let { data }: Props = $props()
function handlePick(pick: PipelineInsertPick) {
if (!pick.language || !pick.path) return
const kindId = pick.kindId as KindId
const outputKind = (pick.outputKind ?? 'none') as PipelineOutputKind
// Native trigger annotation is marker-only — the binding lives on
// the trigger row's own `script_path`, which the user creates
// separately. Seed with `path: undefined`.
data.onAddPipelineScript(
pick.language as ScriptLang,
pick.path,
{ kind: kindId, path: undefined },
outputKind,
pick.aiPrompt
)
}
</script>
<PipelineInsertMenu
kinds={[
{
id: 'data_upload',
label: 'On data upload',
description: 'UI-first: run by uploading a file via the S3 picker',
icon: Upload
},
{
id: 'schedule',
label: 'On schedule',
description: 'Triggered by a schedule you create',
icon: Clock
},
{
id: 'webhook',
label: 'On webhook',
description: 'Triggered by an HTTP webhook',
icon: Webhook
},
{
id: 'email',
label: 'On email',
description: 'Triggered by incoming email',
icon: Mail
},
{
id: 'kafka',
label: 'On Kafka',
description: 'Triggered by a Kafka message',
icon: Zap
},
{
id: 'mqtt',
label: 'On MQTT',
description: 'Triggered by an MQTT message',
icon: Radio
},
{
id: 'amqp',
label: 'On AMQP',
description: 'Triggered by an AMQP (RabbitMQ) message',
icon: Radio
},
{
id: 'nats',
label: 'On NATS',
description: 'Triggered by a NATS message',
icon: MessageSquare
},
{
id: 'postgres',
label: 'On Postgres',
description: 'Triggered by a Postgres event',
icon: Database
},
{
id: 'sqs',
label: 'On SQS',
description: 'Triggered by an SQS message',
icon: Send
},
{
id: 'gcp',
label: 'On GCP Pub/Sub',
description: 'Triggered by a Pub/Sub message',
icon: CloudCog
}
]}
languages={PIPELINE_LANGUAGES as any}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
onPick={handlePick}
>
{#snippet trigger()}
<!-- Quiet insert affordance, mirroring the flow editor's inline +
buttons (bg-surface + gray border + secondary text) — a filled
accent circle outweighed every real node on the canvas. -->
<button
type="button"
class="w-8 h-8 rounded-full flex items-center justify-center bg-surface border border-gray-400 dark:border-gray-600 text-secondary shadow-sm hover:bg-surface-hover transition-colors cursor-pointer"
title="Add to pipeline"
>
<Plus size={16} />
</button>
{/snippet}
</PipelineInsertMenu>
<!-- The layout gives this node a full node-width slot; centering the pill in
it lines it up with the nodes below whatever its label's width. -->
<div class="flex justify-center" style="width: {NODE.width}px;">
<AddDataSourceMenu
onAddPipelineScript={data.onAddPipelineScript}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
>
{#snippet trigger()}
<!-- Quiet insert affordance, mirroring the flow editor's inline +
buttons (bg-surface + gray border + secondary text) — a filled
accent pill would outweigh every real node on the canvas. -->
<button
type="button"
class="h-8 px-3 rounded-full flex items-center gap-1.5 whitespace-nowrap bg-surface border border-gray-400 dark:border-gray-600 text-xs font-normal text-secondary shadow-sm hover:bg-surface-hover transition-colors cursor-pointer"
>
<Plus size={16} class="shrink-0" />
Add data source
</button>
{/snippet}
</AddDataSourceMenu>
</div>
frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte
+1156 -73
View File
File diff suppressed because it is too large Load Diff
frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte
+229 -158
View File
@@ -20,10 +20,17 @@
SquareFunction,
Trash2,
X,
Pencil
Pencil,
ChevronDown,
Zap
} from 'lucide-svelte'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import { tick } from 'svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import {
upstreamScriptItems,
upstreamTriggerItems,
type UpstreamScript,
type UpstreamTriggerAction
} from './upstreamScriptItems'
import { inferArgs } from '$lib/infer'
import { emptySchema, sendUserToast } from '$lib/utils'
import type { Schema } from '$lib/common'
@@ -41,6 +48,7 @@
import { extractDraftMacros } from './resolveGraph'
import { assetColumnNodes, type ColumnLineageGraph } from './columnLineageGraph'
import SummaryPathDisplay from '$lib/components/SummaryPathDisplay.svelte'
import Badge from '$lib/components/common/badge/Badge.svelte'
import S3FilePreview from '$lib/components/S3FilePreview.svelte'
import DataTablePreview from './DataTablePreview.svelte'
import DucklakeAssetPanel from './DucklakeAssetPanel.svelte'
@@ -117,7 +125,9 @@
// (`onDraftPersist`). `onDraftPersist` stays the authoritative commit on
// navigate-away; this is the continuous feed that keeps the draft's
// server copy current while the user is still typing.
onContentChange?: (scriptPath: string | undefined, content: string) => void
// `base`: the deployed copy being edited, so the parent can autosave unsaved
// edits to it before they are promoted to a draft. Unset for a draft.
onContentChange?: (scriptPath: string | undefined, content: string, base?: Script) => void
// Fires when the user navigates away from a draft (selects another
// node, closes the pane, etc.) — the parent must persist the new
// content + write outputs into its drafts Map. Without this,
@@ -133,7 +143,8 @@
writes: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Body-inferred reads, so the parent's draft keeps its input
// lineage while inactive (no live inference runs for it).
reads: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Undefined when nothing was inferred: keep what the draft has.
reads?: { kind: AssetWithAltAccessType['kind']; path: string }[]
// Full edited script — set when the source is a persisted
// script (needed to seed a brand-new draft entry).
script?: Script
@@ -156,6 +167,12 @@
path: string
unsaved?: boolean
}>
/** The selected asset's producing scripts, for its "Edit script" action,
* which calls `onEditScript` with the one picked. */
producerScripts?: UpstreamScript[]
onEditScript?: (path: string) => void
/** The selected asset's upstream triggers that can be opened from here. */
producerTriggers?: UpstreamTriggerAction[]
// Pipeline-wide column-lineage graph (built by the parent page from the
// resolved graph). Drives the transitive column-lineage trace shown for a
// selected materialized asset.
@@ -215,15 +232,18 @@
// the edges. Counter rather than boolean so back-to-back runs
// re-fire the effect even if no other prop changed.
requestRunSignal?: number
// Folder-scoped non-editable prefix shown next to the suffix
// editor when the user renames a draft (e.g. `f/<folder>/`). The
// new path = pathPrefix + suffix.
pathPrefix?: string
// Called when the user renames a draft — the parent reseats the
// path key in its drafts map and updates activeDraftPath. Returns
// true on success; false on collision/validation failure so the
// popover can keep itself open and surface the error inline.
onDraftPathChange?: (oldPath: string, newPath: string) => boolean | string
onDraftMetaChange?: (
path: string,
meta: { summary: string; labels: string[] | undefined }
) => void
// The pipeline's folder prefix (`f/<folder>/`). A draft renamed outside it
// would be deployed out of the pipeline, so the rename is refused.
pathPrefix?: string
// Bumped by the parent (e.g. from the runnable-node action menu) to
// auto-open the archive/delete confirmation modal for the currently
// loaded persisted script. No-ops while the pane is showing a draft
@@ -280,6 +300,19 @@
// can persist a data-upload entry's staged input (drives node readiness).
// `isValid` is the full-schema validity, not just "file present".
onRunFormArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
// Show only the open script's inputs and runs (a data-upload click), not
// its editor. `onRunInputs` runs it with them, drafts included, and
// `onShowScript` leaves this view for the script.
inputsOnly?: boolean
/** `live` is the pane's buffer when it differs from what is stored: always
* for a draft, and for a deployed script edited before switching to the
* inputs view. A run with it previews that body. */
onRunInputs?: (
path: string,
args: Record<string, any>,
live?: { content: string; language: ScriptLang }
) => Promise<string | undefined>
onShowScript?: () => void
}
let {
selection,
@@ -301,6 +334,9 @@
onScriptRenamed,
onScriptRemoved,
selectionProducers = [],
producerScripts = [],
onEditScript,
producerTriggers = [],
selectionColumnGraph,
selectionColumnLoading = false,
selectionColumnTruncated = false,
@@ -315,8 +351,9 @@
activeRunnable,
onTestStateChange,
requestRunSignal,
pathPrefix = '',
onDraftPathChange,
onDraftMetaChange,
pathPrefix = '',
requestRemoveSignal,
downstreamSubscribers = 0,
onStartBoundedRun,
@@ -328,10 +365,15 @@
onRunByPath,
onRunCascadeByPath,
runFormInitialArgs,
onRunFormArgsChange
onRunFormArgsChange,
inputsOnly = false,
onRunInputs,
onShowScript
}: Props = $props()
let readOnly = $derived(mode !== 'edit')
// Someone who cannot switch the pipeline to edit mode only gets to read it.
let editScriptLabel = $derived(readOnly && !onRequestEdit ? 'Open script' : 'Edit script')
// Root element of the pane — used to scope the S3-input lookup for the
// data_upload focus effect.
@@ -408,7 +450,7 @@
// PipelineScriptView renders instead of the test panel.
let viewReadyForTarget = $derived.by(
() =>
readOnly &&
(readOnly || inputsOnly) &&
runTargetPath !== undefined &&
script?.path === runTargetPath &&
(isDraft || (!scriptRes.loading && scriptRes.current?.path === runTargetPath))
@@ -484,6 +526,12 @@
// alongside `liveBodyAssets` and forwarded so the live graph can show
// inferred column lineage on the edited script before it deploys.
let liveColumnLineage = $state<ColumnLineage[] | undefined>(undefined)
// The script whose editor last bound `liveBodyAssets` (undefined there means it
// inferred nothing). They stay bound after that editor unmounts, so a script
// that never had one (the inputs-only view) must not report or persist them as
// its lineage. Set by an effect declared after the save-on-switch one, whose
// cleanup still needs the previous owner when the selection moves.
let liveAssetsOwner: string | undefined = undefined
// Bumped when the runs panel reports a watched job has reached a
// terminal state. Drives S3FilePreview's refreshKey so the preview
@@ -549,7 +597,9 @@
let args = $state<Record<string, any>>({})
let argsSeedPath: string | undefined = undefined
$effect.pre(() => {
const p = script?.path
// Re-seeded on leaving the inputs-only view too, so the file picked there
// shows in the editor's test form.
const p = script?.path === undefined ? undefined : `${script.path}|${inputsOnly}`
if (p === argsSeedPath) return
argsSeedPath = p
args = runFormInitialArgs ? structuredClone($state.snapshot(runFormInitialArgs)) : {}
@@ -559,7 +609,8 @@
// required field (not just the S3 file) is satisfied.
let runFormIsValid = $state(true)
$effect(() => {
if (readOnly || !script) return
// Inputs-only mounts no ScriptEditor: PipelineScriptView reports its own args.
if (readOnly || inputsOnly || !script) return
onRunFormArgsChange?.(script.path, $state.snapshot(args), runFormIsValid)
})
@@ -596,18 +647,25 @@
b: Array<{ kind: string; path: string }>
) => a.length === b.length && a.every((x, i) => x.kind === b[i]?.kind && x.path === b[i]?.path)
return () => {
const writes = (liveBodyAssets ?? [])
// No editor inferred this script (the inputs-only view): keep its stored
// lineage rather than persisting it as empty.
const inferred = liveAssetsOwner === captured.path
const writes = !inferred
? (writesAtRegister ?? [])
: (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'w' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
const reads = (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'r' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
const reads = !inferred
? readsAtRegister
: (liveBodyAssets ?? [])
.filter((a) => {
const t = a.access_type ?? a.alt_access_type
return t === 'r' || t === 'rw'
})
.map((a) => ({ kind: a.kind, path: a.path }))
// Draft runs: emit only when content or lineage changed in THIS clone.
// An unconditional emit ping-pongs forever when the entry is rewritten
// externally while the pane stays mounted (rename rekey, AI edit):
@@ -617,6 +675,7 @@
(captured.content ?? '') === contentAtRegister &&
refsEq(writesAtRegister ?? [], writes) &&
readsAtRegister != undefined &&
reads != undefined &&
refsEq(readsAtRegister, reads)
)
return
@@ -651,12 +710,42 @@
}
})
$effect(() => {
if (script && !readOnly && !inputsOnly) liveAssetsOwner = script.path
})
let saving = $state(false)
// What this pane last deployed, so the persist-back cleanup can tell "the
// buffer equals the new deployed head" from real unsaved edits (plain
// variable: only read inside the untracked cleanup).
let deployedFromPane: { path: string; content: string } | undefined = undefined
let isDraft = $derived(draftScript != undefined)
function liveBody(): { content: string; language: ScriptLang } | undefined {
if (!script) return undefined
const live = { content: script.content ?? '', language: script.language as ScriptLang }
if (isDraft) return live
const orig = scriptRes.current
return orig?.path === script.path && (orig.content ?? '') !== live.content ? live : undefined
}
// A draft's stored schema is empty until an editor infers it, and the inputs
// view mounts none: infer from the body here so its fields (the file picker)
// show up.
const inputsSchema = resource(
[() => (inputsOnly && isDraft ? script?.content : undefined), () => script?.language],
async ([content, language]) => {
if (content === undefined || !language) return undefined
const schema = emptySchema()
try {
await inferArgs(language, content, schema as Schema)
} catch {
return undefined
}
return schema
}
)
let inputsScript = $derived(
script && inputsSchema.current ? { ...script, schema: inputsSchema.current } : undefined
)
// Single trash-bin button opens one modal that exposes both Archive
// (always available) and Delete permanently (admin-only). Archive is
@@ -763,12 +852,17 @@
onAnnotationsChange?.(script?.path, liveAnnotations)
})
$effect(() => {
if (readOnly) return
if (readOnly || (inputsOnly && liveAssetsOwner !== script?.path)) return
onAssetsChange?.(script?.path, liveBodyAssets ?? [], liveColumnLineage)
})
$effect(() => {
if (readOnly) return
onContentChange?.(script?.path, script?.content ?? '')
const base = draftScript ? undefined : scriptRes.current
onContentChange?.(
script?.path,
script?.content ?? '',
base && base.path === script?.path ? base : undefined
)
})
async function save() {
@@ -910,49 +1004,6 @@
JSON.stringify(script.schema ?? null) === JSON.stringify(orig.schema ?? null)
)
})
// Suffix editor for the draft-path popover. Seeded from the current
// path each time the popover opens so the user starts with what they
// see, not stale state from an earlier rename.
let draftPathSuffix = $state('')
let draftPathError = $state<string | undefined>(undefined)
let draftPathInput: HTMLInputElement | undefined = $state(undefined)
function suffixOf(fullPath: string): string {
return fullPath.startsWith(pathPrefix) ? fullPath.slice(pathPrefix.length) : fullPath
}
async function openDraftPathEditor() {
draftPathSuffix = script ? suffixOf(script.path) : ''
draftPathError = undefined
await tick()
draftPathInput?.focus()
draftPathInput?.select()
}
function confirmDraftPath(close: () => void) {
if (!script) return
const suffix = draftPathSuffix.trim()
if (!suffix) {
draftPathError = 'Path cannot be empty'
return
}
const newPath = pathPrefix + suffix
if (newPath === script.path) {
close()
return
}
const result = onDraftPathChange?.(script.path, newPath)
if (result === true || result === undefined) {
script.path = newPath
draftPathError = undefined
close()
} else if (typeof result === 'string') {
draftPathError = result
} else {
draftPathError = 'Path already in use'
}
}
</script>
<div class="flex flex-col h-full bg-surface" bind:this={paneEl}>
@@ -964,80 +1015,34 @@
{@const draftScriptPath = script.path}
<Code2 size={16} class="shrink-0 text-tertiary" />
{#if onDraftPathChange && !readOnly}
<!-- Inline rename popover for drafts. The persisted-script
branch uses SummaryPathDisplay which round-trips through
updateItemPathAndSummary; drafts have no server row yet,
so we just rekey the parent's drafts map locally. -->
<Popover
placement="bottom-start"
contentClasses="p-3"
usePointerDownOutside
on:openChange={(e) => {
if (e.detail) openDraftPathEditor()
<!-- Same summary/path popover as a deployed script, saved into the
drafts map instead of through the API: a draft has no row yet. -->
<SummaryPathDisplay
summary={script.summary ?? ''}
path={draftScriptPath}
labels={script.labels ?? []}
kind="script"
onSaved={() => {}}
saveOverride={({ path, summary, labels }) => {
if (pathPrefix && (!path.startsWith(pathPrefix) || path === pathPrefix)) {
return `A pipeline script stays in ${pathPrefix}: pick a name inside that folder`
}
// Before the rename, which carries the draft entry over as it is.
onDraftMetaChange?.(draftScriptPath, { summary, labels })
if (script) {
script.summary = summary
script.labels = labels
}
if (path === draftScriptPath) return undefined
const result = onDraftPathChange(draftScriptPath, path)
if (result !== true && result !== undefined) {
return typeof result === 'string' ? result : 'Path already in use'
}
if (script) script.path = path
return undefined
}}
>
{#snippet trigger()}
<button
type="button"
class="flex flex-col min-w-0 text-left px-2 py-1 rounded-md hover:bg-surface-hover transition-colors group"
title="Edit draft path"
>
<span
class="text-3xs uppercase tracking-wide text-tertiary flex items-center gap-1"
>
Draft pipeline script
<Pencil size={9} class="opacity-0 group-hover:opacity-60 transition-opacity" />
</span>
<span class="text-xs font-mono truncate" title={draftScriptPath}
>{draftScriptPath}</span
>
</button>
{/snippet}
{#snippet content({ close })}
<div class="flex flex-col gap-2 w-[420px]">
<span class="text-2xs font-normal text-secondary">Path</span>
<div
class="flex items-stretch border rounded-md bg-surface overflow-hidden focus-within:ring-2 focus-within:ring-blue-400"
>
{#if pathPrefix}
<span
class="flex items-center px-2 bg-surface-secondary text-tertiary text-sm font-mono border-r select-none"
>
{pathPrefix}
</span>
{/if}
<input
bind:this={draftPathInput}
bind:value={draftPathSuffix}
onkeydown={(e) => {
if (e.key === 'Enter') {
e.preventDefault()
confirmDraftPath(close)
} else if (e.key === 'Escape') {
e.preventDefault()
close()
}
}}
class="flex-1 min-w-0 px-2 py-1.5 text-sm font-mono bg-transparent focus:outline-none"
placeholder="my_script"
/>
</div>
{#if draftPathError}
<span class="text-2xs text-red-500">{draftPathError}</span>
{/if}
<div class="flex justify-end">
<Button
variant="accent"
unifiedSize="sm"
disabled={!draftPathSuffix.trim()}
onClick={() => confirmDraftPath(close)}
>
Rename
</Button>
</div>
</div>
{/snippet}
</Popover>
/>
<Badge color="gray">Draft</Badge>
{:else}
<div class="flex flex-col min-w-0">
<span class="text-3xs uppercase tracking-wide text-tertiary">
@@ -1097,6 +1102,66 @@
{/if}
</div>
<div class="flex items-center gap-1 shrink-0">
{#if selection?.kind === 'asset' && onEditScript && producerScripts.length === 1}
<Button
variant="default"
unifiedSize="sm"
startIcon={{ icon: Pencil }}
onclick={() => onEditScript(producerScripts[0].path)}
title={`Open ${producerScripts[0].path}, which builds this asset`}
>
{editScriptLabel}
</Button>
{:else if selection?.kind === 'asset' && onEditScript && producerScripts.length > 1}
<DropdownV2
items={upstreamScriptItems(producerScripts, onEditScript)}
placement="bottom-end"
enableFlyTransition
>
{#snippet buttonReplacement()}
<Button
nonCaptureEvent
variant="default"
unifiedSize="sm"
startIcon={{ icon: Pencil }}
endIcon={{ icon: ChevronDown }}
title={`${producerScripts.length} scripts build this asset`}
>
{editScriptLabel}
</Button>
{/snippet}
</DropdownV2>
{/if}
{#if selection?.kind === 'asset' && producerTriggers.length === 1}
<Button
variant="default"
unifiedSize="sm"
startIcon={{ icon: Zap }}
onclick={() => producerTriggers[0].onOpen()}
title={producerTriggers[0].detail}
>
{producerTriggers[0].label}
</Button>
{:else if selection?.kind === 'asset' && producerTriggers.length > 1}
<DropdownV2
items={upstreamTriggerItems(producerTriggers)}
placement="bottom-end"
enableFlyTransition
>
{#snippet buttonReplacement()}
<Button
nonCaptureEvent
variant="default"
unifiedSize="sm"
startIcon={{ icon: Zap }}
endIcon={{ icon: ChevronDown }}
title={`${producerTriggers.length} triggers start the scripts that build this asset`}
>
Triggers
</Button>
{/snippet}
</DropdownV2>
{/if}
{#if readOnly && onRequestEdit}
<Button
variant="accent"
@@ -1118,21 +1183,10 @@
title="Discard draft"
/>
{/if}
<!-- Action order, left → right: trash, external link, save, close.
Save sits closest to Close so the primary commit action is
anchored at the right edge of the bar; trash lives at the
far left so destructive ops are visually separated from
navigation/commit. Mirrors the draft Discard placement. -->
{#if !readOnly && !isDraft && isScriptView && script?.hash}
<Button
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: Trash2 }}
onclick={() => (removeOpen = true)}
iconOnly
title="Archive or delete"
/>
{/if}
<!-- Action order, left → right: external link, save, close. Save sits
closest to Close so the primary commit action is anchored at the
right edge of the bar. Removing a script goes through its node's
menu. -->
{#if !readOnly && !isDraft && selection?.kind === 'runnable'}
<Button
variant="subtle"
@@ -1338,14 +1392,31 @@
{/if}
</div>
{:else if !isDraft && scriptRes.loading && !script}
<div class="absolute inset-0 flex items-center justify-center gap-2 text-tertiary">
<Loader2 size={16} class="animate-spin" />
<span class="text-xs">Loading script…</span>
</div>
<!-- Blank while the script loads: the fetch is short, and a spinner only flashes. -->
{:else if !isDraft && scriptRes.error}
<div class="p-3 text-xs text-red-500">
Failed to load: {scriptRes.error.message}
</div>
{:else if script && inputsOnly}
{#key script.path}
<PipelineScriptView
{isDraft}
canRun
runsDrafts
inputsOnly
{onShowScript}
onRun={onRunInputs && ((path, args) => onRunInputs(path, args, liveBody()))}
script={inputsScript ?? script}
{runsRefreshKey}
{runsPendingJobId}
initialArgs={runFormInitialArgs}
onArgsChange={onRunFormArgsChange}
onRunCompleted={() => {
previewRefreshKey += 1
onRunCompleted?.()
}}
/>
{/key}
{:else if script && readOnly}
<!-- Read-only modes: no Monaco/ScriptEditor (operators are
backend-blocked from previews anyway) — highlighted source,
frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte
+32 -22
View File
@@ -1,6 +1,7 @@
<script lang="ts">
import { BaseEdge, getBezierPath, type EdgeProps } from '@xyflow/svelte'
import { NODE } from '$lib/components/graph/util'
import { PIPELINE_NODE_HEIGHT } from './assetGraphLayout'
import { FlaskConical, Columns3, SquareFunction, BellOff } from 'lucide-svelte'
import type { ColumnLineage, DataTest } from './parsePipelineAnnotations'
@@ -103,33 +104,42 @@
// straight under the nodes in between — the child-and-grandchild-of-the-
// same-node case, where the tidy-tree layout puts all three on one
// column. Detour those through the gutter beside the column.
const SKIP_DY = NODE.gap.vertical * 2 + NODE.height
const SKIP_DY = NODE.gap.vertical * 2 + PIPELINE_NODE_HEIGHT
const NEAR_VERTICAL_DX = NODE.width / 4
// Lane offset: half a node plus a margin that stays inside the
// inter-column gutter (gap.horizontal / 2).
const LANE = NODE.width / 2 + NODE.gap.horizontal / 2
const CORNER = 14
// Rounded-orthogonal detour: drop out of the source, swing into the
// gutter lane at `midX`, run it down past the intermediate rows, swing
// back into the target.
// Detour through the gutter lane at `midX`, drawn with the same curves as
// every other edge: curve out of the source into the lane within the gap
// below it, run down the lane past the rows in between, curve into the
// target within the gap above it.
function gutterPath(midX: number): string {
const yTop = sourceY + 24
const yBot = targetY - 24
const outDir = midX > sourceX ? 1 : -1
const inDir = targetX > midX ? 1 : -1
return [
`M ${sourceX} ${sourceY}`,
`L ${sourceX} ${yTop - CORNER}`,
`Q ${sourceX} ${yTop} ${sourceX + outDir * CORNER} ${yTop}`,
`L ${midX - outDir * CORNER} ${yTop}`,
`Q ${midX} ${yTop} ${midX} ${yTop + CORNER}`,
`L ${midX} ${yBot - CORNER}`,
`Q ${midX} ${yBot} ${midX + inDir * CORNER} ${yBot}`,
`L ${targetX - inDir * CORNER} ${yBot}`,
`Q ${targetX} ${yBot} ${targetX} ${yBot + CORNER}`,
`L ${targetX} ${targetY}`
].join(' ')
const bend = NODE.gap.vertical
const yTop = sourceY + bend
// Clamp only downward edges: an upward (feedback) edge must still run up the
// lane, or its last curve cuts across the node the detour was clearing.
const yBot = targetY > sourceY ? Math.max(yTop, targetY - bend) : targetY - bend
const [out] = getBezierPath({
sourceX,
sourceY,
sourcePosition,
targetX: midX,
targetY: yTop,
targetPosition,
curvature: 0.25
})
const [into] = getBezierPath({
sourceX: midX,
sourceY: yBot,
sourcePosition,
targetX,
targetY,
targetPosition,
curvature: 0.25
})
// Continue from the lane rather than lifting the pen.
return `${out} L${midX},${yBot} ${into.replace(/^M/, 'L')}`
}
// Long edges bend out of the source within the layer gap and descend in
@@ -178,7 +188,7 @@
path={edgePath}
{markerEnd}
{style}
interactionWidth={0}
interactionWidth={12}
label={undefined}
labelStyle={undefined}
/>
frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte
+568 -169
View File
@@ -2,9 +2,14 @@
import { Handle, Position } from '@xyflow/svelte'
import { twMerge } from 'tailwind-merge'
import AssetGenericIcon from '$lib/components/icons/AssetGenericIcon.svelte'
import { formatShortAssetPath, type AssetKind } from '$lib/components/assets/lib'
import { NODE } from '$lib/components/graph/util'
import PipelineInsertMenu, { type PipelineInsertPick } from './PipelineInsertMenu.svelte'
import { formatAssetKind, formatShortAssetPath, type AssetKind } from '$lib/components/assets/lib'
import PipelineNodeCard from './PipelineNodeCard.svelte'
import NodeActionsMenu from './NodeActionsMenu.svelte'
import NodeFixButton, { type NodeFix } from './NodeFixButton.svelte'
import NodeOnDot from './NodeOnDot.svelte'
import { PIPELINE_NODE_HEADER } from './assetGraphLayout'
import AddDownstreamMenu from './AddDownstreamMenu.svelte'
import AddDownstreamPill from './AddDownstreamPill.svelte'
import {
ArrowUpRight,
Code2,
@@ -12,19 +17,28 @@
History,
Play,
Loader2,
Plus,
ShieldCheck,
ShieldAlert,
CheckCircle2,
ChevronDown,
Pencil,
CircleSlash,
Trash2,
Upload,
XCircle
} from 'lucide-svelte'
import { Button } from '$lib/components/common'
import type { ScriptLang } from '$lib/gen'
import { enterpriseLicense } from '$lib/stores'
import { sendUserToast } from '$lib/utils'
import { PIPELINE_LANGUAGES } from './pipelineLanguages'
import { sendUserToast, type Item } from '$lib/utils'
import type { PipelineOutputKind } from './pipelineTemplates'
import type { DbtAssetProvenance } from './types'
import DbtIcon from '$lib/components/icons/DbtIcon.svelte'
import LanguageIcon from '$lib/components/common/languageIcons/LanguageIcon.svelte'
import RunStateChip from './RunStateChip.svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import { upstreamScriptItems } from './upstreamScriptItems'
import type { RunnableRunState } from './activeRunnables.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
const operatingWorkspace = useOperatingWorkspace()
@@ -33,6 +47,49 @@
// `content` / `language` so the page-level run handler can dispatch to
// `runScriptPreview` instead of `runScriptByPath` (which 404s for
// non-deployed scripts).
/** What starts a producing script, as its chip shows it. */
export type TriggerChipData = {
label: string
/** The native trigger kind; a data upload renders as a call to action. */
kind?: string
/** No chip: the node's incoming edges already show what starts it. */
hidden?: boolean
missing?: boolean
draft?: boolean
/** Opens the trigger's editor; unset for what has none (manual, asset change). */
onOpen?: () => void
selected?: boolean
}
export type AssetUpstreamChips =
| { none: true }
| {
multiple: true
/** The producing scripts; one when only its triggers are ambiguous. */
scripts: Array<{
runnableId: string
path: string
summary?: string
language?: ScriptLang
unsaved?: boolean
onOpen: () => void
}>
/** The scripts' own triggers, each named with its script. */
triggers: Array<TriggerChipData & { script: string }>
selected?: boolean
}
| {
runnableId: string
path: string
summary?: string
language?: ScriptLang
unsaved?: boolean
runState?: RunnableRunState
trigger: TriggerChipData
onOpen: () => void
selected?: boolean
}
export type AssetProducer = {
kind: 'script' | 'flow'
path: string
@@ -51,6 +108,8 @@
// "current view of <dim>" marker so it reads as a derived node, not an
// unrelated table.
derived_from?: string
// Why the node is misconfigured, when it is: renders it red.
error?: string
// dbt provenance when this warehouse table is a dbt node: which model
// it is, how dbt materializes it, its tags and its generic tests.
dbt?: DbtAssetProvenance
@@ -65,10 +124,14 @@
language: ScriptLang,
scriptPath: string,
outputKind: PipelineOutputKind,
aiPrompt?: string
aiPrompt?: string,
options?: import('./PipelineInsertMenu.svelte').PipelineInsertOptions
) => void
pathPrefix?: string
defaultPathSuffix?: string
/** Starts dragging the "+" onto a script (subscribing it to this asset)
* or onto the blank canvas (opening the add menu there). */
onStartFeedDrag?: (e: PointerEvent) => void
// Producer scripts/flows that write to this asset, supplied by
// the canvas from the graph's write/rw edges. Drives the on-hover
// "Run" button. Includes unsaved/draft producers so the button
@@ -99,6 +162,41 @@
// instead of only settling once the job ends.
runStatus?: 'running' | 'materialized' | 'failed'
runRowCount?: number | null
/** Set in the assets-only view, where the producing script and its
* trigger are folded into the asset: `none` for an asset nothing in the
* pipeline writes, `multiple` when the producer or its trigger is
* ambiguous. */
upstream?: AssetUpstreamChips
/** What is wrong with the node, and how to fix it: replaces the "+". */
fix?: NodeFix
/** Assets-only: errors of the scripts and triggers folded into this node. */
foldedFixes?: NodeFix[]
/** Assets-only: starts dragging the top dot onto an asset, subscribing the
* scripts that build this one to it. */
onStartOnDrag?: (e: PointerEvent) => void
/** The assets-only node for the scripts that build no asset. */
noAsset?: boolean
/** The No-asset node's script, by its file name. */
noAssetName?: string
/** Assets-only: the node is only its script's banner. */
bannerOnly?: boolean
/** Assets-only: a script folded into the node has an error. */
scriptError?: boolean
/** No script in the workspace writes it; a flow's write fires no asset trigger. */
neverWritten?: boolean
/** Card width in the assets-only view, sized to its text. */
width?: number
/** Assets-only view: the producing script and trigger no other asset
* shares, which the menu offers to delete one by one or together. */
upstreamDelete?: import('./assetsOnlyView').AssetUpstreamDelete
/** When no combined delete is possible: the folded scripts, each deletable
* on its own. */
scriptDeletes?: Array<{ path: string; unsaved: boolean }>
triggerDeletes?: Array<NonNullable<import('./assetsOnlyView').AssetUpstreamDelete['trigger']>>
onDeleteUpstream?: (target: import('./assetsOnlyView').AssetUpstreamDelete) => void
deleteVerb?: 'Delete' | 'Archive'
/** Why nothing can be deleted, when the producer is shared. */
deleteBlocked?: string
}
// SvelteFlow injects this on the node component when the user clicks
// the node. Combined with our own `hovered` state to drive the
@@ -140,19 +238,94 @@
}
}
function handlePick(pick: PipelineInsertPick) {
if (pick.kindId === 'pipeline_script' && pick.language && pick.path) {
data.onAddScript?.(
{ kind: data.asset_kind, path: data.path },
pick.language as ScriptLang,
pick.path,
(pick.outputKind ?? 'none') as PipelineOutputKind,
pick.aiPrompt
)
}
}
let showAdd = $derived(data.onAddScript != undefined)
// A red asset needs fixing where its error comes from, not a downstream step;
// and a step downstream of what nothing writes would never run on its writes.
let showAdd = $derived(data.onAddScript != undefined && !data.error && !data.neverWritten)
let menuItems: Item[] = $derived(
data.onDeleteUpstream && data.upstreamDelete
? deleteItems(data.upstreamDelete, data.onDeleteUpstream)
: data.onDeleteUpstream && (data.scriptDeletes?.length || data.triggerDeletes?.length)
? [
...(data.scriptDeletes ?? []).map((script) => ({ script })),
...(data.triggerDeletes ?? []).map((trigger) => ({ trigger }))
].flatMap((t) => deleteItems(t, data.onDeleteUpstream!))
: data.deleteBlocked
? [
{
displayName: `Can't delete: ${data.deleteBlocked}`,
icon: Trash2,
disabled: true
}
]
: []
)
const lastSegment = (p: string) => p.split('/').pop() ?? p
// A script is archived rather than deleted when the user may not delete it;
// a trigger is always deleted.
function deleteItems(
t: import('./assetsOnlyView').AssetUpstreamDelete,
run: (target: import('./assetsOnlyView').AssetUpstreamDelete) => void
): Item[] {
const items: Item[] = []
if (t.script) {
const script = t.script
items.push({
displayName: `${data.deleteVerb ?? 'Delete'} script ${lastSegment(script.path)}`,
icon: Trash2,
type: 'delete' as const,
action: () => run({ script })
})
}
if (t.trigger) {
const trigger = t.trigger
items.push({
displayName: `Delete ${trigger.kind} trigger ${lastSegment(trigger.path)}`,
icon: Trash2,
type: 'delete' as const,
action: () => run({ trigger })
})
}
if (t.script && t.trigger) {
items.push({
displayName: `${data.deleteVerb === 'Archive' ? 'Remove' : 'Delete'} all 2 items`,
icon: Trash2,
type: 'delete' as const,
action: () => run(t)
})
}
return items
}
let upstream = $derived(data.upstream)
let HEADER_SELECTED = $derived(
data.scriptError
? 'bg-red-200 dark:bg-red-600 hover:bg-red-200 dark:hover:bg-red-600'
: 'bg-surface-accent-selected hover:bg-surface-accent-selected text-accent'
)
const CHIP_SELECTED =
'bg-surface-accent-selected border-border-selected hover:bg-surface-accent-selected text-accent'
let producer = $derived(upstream && 'path' in upstream ? upstream : undefined)
let coProducers = $derived(upstream && 'scripts' in upstream ? upstream.scripts : [])
let hasHeader = $derived(!!producer || coProducers.length > 0)
// The corner controls sit on the card's corner, below the header tab:
// -top-2 plus PIPELINE_NODE_HEADER (22px).
const CARD_CORNER_TOP = 'top-[14px]'
let coSelected = $derived(!!(upstream && 'scripts' in upstream && upstream.selected))
let coProducerItems = $derived(
upstreamScriptItems(coProducers, (path) => coProducers.find((s) => s.path === path)?.onOpen())
)
let scriptMenuOpen = $state(false)
let coTriggers = $derived(upstream && 'triggers' in upstream ? upstream.triggers : [])
let coTriggerItems: Item[] = $derived(
coTriggers.map((t) => ({
displayName: `${t.label}${t.draft ? ' · draft' : ''}`,
description: coProducers.length > 1 ? t.script : undefined,
disabled: !t.onOpen,
action: () => t.onOpen?.()
}))
)
let triggerMenuOpen = $state(false)
// dbt badge. `materialized` is dbt's own word rather than the Windmill
// strategy because `view` and `ephemeral` have no strategy, and showing the
@@ -220,6 +393,185 @@
)
</script>
{#snippet scriptHeader()}
<!-- The script building this asset, folded into it: a strip across the top
naming it, which opens it. Several scripts open a menu to pick one. -->
{@const one = producer ?? (coProducers.length === 1 ? coProducers[0] : undefined)}
{@const headerClass = twMerge(
'flex items-center gap-1.5 w-full min-w-0 px-2 text-3xs font-normal leading-none text-secondary hover:bg-surface-hover hover:text-primary',
data.scriptError &&
'bg-red-100 dark:bg-red-700 text-red-700 dark:text-red-200 hover:bg-red-200 dark:hover:bg-red-600 hover:text-red-800'
)}
{#if one}
{@const isSelected = producer ? !!producer.selected : coSelected}
<button
type="button"
class={twMerge(headerClass, isSelected && HEADER_SELECTED)}
title={`Open ${one.summary ? `${one.summary} (${one.path})` : one.path}${one.unsaved ? ' · draft' : ''}`}
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
onclick={(e) => {
e.stopPropagation()
one.onOpen()
}}
>
{#if one.language}
<LanguageIcon lang={one.language} width={11} height={11} />
{:else}
<Code2 size={11} class="shrink-0" />
{/if}
<span class="truncate">{one.summary || one.path}</span>
{#if one.unsaved}<span class="shrink-0 text-tertiary">draft</span>{/if}
{#if producer?.runState}
<RunStateChip runState={producer.runState} class="ml-auto h-4 px-1 rounded" />
{/if}
<Pencil size={10} class={twMerge('shrink-0 opacity-70', !producer?.runState && 'ml-auto')} />
</button>
{:else if coProducers.length > 1}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="w-full min-w-0 flex"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<DropdownV2
items={coProducerItems}
placement="bottom-start"
bind:open={scriptMenuOpen}
fixedHeight={false}
usePointerDownOutside
enableFlyTransition
class="w-full"
>
{#snippet buttonReplacement()}
<span
class={twMerge(headerClass, 'h-full', (coSelected || scriptMenuOpen) && HEADER_SELECTED)}
title={`Built by ${coProducers.length} scripts: pick one to open`}
>
{#each coProducers.slice(0, 2) as sc (sc.path)}
{#if sc.language}
<LanguageIcon lang={sc.language} width={11} height={11} />
{:else}
<Code2 size={11} class="shrink-0" />
{/if}
{/each}
<span class="truncate">{coProducers.length} scripts</span>
<ChevronDown size={11} class="shrink-0 ml-auto" />
</span>
{/snippet}
</DropdownV2>
</div>
{/if}
{/snippet}
{#snippet upstreamRow()}
{#if data.foldedFixes?.length}
<!-- A folded script or trigger is misconfigured: its Fix, one at a time. -->
<NodeFixButton
chip
fix={data.foldedFixes[0]}
label={`${data.foldedFixes.length} error${data.foldedFixes.length === 1 ? '' : 's'} · Fix`}
/>
{:else if upstream && coTriggers.length > 1 && !coTriggers.some((t) => t.onOpen)}
<!-- Several triggers, none editable here (View mode): a plain label. -->
<span
class="truncate rounded-md border px-1.5 py-1 text-3xs leading-none font-normal border-gray-300 dark:border-gray-600 text-secondary"
>Multiple triggers</span
>
{:else if upstream && coTriggers.length > 1}
<!-- Several triggers start what builds this asset: one chip, a menu to pick
the one to edit. -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="min-w-0 flex"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<DropdownV2
items={coTriggerItems}
placement="bottom-start"
bind:open={triggerMenuOpen}
fixedHeight={false}
usePointerDownOutside
enableFlyTransition
>
{#snippet buttonReplacement()}
<span
class={twMerge(
'flex items-center gap-1 min-w-0 rounded-md border px-1.5 py-1 text-3xs leading-none font-normal border-gray-300 dark:border-gray-600 text-secondary bg-surface hover:bg-surface-hover cursor-pointer',
(triggerMenuOpen || coTriggers.some((t) => t.selected)) && CHIP_SELECTED
)}
title="Started by several triggers: pick one to edit"
>
<span class="truncate">Multiple triggers</span>
<ChevronDown size={9} class="shrink-0 opacity-70" />
</span>
{/snippet}
</DropdownV2>
</div>
{:else if upstream}
{@const trigger = producer?.trigger ?? coTriggers[0]}
{@const triggerScript = producer?.path ?? coTriggers[0]?.script}
{@const chipClass = twMerge(
'truncate rounded-md border px-1.5 py-1 text-3xs leading-none font-normal',
trigger?.missing
? 'border-red-300 dark:border-red-600 text-red-700 dark:text-red-300'
: 'border-gray-300 dark:border-gray-600 text-secondary',
trigger?.draft && 'border-dashed',
trigger?.onOpen && 'bg-surface hover:bg-surface-hover cursor-pointer',
trigger?.selected && CHIP_SELECTED
)}
{#if trigger?.kind === 'data_upload' && trigger.onOpen}
<!-- The one trigger that needs the user: nothing runs until a file is
provided, so it reads as the node's main action rather than a chip. -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<span
class="flex min-w-0"
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
>
<Button
variant="accent"
unifiedSize="sm"
startIcon={{ icon: Upload }}
title={`Upload the data ${triggerScript} runs on`}
onClick={(e) => {
e?.stopPropagation()
trigger.onOpen?.()
}}
>
Upload data
</Button>
</span>
{:else if trigger?.onOpen}
<button
type="button"
class={twMerge(chipClass, 'flex items-center gap-1 min-w-0')}
title={trigger.missing
? `No ${trigger.label} trigger targets ${triggerScript} yet: click to create one`
: `Edit the ${trigger.label} trigger`}
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
onclick={(e) => {
e.stopPropagation()
trigger.onOpen?.()
}}
>
<span class="truncate">{trigger.label}{trigger.draft ? ' · draft' : ''}</span>
{#if !trigger.missing}<Pencil size={9} class="shrink-0 opacity-70" />{/if}
</button>
{:else}
<span class={chipClass}>
{#if 'none' in upstream}
External source
{:else if trigger}
{trigger.label}{trigger.draft ? ' · draft' : ''}
{/if}
</span>
{/if}
{/if}
{/snippet}
<!-- onmouseenter/leave on the wrapper (not the inner card) so the run
button — which floats outside the card — keeps the hover state alive
when the cursor moves between the card and the button. -->
@@ -236,123 +588,169 @@
<div class="wm-recompute-flash pointer-events-none absolute inset-0 z-10 rounded-md"></div>
{/key}
{/if}
<!-- Mirrors the flow editor's asset pill: quiet surface + gray border at
rest, accent reserved for the selected state. -->
<div
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden border',
'bg-surface-secondary border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500 transition-colors',
selected && 'bg-surface-accent-selected border-border-selected'
<NodeActionsMenu
items={menuItems}
hover={hovered}
kebabClass={twMerge(
showGuardBadge && '-right-8',
hasHeader && !data.bannerOnly && CARD_CORNER_TOP
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={data.path}
>
<!-- Data identity carries the accent (luminance blue), pairing with
the blue write edges that produce these assets — scripts stay
neutral, so script vs data reads at a glance. -->
<AssetGenericIcon
assetKind={data.asset_kind}
class={`shrink-0 ml-2 mr-2 ${selected ? 'text-accent' : 'text-blue-600 dark:text-blue-400'}`}
size="14px"
/>
{#if data.runStatus}
<!-- The run in view, per relation: a spinner while its producer is
building it, then its outcome. Left of the name so the eye finds
the moving nodes first on a wide graph. -->
<span
class="shrink-0 mr-1 {data.runStatus === 'failed'
? 'text-red-600 dark:text-red-400'
: data.runStatus === 'materialized'
? 'text-green-600 dark:text-green-400'
: 'text-blue-600 dark:text-blue-400'}"
title={data.runStatus}
{#if data.bannerOnly}
<!-- A script that builds nothing and runs only on writes to what it reads:
the incoming edges say how it starts, so its banner is the whole node. -->
<div
class={twMerge(
'flex items-stretch min-w-0 rounded-md overflow-hidden bg-surface border border-gray-200 dark:border-gray-700 drop-shadow-sm',
(producer ? producer.selected : coSelected) &&
'border-border-selected dark:border-border-selected'
)}
style="width: {data.width ?? 160}px; height: {PIPELINE_NODE_HEADER}px;"
>
{#if data.runStatus === 'running'}
<Loader2 size={11} class="animate-spin" />
{:else if data.runStatus === 'failed'}
<XCircle size={11} />
{@render scriptHeader()}
</div>
{:else}
<PipelineNodeCard
kindLabel={data.noAsset ? 'No asset' : formatAssetKind(asset)}
title={data.noAsset ? (data.noAssetName ?? '') : formatShortAssetPath(asset)}
tooltip={data.noAsset
? 'This script builds no asset'
: data.error
? `${data.path}: ${data.error}`
: data.path}
{selected}
tone={data.error ? 'error' : undefined}
subtitle={data.foldedFixes?.length ||
(upstream &&
!producer?.trigger?.hidden &&
!('triggers' in upstream && upstream.triggers.length === 0))
? upstreamRow
: undefined}
header={hasHeader ? scriptHeader : undefined}
headerTone={data.scriptError ? 'error' : undefined}
headerSelected={producer ? !!producer.selected : coSelected}
width={data.width}
surface={upstream ? 'primary' : 'secondary'}
>
{#snippet icon()}
<!-- Data identity carries the accent (luminance blue), pairing with
the blue write edges that produce these assets — scripts stay
neutral, so script vs data reads at a glance. -->
{#if data.noAsset}
<CircleSlash size={14} class="text-tertiary" />
{:else}
<CheckCircle2 size={11} />
<AssetGenericIcon
assetKind={data.asset_kind}
class={data.error
? 'text-red-600 dark:text-red-200'
: selected
? 'text-accent'
: 'text-blue-600 dark:text-blue-400'}
size="14px"
/>
{/if}
</span>
<!-- Rows the run wrote. Recorded per relation already, and the cheapest
answer to "did this model actually produce anything" — a model that
built green but emitted 0 rows is the failure that looks like a
success. Only once settled: mid-build the number is not yet real. -->
{#if data.runStatus === 'materialized' && data.runRowCount != undefined}
<span
class="shrink-0 mr-1 text-3xs tabular-nums text-tertiary"
title="{data.runRowCount} rows written by this run"
>
{Intl.NumberFormat().format(data.runRowCount)}
</span>
{/if}
{/if}
<span class="flex-1 min-w-0 pr-1 py-0.5 text-2xs font-mono text-emphasis truncate">
{formatShortAssetPath(asset)}
</span>
<!-- Fork data-environment chip: in a fork every asset shares its parent's
name, so the env it resolves to must read at a glance. Labeled + tinted
(amber "parent" = deferred read of the parent's current table via a
view; emerald "fork" = the fork's own materialized copy) rather than a
bare icon, which was too easy to miss. The title carries the detail. -->
{#if data.fork_materialization === 'deferred'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300 border border-amber-300 dark:border-amber-700"
title="Deferred to parent workspace: reads the parent's current data. Materialize it in this fork to iterate on it."
>
<ArrowUpRight size={10} />
parent
</span>
{:else if data.fork_materialization === 'fork'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300 border border-emerald-300 dark:border-emerald-700"
title="Materialized in this fork: reads and writes use the fork's isolated copy."
>
<GitFork size={10} />
fork
</span>
{/if}
<!-- dbt chip: names the materialization dbt declares, plus a test count
when the model carries generic tests. Orange keeps it visually
separate from the fork/SCD2 chips, which describe Windmill state.
Where the project node is on the graph it also stands in for the edge
to it: hovering lights that node up, clicking selects it. Where it is
not — the run page, and a pipeline holding a dbt project — there is
nothing to point at, so the chip renders inert. -->
{#if data.dbt}
<button
type="button"
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold tracking-wide bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300 border border-orange-300 dark:border-orange-700 {data.onDbtSelect
? 'hover:brightness-95 cursor-pointer'
: 'cursor-default'}"
title={dbtTitle}
onmouseenter={() => data.onDbtHover?.(true)}
onmouseleave={() => data.onDbtHover?.(false)}
onclick={(e) => {
e.stopPropagation()
data.onDbtSelect?.()
}}
>
<DbtIcon width={9} height={9} />
{dbtLabel}
{#if data.dbt.data_tests?.length}
<span class="opacity-70">&middot; {data.dbt.data_tests.length}T</span>
{/snippet}
{#snippet leading()}
{#if data.runStatus}
<!-- The run in view, per relation: a spinner while its producer is
building it, then its outcome. Left of the name so the eye finds
the moving nodes first on a wide graph. -->
<span
class="shrink-0 mr-1 {data.runStatus === 'failed'
? 'text-red-600 dark:text-red-400'
: data.runStatus === 'materialized'
? 'text-green-600 dark:text-green-400'
: 'text-blue-600 dark:text-blue-400'}"
title={data.runStatus}
>
{#if data.runStatus === 'running'}
<Loader2 size={11} class="animate-spin" />
{:else if data.runStatus === 'failed'}
<XCircle size={11} />
{:else}
<CheckCircle2 size={11} />
{/if}
</span>
<!-- Rows the run wrote. Recorded per relation already, and the cheapest
answer to "did this model actually produce anything" — a model that
built green but emitted 0 rows is the failure that looks like a
success. Only once settled: mid-build the number is not yet real. -->
{#if data.runStatus === 'materialized' && data.runRowCount != undefined}
<span
class="shrink-0 mr-1 text-3xs tabular-nums text-tertiary"
title="{data.runRowCount} rows written by this run"
>
{Intl.NumberFormat().format(data.runRowCount)}
</span>
{/if}
{/if}
</button>
{/snippet}
{#snippet trailing()}
<!-- Fork data-environment chip: in a fork every asset shares its parent's
name, so the env it resolves to must read at a glance. Labeled + tinted
(amber "parent" = deferred read of the parent's current table via a
view; emerald "fork" = the fork's own materialized copy) rather than a
bare icon, which was too easy to miss. The title carries the detail. -->
{#if data.fork_materialization === 'deferred'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300 border border-amber-300 dark:border-amber-700"
title="Deferred to parent workspace: reads the parent's current data. Materialize it in this fork to iterate on it."
>
<ArrowUpRight size={10} />
parent
</span>
{:else if data.fork_materialization === 'fork'}
<span
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold uppercase tracking-wide bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300 border border-emerald-300 dark:border-emerald-700"
title="Materialized in this fork: reads and writes use the fork's isolated copy."
>
<GitFork size={10} />
fork
</span>
{/if}
<!-- dbt chip: names the materialization dbt declares, plus a test count
when the model carries generic tests. Orange keeps it visually
separate from the fork/SCD2 chips, which describe Windmill state.
Where the project node is on the graph it also stands in for the edge
to it: hovering lights that node up, clicking selects it. Where it is
not — the run page, and a pipeline holding a dbt project — there is
nothing to point at, so the chip renders inert. -->
{#if data.dbt}
<button
type="button"
class="shrink-0 mr-1.5 flex items-center gap-0.5 rounded px-1 py-px text-3xs font-semibold tracking-wide bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300 border border-orange-300 dark:border-orange-700 {data.onDbtSelect
? 'hover:brightness-95 cursor-pointer'
: 'cursor-default'}"
title={dbtTitle}
onmouseenter={() => data.onDbtHover?.(true)}
onmouseleave={() => data.onDbtHover?.(false)}
onclick={(e) => {
e.stopPropagation()
data.onDbtSelect?.()
}}
>
<DbtIcon width={9} height={9} />
{dbtLabel}
{#if data.dbt.data_tests?.length}
<span class="opacity-70">&middot; {data.dbt.data_tests.length}T</span>
{/if}
</button>
{/if}
<!-- SCD2 companion marker: this node is the `<dim>_current` "latest row
per key" view its producer maintains alongside the base dimension.
Icon-only (the pill already truncates); the title names the base. -->
{#if data.derived_from}
<span
class="shrink-0 mr-1.5 text-violet-600 dark:text-violet-400"
title={`SCD2 current view of ${data.derived_from}: latest row per key, maintained by the same producer as the base dimension.`}
>
<History size={12} />
</span>
{/if}
{/snippet}
</PipelineNodeCard>
{/if}
<!-- SCD2 companion marker: this node is the `<dim>_current` "latest row
per key" view its producer maintains alongside the base dimension.
Icon-only (the pill already truncates); the title names the base. -->
{#if data.derived_from}
<span
class="shrink-0 mr-1.5 text-violet-600 dark:text-violet-400"
title={`SCD2 current view of ${data.derived_from}: latest row per key, maintained by the same producer as the base dimension.`}
>
<History size={12} />
</span>
{/if}
</div>
</NodeActionsMenu>
{#if showGuardBadge}
<!-- Data-test outcome badge. Floats off the TOP-RIGHT corner (opposite the
left-edge run button and the bottom + inserter) so it never collides
@@ -361,6 +759,7 @@
<div
class={twMerge(
'absolute -top-2 -right-2 z-10 rounded-full w-5 h-5 grid place-items-center border shadow-sm',
hasHeader && CARD_CORNER_TOP,
guardClass
)}
title={guardTitle}
@@ -376,7 +775,13 @@
cards. Drafts are runnable too via runScriptPreview, so no
greyed-out state — the page-supplied callback handles the
dispatch. -->
<div class="absolute -left-3 top-1/2 -translate-y-1/2 z-10">
<!-- Centred on the card, below the header tab when there is one. -->
<div
class={twMerge(
'absolute -left-3 top-1/2 -translate-y-1/2 z-10',
hasHeader && !data.bannerOnly && 'top-[calc(50%+11px)]'
)}
>
<button
type="button"
onclick={runProducers}
@@ -398,53 +803,47 @@
</button>
</div>
{/if}
{#if showAdd}
<!-- Always-visible + below the asset for downstream pipeline-script
creation. Half-overlapping the bottom edge so it visually attaches
to the node like the flow editor's between-step inserter. -->
{#if data.onStartOnDrag}
<NodeOnDot onStart={data.onStartOnDrag} visible={hovered} />
{/if}
{#if data.fix}
<!-- A red node's way out replaces its "+": adding downstream of a broken
node is not what it needs. -->
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<PipelineInsertMenu
kinds={[
{
id: 'pipeline_script',
label: 'Add downstream pipeline script',
description: 'Triggered when this asset changes',
icon: Code2
}
]}
languages={PIPELINE_LANGUAGES as any}
pathPrefix={data.pathPrefix ?? ''}
defaultPathSuffix={data.defaultPathSuffix ?? ''}
onPick={handlePick}
<NodeFixButton fix={data.fix} />
</div>
{:else if showAdd}
<!-- Always-visible + on the asset's bottom edge for downstream pipeline-
script creation, half-overlapping it like the flow editor's
between-step inserter. -->
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<AddDownstreamMenu
asset={{ kind: data.asset_kind, path: data.path }}
onAddScript={data.onAddScript!}
pathPrefix={data.pathPrefix}
defaultPathSuffix={data.defaultPathSuffix}
>
{#snippet trigger()}
<!--
Sizing notes for the round + button:
- w-6/h-6 (24px) chosen so that with border-2 (2px each
side) the inner area is exactly 20px — divisible by
the 16px icon to leave a 2px gap on every side. At
20px / 12px before, the gap was 4px-each but the
odd rounding interacted badly with svelte-flow's
fractional zoom transforms and the icon drifted
half a pixel off-center on certain zoom levels.
- `grid place-items-center` instead of flex centering:
flex's baseline alignment introduces a sub-pixel
nudge on small elements that grid avoids.
- `leading-none` strips the default line-height
contribution that the SVG inherits via the parent's
text rendering, otherwise the icon is shifted
downward by ~0.5px at fractional zooms.
-->
{#snippet trigger({ open })}
<!-- A click opens the menu; a drag goes to the canvas, which
tells a real drag from a click by how far the pointer moves. -->
<button
type="button"
class="nopan nodrag block"
onclick={(e) => e.stopPropagation()}
class="bg-surface border border-gray-400 dark:border-gray-600 text-secondary hover:bg-surface-hover rounded-full w-6 h-6 grid place-items-center shadow-sm leading-none"
title="Add downstream pipeline script"
onpointerdown={(e) => {
if (e.button !== 0) return
e.preventDefault()
data.onStartFeedDrag?.(e)
}}
aria-label="Add downstream step"
title={data.onStartFeedDrag
? 'Click to add a downstream step, or drag onto a script to run it after each write'
: undefined}
>
<Plus size={16} strokeWidth={2.5} />
<AddDownstreamPill expanded={open} />
</button>
{/snippet}
</PipelineInsertMenu>
</AddDownstreamMenu>
</div>
{/if}
</div>
frontend/src/lib/components/assets/AssetGraph/DucklakeAssetPanel.svelte
+24 -20
View File
@@ -11,7 +11,8 @@
// can't be read), so a selectable version always exists in the table.
import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
import { Table2, History, Columns3 } from 'lucide-svelte'
import { Table2, History, Columns3, Database } from 'lucide-svelte'
import DBManagerContent from '$lib/components/DBManagerContent.svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { resource } from 'runed'
import { fetchDucklakeSnapshots } from '$lib/components/dbOps'
@@ -31,7 +32,7 @@
}
let { path, workspace, schemaCanEvolve = true }: Props = $props()
let tab = $state<'partitions' | 'history' | 'schema'>('partitions')
let tab = $state<'data' | 'partitions' | 'history' | 'schema'>('data')
// The user's explicit snapshot pick (undefined until they click a row).
let selectedVersion = $state<number | undefined>(undefined)
@@ -72,24 +73,28 @@
</script>
<div class="flex flex-col h-full">
{#if !qualifiedTable}
<!-- Catalog-level ducklake node (no table segment, e.g. `ducklake://main`):
snapshot history / time-travel are per-table, so only the partition
grid applies here. -->
<PartitionStatusGrid {path} {workspace} />
{:else}
<div class="flex items-center gap-2 px-3 py-2 border-b shrink-0">
<ToggleButtonGroup selected={tab} on:selected={(e) => (tab = e.detail)}>
{#snippet children({ item })}
<ToggleButton size="sm" value="partitions" label="Partitions" icon={Table2} {item} />
<div class="flex items-center gap-2 px-3 py-2 border-b shrink-0">
<ToggleButtonGroup selected={tab} on:selected={(e) => (tab = e.detail)}>
{#snippet children({ item })}
<ToggleButton size="sm" value="data" label="Data" icon={Database} {item} />
<ToggleButton size="sm" value="partitions" label="Partitions" icon={Table2} {item} />
<!-- Snapshot history and time-travel are per table: a catalog-level node
(`ducklake://main`, no table segment) has only its data and partitions. -->
{#if qualifiedTable}
<ToggleButton size="sm" value="schema" label="Schema" icon={Columns3} {item} />
<ToggleButton size="sm" value="history" label="History" icon={History} {item} />
{/snippet}
</ToggleButtonGroup>
</div>
{/if}
{/snippet}
</ToggleButtonGroup>
</div>
<div class="flex-1 min-h-0">
{#if tab === 'partitions'}
<div class="flex-1 min-h-0">
{#if tab === 'data' && parsed}
<!-- Keyed on the asset: the manager opens its table once, on mount. -->
{#key path}
<DBManagerContent input={parsed} {workspace} />
{/key}
{:else if tab === 'partitions' || !qualifiedTable}
<PartitionStatusGrid {path} {workspace} />
{:else if tab === 'schema'}
<SchemaHistoryPanel {path} {workspace} canEvolve={schemaCanEvolve} />
@@ -117,7 +122,6 @@
</Pane>
</Splitpanes>
</div>
{/if}
</div>
{/if}
{/if}
</div>
</div>
frontend/src/lib/components/assets/AssetGraph/InitialFitView.svelte
+3 -1
View File
@@ -25,6 +25,8 @@
if (!nodesInitialized.current) return
if (!nodes.some((n) => n.type !== 'add')) return
fittedFor = fitKey
void fitView({ padding: 0.1, maxZoom: 1 })
// A re-fit over nodes that keep their ids finds them already measured, one
// render before their new positions reach the flow.
requestAnimationFrame(() => void fitView({ padding: 0.1, maxZoom: 1 }))
})
</script>
frontend/src/lib/components/assets/AssetGraph/NodeActionsMenu.svelte
+68
View File
@@ -0,0 +1,68 @@
<!--
@component
A pipeline node's actions, reachable two ways: the kebab revealed on hover at the
node's top-right corner, and a right-click menu over the node carrying the same
items. Place it inside the node's `relative` wrapper, around the node's card.
-->
<script lang="ts">
import type { Snippet } from 'svelte'
import { twMerge } from 'tailwind-merge'
import { EllipsisVertical } from 'lucide-svelte'
import { stopPropagation, preventDefault } from 'svelte/legacy'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import ContextMenu from '$lib/components/common/contextmenu/ContextMenu.svelte'
import { contextMenuItemsFromMenu } from '$lib/components/common/contextmenu/fromMenuItems'
import type { Item } from '$lib/utils'
interface Props {
items: Item[]
/** The pointer is over the node: shows the kebab. */
hover: boolean
/** Moves the kebab, for a node with its own badge on that corner. */
kebabClass?: string
children: Snippet
}
let { items, hover, kebabClass, children }: Props = $props()
let menuOpen = $state(false)
let contextItems = $derived(contextMenuItemsFromMenu(items))
</script>
{#if items.length > 0}
<ContextMenu items={contextItems}>
{@render children()}
</ContextMenu>
<!-- Rendered only on hover or while open, so the canvas stays clean at rest.
`pointerdown` is stopped so svelte-flow doesn't select the node when the
user reaches for the menu. -->
<div
class={twMerge('absolute -top-2 -right-2 h-7 p-1 min-w-7', kebabClass)}
style="will-change: transform;"
>
<DropdownV2
{items}
placement="bottom-end"
bind:open={menuOpen}
fixedHeight={false}
usePointerDownOutside
enableFlyTransition
>
{#snippet buttonReplacement()}
<button
class={twMerge(
'center-center p-1 text-secondary shadow-sm bg-surface duration-0 hover:bg-surface-tertiary',
hover || menuOpen ? 'block' : '!hidden',
'shadow-md rounded-md'
)}
onpointerdown={stopPropagation(preventDefault(() => {}))}
title="Actions"
>
<EllipsisVertical size={12} />
</button>
{/snippet}
</DropdownV2>
</div>
{:else}
{@render children()}
{/if}
frontend/src/lib/components/assets/AssetGraph/NodeFixButton.svelte
+124
View File
@@ -0,0 +1,124 @@
<!--
@component
The "Fix" pill on the bottom edge of a misconfigured (red) pipeline node: what is
wrong, and the actions that fix it. Styled like the node's "+" pill; the node's
red fill already says something is wrong.
-->
<script lang="ts" module>
export type NodeFixAction = {
label: string
/** The side effect, spelled out: what changes, where. */
detail?: string
/** `anchor`: where the Fix pill sits, for an action opening a menu there. */
run: (anchor: { x: number; y: number }) => void
}
/** Prose, with the entities it names as links that select them. */
export type FixExplainer = Array<string | { text: string; onClick: () => void }>
export type NodeFix = { explainer: FixExplainer; actions: NodeFixAction[] }
/** A fix as its supplier declares it: an action, or opening the schedule
* wizard (which the canvas owns). */
export type NodeFixSpec = {
explainer: FixExplainer
actions: Array<{
label: string
detail?: string
run?: () => void
/** Opens the schedule wizard for this script; its confirm calls `onSchedule`. */
scheduleFor?: {
script: string
onSchedule: (s: import('./PipelineInsertMenu.svelte').PipelineInsertSchedule) => void
}
}>
}
</script>
<script lang="ts">
import Popover from '$lib/components/meltComponents/Popover.svelte'
import { Button } from '$lib/components/common'
import { Wrench } from 'lucide-svelte'
let {
fix,
label = 'Fix',
chip = false
}: {
fix: NodeFix
label?: string
/** A small chip in a node's text, rather than the pill on its edge. */
chip?: boolean
} = $props()
let open = $state(false)
let anchorEl: HTMLElement | undefined = $state()
</script>
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
bind:this={anchorEl}
onpointerdown={(e) => e.stopPropagation()}
onkeydown={(e) => e.stopPropagation()}
onclick={(e) => e.stopPropagation()}
>
<Popover
bind:isOpen={open}
placement="bottom"
enableFlyTransition
usePointerDownOutside
contentClasses="p-3 w-96"
>
{#snippet trigger()}
{#if chip}
<span
class="flex items-center gap-1 rounded-md border px-1.5 py-1 text-3xs leading-none font-normal whitespace-nowrap bg-surface border-red-300 dark:border-red-600 text-red-700 dark:text-red-300 hover:bg-surface-hover"
>
<Wrench size={10} class="shrink-0" />
{label}
</span>
{:else}
<span
class="h-6 px-2 rounded-full flex items-center gap-1 bg-surface border border-gray-400 dark:border-gray-600 text-secondary hover:text-primary hover:border-gray-500 shadow-sm text-2xs font-normal leading-none whitespace-nowrap"
>
<Wrench size={12} class="shrink-0" />
{label}
</span>
{/if}
{/snippet}
{#snippet content()}
<div class="flex flex-col gap-3">
<p class="text-xs text-primary">
{#each fix.explainer as part}
{#if typeof part === 'string'}{part}{:else}<button
type="button"
class="text-accent hover:underline"
onclick={() => {
open = false
part.onClick()
}}>{part.text}</button
>{/if}
{/each}
</p>
<div class="flex flex-col gap-2">
{#each fix.actions as action, i (action.label)}
<div class="flex flex-col gap-0.5">
<Button
variant={i === 0 ? 'accent' : 'default'}
unifiedSize="sm"
onclick={() => {
const r = anchorEl?.getBoundingClientRect()
open = false
action.run({ x: r ? r.left + r.width / 2 : 0, y: r ? r.bottom : 0 })
}}
>
{action.label}
</Button>
{#if action.detail}
<span class="text-2xs text-secondary">{action.detail}</span>
{/if}
</div>
{/each}
</div>
</div>
{/snippet}
</Popover>
</div>
frontend/src/lib/components/assets/AssetGraph/NodeOnDot.svelte
+23
View File
@@ -0,0 +1,23 @@
<!--
@component
The dot on a pipeline node's top edge: dragged onto an asset, it makes the node's
script run after each write to that asset.
-->
<script lang="ts">
let { onStart, visible }: { onStart: (e: PointerEvent) => void; visible: boolean } = $props()
</script>
<button
type="button"
class="absolute -top-1.5 left-1/2 -translate-x-1/2 z-10 w-3 h-3 rounded-full bg-surface border-2 border-blue-500 cursor-crosshair transition-opacity {visible
? 'opacity-100'
: 'opacity-0'}"
title="Drag onto an asset to run after each write to it"
aria-label="Drag onto an asset to run after each write to it"
onpointerdown={(e) => {
e.stopPropagation()
e.preventDefault()
onStart(e)
}}
onclick={(e) => e.stopPropagation()}
></button>
frontend/src/lib/components/assets/AssetGraph/PanToNode.svelte
+3 -2
View File
@@ -2,6 +2,7 @@
import { useSvelteFlow, type Node } from '@xyflow/svelte'
import { untrack } from 'svelte'
import { NODE } from '$lib/components/graph/util'
import { PIPELINE_NODE_HEIGHT } from './assetGraphLayout'
// Smoothly pans the viewport so a freshly-created node lands in the middle,
// keeping the current zoom. Lives inside <SvelteFlow> so useSvelteFlow() has
@@ -27,8 +28,8 @@
untrack(() => {
const node = ns.find((n) => n.id === id)
if (!node) return // not laid out yet — re-runs when `nodes` updates
const cx = node.position.x + NODE.width / 2
const cy = node.position.y + NODE.height / 2
const cx = node.position.x + (node.measured?.width ?? NODE.width) / 2
const cy = node.position.y + PIPELINE_NODE_HEIGHT / 2
const key = `${id}:${cx}:${cy}`
if (key === lastKey) return
lastKey = key
frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte
+4 -4
View File
@@ -416,7 +416,7 @@
onmouseenter={() => onHoverRun?.([e.path])}
onmouseleave={() => onHoverRun?.(undefined)}
class={twMerge(
'w-full flex items-center gap-2 px-2 py-1 rounded-sm hover:bg-surface-hover text-xs text-left',
'w-full flex items-center gap-2 px-2 py-1.5 rounded-sm font-normal hover:bg-surface-hover text-xs text-left',
isMember && 'pl-7',
expanded && 'bg-surface-selected'
)}
@@ -445,7 +445,7 @@
{:else}
<Code2 size={12} class="shrink-0 text-tertiary" />
{/if}
<span class="flex-1 min-w-0 truncate font-mono text-2xs" title={e.path}>
<span class="flex-1 min-w-0 truncate text-xs" title={e.path}>
{e.path}
</span>
{#if showSource}
@@ -533,7 +533,7 @@
onclick={() => toggleGroup(g.key)}
onmouseenter={() => onHoverRun?.(g.members.map((m) => m.path))}
onmouseleave={() => onHoverRun?.(undefined)}
class="w-full flex items-center gap-2 px-2 py-1 rounded-sm hover:bg-surface-hover text-xs text-left"
class="w-full flex items-center gap-2 px-2 py-1.5 rounded-sm font-normal hover:bg-surface-hover text-xs text-left"
title={collapsed ? 'Expand cascade runs' : 'Collapse cascade runs'}
>
<span class="shrink-0 text-tertiary">
@@ -549,7 +549,7 @@
{/if}
</span>
<Workflow size={12} class="shrink-0 text-indigo-500" />
<span class="flex-1 min-w-0 truncate font-mono text-2xs" title={groupTitle(g)}>
<span class="flex-1 min-w-0 truncate text-xs" title={groupTitle(g)}>
{groupTitle(g)}
</span>
<span
frontend/src/lib/components/assets/AssetGraph/PipelineDbtSignpost.svelte
-40
View File
@@ -1,40 +0,0 @@
<script lang="ts">
import { base } from '$lib/base'
import Alert from '$lib/components/common/alert/Alert.svelte'
import Button from '$lib/components/common/button/Button.svelte'
import DbtIcon from '$lib/components/icons/DbtIcon.svelte'
import { BookOpen } from 'lucide-svelte'
const DBT_DOCS = 'https://www.windmill.dev/docs/getting_started/scripts_quickstart/dbt'
</script>
<Alert type="info" title="Prefer using dbt? Windmill also runs dbt" size="xs">
<div class="flex flex-col gap-2">
<p>
A pipeline is Windmill's own abstraction; an existing dbt project stays a dbt project, and
runs here unchanged as its own kind of script, with dbt still owning its models, refs and
tests. All it needs is a warehouse configured under
<a class="underline" href="{base}/workspace_settings?tab=dbt">Settings → dbt</a>. Its models
show up in the asset graph as <span class="font-mono">dbt://</span> nodes, visible next to whatever
pipelines you do build, so you can adopt pipelines later, or never.
</p>
<div class="flex flex-row flex-wrap items-center gap-2">
<!-- The dbt mark goes in as a child rather than `startIcon`: that slot
sizes its icon with lucide's `size`, which DbtIcon has no prop for. -->
<Button variant="default" unifiedSize="sm" href="{base}/scripts/add?lang=dbt">
<DbtIcon width={14} height={14} />
New dbt script
</Button>
<Button
variant="subtle"
unifiedSize="sm"
href={DBT_DOCS}
target="_blank"
rel="noreferrer"
startIcon={{ icon: BookOpen }}
>
dbt documentation
</Button>
</div>
</div>
</Alert>
frontend/src/lib/components/assets/AssetGraph/PipelineDeployErrors.svelte
+49
View File
@@ -0,0 +1,49 @@
<script lang="ts">
import { AlertTriangle } from 'lucide-svelte'
import Popover from '$lib/components/meltComponents/Popover.svelte'
// Which drafts failed to deploy and why, anchored next to the deploy button so
// the reasons are one click away without leaving the editor. Deployed drafts
// leave the map; what is listed is still unresolved.
let {
errors,
open = $bindable()
}: {
errors: ReadonlyMap<string, string>
open?: boolean
} = $props()
</script>
<Popover
placement="bottom-end"
contentClasses="p-3 max-w-[480px]"
usePointerDownOutside
enableFlyTransition
bind:isOpen={open}
>
{#snippet trigger()}
<button
type="button"
class="flex items-center gap-1.5 px-2 py-1 rounded-md text-red-600 dark:text-red-400 bg-red-50 dark:bg-red-900/30 hover:bg-red-100 dark:hover:bg-red-900/50 transition-colors text-xs font-medium"
title="View save errors"
>
<AlertTriangle size={14} />
<span>{errors.size} failed</span>
</button>
{/snippet}
{#snippet content()}
<div class="flex flex-col gap-2">
<span class="text-xs font-semibold text-emphasis">Save errors</span>
<div class="flex flex-col gap-2 max-h-72 overflow-y-auto">
{#each [...errors.entries()] as [path, message] (path)}
<div class="flex flex-col gap-0.5 border-l-2 border-red-400 pl-2">
<span class="text-2xs font-mono text-emphasis">{path}</span>
<span class="text-2xs text-red-600 dark:text-red-400 break-words">
{message}
</span>
</div>
{/each}
</div>
</div>
{/snippet}
</Popover>
frontend/src/lib/components/assets/AssetGraph/PipelineDeployTriggersModal.svelte
+47
View File
@@ -0,0 +1,47 @@
<script lang="ts">
import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
import type { PipelineTriggerDraft } from './types'
// Triggers start running once deployed, so deploying a pipeline's trigger
// drafts asks first, listing what will start.
let {
open,
triggerDrafts,
onConfirmed,
onCanceled
}: {
open: boolean
triggerDrafts: ReadonlyMap<string, PipelineTriggerDraft>
onConfirmed: () => void
onCanceled: () => void
} = $props()
</script>
<ConfirmationModal
{open}
title="Deploy triggers?"
confirmationText="Deploy"
type="info"
{onConfirmed}
{onCanceled}
>
<div class="flex flex-col gap-2 text-xs text-secondary">
<p>
{triggerDrafts.size === 1
? 'Saving the pipeline also creates this trigger. It starts running its script once deployed.'
: `Saving the pipeline also creates these ${triggerDrafts.size} triggers. They start running their scripts once deployed.`}
</p>
<ul class="flex flex-col gap-1">
{#each [...triggerDrafts] as [key, d] (key)}
<li class="flex flex-col">
<span class="font-mono text-2xs text-emphasis truncate">{d.config.path}</span>
<span class="text-2xs text-hint truncate">
{d.kind}{#if d.kind === 'schedule'}
· <span class="font-mono">{d.config.schedule}</span> ({d.config.timezone}){/if}
→ {d.config.script_path}
</span>
</li>
{/each}
</ul>
</div>
</ConfirmationModal>
frontend/src/lib/components/assets/AssetGraph/PipelineFolderList.svelte
+51 -42
View File
@@ -18,11 +18,18 @@
mode?: 'view' | 'edit'
// Called right before navigating — lets a wrapping modal close itself.
onOpen?: () => void
// Only the folder picker, for hosts that don't list pipelines.
hideExisting?: boolean
}
let { currentFolder = undefined, mode = 'view', onOpen = undefined }: Props = $props()
let {
currentFolder = undefined,
mode = 'view',
onOpen = undefined,
hideExisting = false
}: Props = $props()
let pipelines = resource(
() => $workspaceStore,
() => (hideExisting ? undefined : $workspaceStore),
async (ws) => {
if (!ws) return [] as ListPipelineFoldersResponse
return await AssetService.listPipelineFolders({ workspace: ws })
@@ -49,50 +56,52 @@
</script>
<div class="flex flex-col gap-6 w-full min-w-0">
<section class="flex flex-col gap-2">
<h3 class="text-xs font-semibold text-secondary uppercase tracking-wide">
Existing pipelines
</h3>
{#if pipelines.loading && !pipelines.current}
<div class="text-tertiary text-sm flex items-center gap-2">
<Loader2 size={14} class="animate-spin" />
Loading…
</div>
{:else if pipelines.error}
<div class="text-red-500 text-sm">Failed: {pipelines.error.message}</div>
{:else if visiblePipelines.length === 0}
<div class="text-tertiary text-sm">
{currentFolder
? 'No other pipelines in this workspace.'
: 'No pipelines yet. A pipeline is any folder whose scripts carry pipeline annotations.'}
</div>
{:else}
<div class="flex flex-col border rounded-md overflow-hidden max-h-64 overflow-y-auto">
{#each visiblePipelines as p (p.folder)}
<button
type="button"
class="flex items-center justify-between px-3 py-2 border-b last:border-b-0 bg-surface hover:bg-surface-hover transition-colors text-left"
onclick={() => openExistingPipeline(p.folder)}
>
<span class="font-mono text-sm">f/{p.folder}</span>
<span class="text-2xs text-tertiary">
{p.script_count}
{p.script_count === 1 ? 'script' : 'scripts'}
</span>
</button>
{/each}
</div>
{/if}
</section>
{#if !hideExisting}
<section class="flex flex-col gap-2">
<h3 class="text-sm font-semibold text-emphasis">Existing pipelines</h3>
{#if pipelines.loading && !pipelines.current}
<div class="text-tertiary text-sm flex items-center gap-2">
<Loader2 size={14} class="animate-spin" />
Loading…
</div>
{:else if pipelines.error}
<div class="text-red-500 text-sm">Failed: {pipelines.error.message}</div>
{:else if visiblePipelines.length === 0}
<div class="text-tertiary text-sm">
{currentFolder
? 'No other pipelines in this workspace.'
: 'No pipelines yet. A pipeline is any folder whose scripts carry pipeline annotations.'}
</div>
{:else}
<div class="flex flex-col border rounded-md overflow-hidden max-h-64 overflow-y-auto">
{#each visiblePipelines as p (p.folder)}
<button
type="button"
class="flex items-center justify-between px-3 py-2 border-b last:border-b-0 bg-surface hover:bg-surface-hover transition-colors text-left"
onclick={() => openExistingPipeline(p.folder)}
>
<span class="font-mono text-sm">f/{p.folder}</span>
<span class="text-2xs text-tertiary">
{p.script_count}
{p.script_count === 1 ? 'script' : 'scripts'}
</span>
</button>
{/each}
</div>
{/if}
</section>
{/if}
{#if !$userStore?.operator}
<!-- Operators only view existing pipelines — opening an arbitrary
folder is a build-a-new-pipeline affordance. -->
<section class="flex flex-col gap-2">
<h3 class="text-xs font-semibold text-secondary uppercase tracking-wide">
Pick or create a folder
</h3>
<div class="flex items-center gap-2">
<section class="flex flex-col gap-1">
<h3 class="text-sm font-semibold text-emphasis">Pick or create a folder</h3>
<p class="text-xs text-secondary">
Each folder holds one pipeline, made of its pipeline scripts. Pick a folder to open its
pipeline, or create one to start a new pipeline.
</p>
<div class="flex items-center gap-2 mt-1">
<div class="flex-1 min-w-0">
<FolderPicker bind:folderName={pickedFolder} />
</div>
frontend/src/lib/components/assets/AssetGraph/PipelineGraphEditor.svelte
+366 -81
View File
@@ -1,6 +1,15 @@
<script lang="ts">
import { untrack, type Snippet } from 'svelte'
import { PIPELINE_DRAFT_KIND, pipelineBundlePath } from '$lib/pipelinePaths'
import { MousePointerClick } from 'lucide-svelte'
import Disposable from '$lib/components/common/drawer/Disposable.svelte'
import Portal from '$lib/components/Portal.svelte'
import { getOverlayHost } from '$lib/components/common/overlayHost.svelte'
import { useFlowPanelMode } from '$lib/components/flows/flowPanelMode.svelte'
import {
PIPELINE_DRAFT_KIND,
pipelineBundlePath,
pipelineLocalMirrorKey
} from '$lib/pipelinePaths'
import { Loader2 } from 'lucide-svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { DraftService } from '$lib/gen'
@@ -11,6 +20,11 @@
import type { ColumnLineageGraph } from './columnLineageGraph'
import HideButton from '$lib/components/apps/editor/settingsPanel/HideButton.svelte'
import AssetGraphCanvas from './AssetGraphCanvas.svelte'
import { TRIGGER_NODE_STYLE } from './TriggerNode.svelte'
import { describeSchedule } from '$lib/utils/describeCron'
import type { UpstreamTriggerAction } from './upstreamScriptItems'
import type { AssetUpstreamDelete } from './assetsOnlyView'
import type { NodeFixSpec } from './NodeFixButton.svelte'
import AssetGraphDetailsPane from './AssetGraphDetailsPane.svelte'
import PipelineEventLog from './PipelineEventLog.svelte'
import type {
@@ -18,8 +32,10 @@
AssetGraphSelection,
NativeTriggerKind,
PipelineMode,
PipelineTriggerDraft,
DbtAssetProvenance
} from './types'
import { isDraftableTriggerKind, triggerDraftKey } from './pipelineTriggerDrafts'
import type { AssetKind, Script, ScriptLang } from '$lib/gen'
import type { RunnableRunState, PipelineEvent } from './activeRunnables.svelte'
import type { PipelineOutputKind } from './pipelineTemplates'
@@ -41,6 +57,7 @@
defaultPathSuffix = 'new_pipeline_script',
panelHidden = false,
onTogglePanelHidden,
modalPanel = false,
prefetchingAssets = false,
hoveredPaths = [],
selectedRunPaths = [],
@@ -65,6 +82,11 @@
onAddScriptForAsset,
onAddPipelineScript,
onRunnableMenuRemove,
onDeleteAssetUpstream,
nodeFixes,
assetDeleteVerb,
onEdgeAction,
onAddAssetTrigger,
onRunProducer,
idlePane,
onRequestEdit,
@@ -73,6 +95,9 @@
onRunCascadeByPath,
runFormInitialArgs,
onRunFormArgsChange,
inputsOnly = false,
onRunInputs,
onShowScript,
readyDataUploadPaths,
resolveLocalScript,
localScriptsVersion,
@@ -126,6 +151,9 @@
pathPrefix: string
defaultPathSuffix?: string
panelHidden?: boolean
/** Let the details pane open as a modal over a full-width graph when the editor
* is too narrow to dock it — the flow editor's step panel rule. */
modalPanel?: boolean
onTogglePanelHidden?: () => void
prefetchingAssets?: boolean
hoveredPaths?: string[]
@@ -153,16 +181,29 @@
language: ScriptLang,
scriptPath: string,
outputKind: PipelineOutputKind,
aiPrompt?: string
aiPrompt?: string,
options?: import('./PipelineInsertMenu.svelte').PipelineInsertOptions
) => void
onAddPipelineScript?: (
language: ScriptLang,
path: string,
source: { kind: NativeTriggerKind; path: string | undefined },
outputKind: PipelineOutputKind,
aiPrompt?: string
aiPrompt?: string,
options?: import('./PipelineInsertMenu.svelte').PipelineInsertOptions
) => void
onRunnableMenuRemove?: (...args: any[]) => void
onDeleteAssetUpstream?: (target: AssetUpstreamDelete) => void
nodeFixes?: ReadonlyMap<string, NodeFixSpec>
assetDeleteVerb?: 'Delete' | 'Archive'
onEdgeAction?: (a: {
action: 'listen' | 'stop'
targets: Array<{ script: string; asset: { kind: AssetKind; path: string }; reads: boolean }>
}) => void
onAddAssetTrigger?: (a: {
scripts: Array<{ script: string; reads: boolean }>
asset: { kind: AssetKind; path: string }
}) => void
onRunProducer?: (producer: RunProducer) => Promise<string | undefined>
idlePane?: Snippet
onRequestEdit?: () => void
@@ -175,8 +216,16 @@
// callback to persist them as they change — see the page's dataUploadArgs.
runFormInitialArgs?: Record<string, any>
onRunFormArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
/** See AssetGraphDetailsPane's `inputsOnly`. */
inputsOnly?: boolean
onRunInputs?: (
path: string,
args: Record<string, any>,
live?: { content: string; language: ScriptLang }
) => Promise<string | undefined>
onShowScript?: () => void
// Data-upload entry scripts whose staged upload is ready (green node).
readyDataUploadPaths?: Set<string>
readyDataUploadPaths?: ReadonlySet<string>
/** Local-dev (`/pipeline_dev`): resolve a node to its working-tree content
* so the details pane skips the (nonexistent) deployed-script fetch. May
* be async (to infer the args schema for the run form). */
@@ -247,18 +296,81 @@
? (editor.selection != undefined || editor.activeDraftPath != undefined) && !panelHidden
: !panelHidden
)
// Docked beside the graph, or in a modal over it (modal mode). In modal mode the
// details only show once asked for: a double-click on a node, a click on the
// node already selected, or a script's open button — never on a selection the
// editor makes itself (an AI-built node), which would pop a modal unasked.
const panelController = useFlowPanelMode({ enabled: () => modalPanel })
$effect(() => panelController.measure(containerWidth))
const panelMode = $derived(panelController.mode)
let panelModalOpen = $state(false)
const dockedPaneOpen = $derived(detailsPaneOpen && panelMode === 'docked')
$effect(() => {
if (untrack(() => panelModalOpen) && (panelMode === 'docked' || !detailsPaneOpen)) {
panelModalOpen = false
}
})
let panelDisposable: Disposable | undefined = $state(undefined)
// Disposable joins the overlay stack through its methods, not by watching `open`.
$effect(() => {
panelModalOpen
untrack(() => {
panelModalOpen ? panelDisposable?.openDrawer() : panelDisposable?.closeDrawer()
})
})
const overlayHost = getOverlayHost()
const modalHost = $derived(overlayHost?.el())
function selectableNodeAt(e: MouseEvent): HTMLElement | null {
return (e.target as HTMLElement | null)?.closest('.svelte-flow__node.selectable') ?? null
}
// Read in the capture phase: once the click bubbles, the canvas has applied
// its selection and a first click looks like a click on the selected node.
let clickStartedOnSelected = false
function noteSelectionBeforeClick(e: MouseEvent) {
clickStartedOnSelected = !!selectableNodeAt(e)?.classList.contains('selected')
}
function openPanelModalIfReselected(e: MouseEvent) {
if (clickStartedOnSelected && selectableNodeAt(e)) panelModalOpen = true
}
function openPanelModalFromGraph(e: MouseEvent) {
if (selectableNodeAt(e)) panelModalOpen = true
}
// A button in the pane that ends it closes the modal in the same update: left to
// the effect above, the modal would first render the pane without its selection.
function closePanelModal() {
if (panelMode === 'modal') panelModalOpen = false
}
function handleCanvasSelect(s: AssetGraphSelection | undefined, opts?: { open?: boolean }) {
onSelect(s)
if (opts?.open && s && panelMode === 'modal') panelModalOpen = true
}
// A data-upload node's click asks for its run form; in modal mode nothing shows
// it unless the modal opens too.
$effect(() => {
if (focusUploadSignal > 0 && untrack(() => panelMode) === 'modal') {
untrack(() => (panelModalOpen = true))
}
})
const showNodeHint = $derived(panelMode === 'modal' && !panelModalOpen)
const nodeHintText = $derived(
effectiveSelection
? 'Click the selected node to open it'
: 'Double click a node to open it'
)
let idleView = $derived(
mode !== 'edit' && editor.selection == undefined && editor.activeDraftPath == undefined
)
// Wrap the writes (and the rightPaneSize read in the else branch) in untrack so
// the effect tracks only `detailsPaneOpen` / `storedRightPaneSize` — without it,
// the effect tracks only `dockedPaneOpen` / `storedRightPaneSize` — without it,
// the Pane `bind:size` feedback loops the effect and pegs the main thread.
$effect(() => {
// Stacked (vertical) splits give the details pane more room — the run
// form + result need height more than the graph needs it.
const fallback = stacked ? 55 : 40
if (detailsPaneOpen) {
if (dockedPaneOpen) {
const restore = storedRightPaneSize
untrack(() => {
rightPaneSize = restore > 0 ? restore : fallback
@@ -272,6 +384,67 @@
}
})
// The selected asset's producing scripts, named and iconed from the graph, for
// the pane's "Edit script".
let producerScripts = $derived(
selectionProducers
.filter((p) => p.kind === 'script')
.map((p) => {
const r = displayGraph.runnables.find((x) => x.usage_kind === 'script' && x.path === p.path)
return { path: p.path, summary: r?.summary, language: r?.language }
})
)
// The selected asset's upstream triggers that can be opened from here, each
// with the same action its trigger node has: edit (edit mode), create a
// missing one (edit mode), or the webhook / data-upload drawers.
let producerTriggers = $derived.by((): UpstreamTriggerAction[] => {
const scripts = new Set(producerScripts.map((p) => p.path))
const seen = new Set<string>()
const out: UpstreamTriggerAction[] = []
for (const t of displayGraph.triggers ?? []) {
if (t.trigger_kind === 'asset' || !scripts.has(t.runnable_path)) continue
const kind = t.trigger_kind
const rowless = kind === 'webhook' || kind === 'data_upload'
const key = `${kind}:${rowless || t.missing ? t.runnable_path : t.path}`
if (seen.has(key)) continue
seen.add(key)
const label = TRIGGER_NODE_STYLE[kind]?.label ?? kind
const script = t.runnable_path
let action: UpstreamTriggerAction | undefined
if (kind === 'webhook') {
if (onOpenWebhook)
action = { label: 'Open webhook', detail: script, onOpen: () => onOpenWebhook(script) }
} else if (kind === 'data_upload') {
if (onOpenDataUpload)
action = { label: 'Upload data', detail: script, onOpen: () => onOpenDataUpload(script) }
} else if (t.missing && !rowless) {
if (onCreateMissingTrigger)
action = {
label: `Create ${label} trigger`,
detail: script,
onOpen: () => onCreateMissingTrigger(kind, script)
}
} else if (t.path && onEditTrigger) {
const path = t.path
action = {
label: kind === 'schedule' ? 'Edit schedule' : `Edit ${label} trigger`,
detail:
kind === 'schedule' && t.schedule
? (t.summary ?? describeSchedule(t.schedule, t.timezone) ?? path)
: path,
onOpen: () => onEditTrigger(kind, path, script)
}
}
if (action) out.push(action)
}
return out
})
function editProducerScript(path: string) {
if (mode !== 'edit') onRequestEdit?.()
onSelect({ kind: 'runnable', runnable_kind: 'script', path })
}
// ===================== Draft autosave (persistDrafts only) =====================
// All of this folder's in-flight drafts live in ONE per-user DB draft (typ
// `data_pipeline`) keyed at the folder, syncing across devices + surfacing in
@@ -279,12 +452,21 @@
// for the one-time migration below; the DB is the source of truth on load.
// FlowBuilder's autosave analogue — gated by `persistDrafts`.
let pipelineDraftPath = $derived(pipelineBundlePath(folder))
let storageKey = $derived(`pipeline-${folder}`)
type PipelineDraftBundle = { drafts: Array<[string, PipelineDraft]>; activeDraftPath?: string }
let storageKey = $derived(pipelineLocalMirrorKey(folder))
type PipelineDraftBundle = {
drafts: Array<[string, PipelineDraft]>
activeDraftPath?: string
triggerDrafts?: PipelineTriggerDraft[]
}
// Hydration is tracked on the editor instance (`editor.hydratedFromDb`), not a
// component flag: the in-session preview reuses one instance across editor
// hide/show, and it must hydrate ONCE per instance, not on every remount.
let lastPersistedBundle: string | undefined = undefined
// What decides whether the bundle needs saving: its drafts, not which one is
// open. Opening a draft is not an edit, so it rides along with the next real
// change instead of saving (and spinning the sync indicator) on a click.
const persistKey = (b: PipelineDraftBundle | undefined) =>
b ? JSON.stringify({ drafts: b.drafts, triggerDrafts: b.triggerDrafts }) : undefined
function restoreBundle(bundle: PipelineDraftBundle) {
if (Array.isArray(bundle.drafts)) {
@@ -300,6 +482,18 @@
if (loaded.size > 0) editor.drafts = loaded
}
if (typeof bundle.activeDraftPath === 'string') editor.activeDraftPath = bundle.activeDraftPath
if (Array.isArray(bundle.triggerDrafts)) {
editor.triggerDrafts = new Map(
bundle.triggerDrafts
.filter(
(d) =>
isDraftableTriggerKind(d?.kind) &&
typeof d.config?.path === 'string' &&
typeof d.config?.script_path === 'string'
)
.map((d) => [triggerDraftKey(d.kind, d.config.path), d])
)
}
}
function readLocalBundle(): PipelineDraftBundle | undefined {
@@ -311,7 +505,8 @@
if (s && (Array.isArray(s.drafts) || typeof s.activeDraftPath === 'string')) {
return {
drafts: Array.isArray(s.drafts) ? s.drafts : [],
activeDraftPath: typeof s.activeDraftPath === 'string' ? s.activeDraftPath : undefined
activeDraftPath: typeof s.activeDraftPath === 'string' ? s.activeDraftPath : undefined,
triggerDrafts: Array.isArray(s.triggerDrafts) ? s.triggerDrafts : undefined
}
}
} catch (e) {
@@ -339,8 +534,8 @@
fromServer = true
}
}
// A folder retarget during the await (the route-page header switcher, or a
// session retarget) changed the target and reset hydratedFromDb. Don't
// A folder switch during the await (the route page's header switcher)
// changed the target and reset hydratedFromDb. Don't
// apply this now-stale folder's bundle or mark the new folder hydrated —
// otherwise the stale result blocks the new folder's hydrate and its
// drafts bleed across folders.
@@ -359,7 +554,7 @@
{ workspace: ws ?? '', itemKind: PIPELINE_DRAFT_KIND, path },
migratedFromLocal ? undefined : serverSavedAt
)
if (!migratedFromLocal) lastPersistedBundle = bundle ? JSON.stringify(bundle) : undefined
if (!migratedFromLocal) lastPersistedBundle = persistKey(bundle)
} catch (e) {
console.warn('failed to load pipeline drafts', e)
} finally {
@@ -412,30 +607,52 @@
return [p, d] as [string, PipelineDraft]
return [p, { ...d, script, outputAssets, inputAssets }] as [string, PipelineDraft]
})
// A deployed script being edited is only promoted to a draft when its pane
// closes; until then its buffer is saved as the draft it will become.
const { scriptPath: livePath, content: liveBody, base } = editor.liveContent
if (
livePath != undefined &&
base &&
!editor.drafts.has(livePath) &&
liveBody !== (base.content ?? '')
) {
const live = editor.liveBodyAssets.scriptPath === livePath
const writes = live ? extractWrites(editor.liveBodyAssets.assets) : []
serialized.push([
livePath,
{
localId: editor.promotedDraftLocalId(livePath),
script: { ...base, content: liveBody },
outputAssets: writes.length > 0 ? writes : undefined,
inputAssets: live ? extractReads(editor.liveBodyAssets.assets) : undefined
}
])
}
const activePath = editor.activeDraftPath
const triggerDrafts = [...editor.triggerDrafts.values()]
const key = storageKey
const ws = workspace
const path = pipelineDraftPath
const hydrated = editor.hydratedFromDb
untrack(() => {
if (!hydrated) return
const isEmpty = serialized.length === 0 && !activePath
const isEmpty = serialized.length === 0 && !activePath && triggerDrafts.length === 0
const bundle: PipelineDraftBundle | undefined = isEmpty
? undefined
: { drafts: serialized, activeDraftPath: activePath }
: {
drafts: serialized,
activeDraftPath: activePath,
...(triggerDrafts.length > 0 ? { triggerDrafts } : {})
}
try {
if (typeof localStorage !== 'undefined') {
if (isEmpty) localStorage.removeItem(key)
else
localStorage.setItem(
key,
encodeState({ drafts: serialized, activeDraftPath: activePath })
)
else localStorage.setItem(key, encodeState(bundle))
}
} catch (e) {
console.warn('failed to mirror pipeline state', e)
}
const serializedBundle = bundle ? JSON.stringify(bundle) : undefined
const serializedBundle = persistKey(bundle)
if (serializedBundle === lastPersistedBundle) return
lastPersistedBundle = serializedBundle
if (!ws) return
@@ -450,10 +667,90 @@
})
</script>
<div class="h-full w-full" bind:clientWidth={containerWidth}>
{#snippet detailsBody(ws: string)}
{#if idleView && idlePane}
{@render idlePane()}
{:else}
<AssetGraphDetailsPane
{mode}
{onRequestEdit}
{canRunByPath}
{onRunByPath}
{onRunCascadeByPath}
{runFormInitialArgs}
{onRunFormArgsChange}
{inputsOnly}
{onRunInputs}
{onShowScript}
{resolveLocalScript}
{localScriptsVersion}
selection={activeDraft ? undefined : editor.selection}
selectionProducers={activeDraft ? [] : selectionProducers}
{producerScripts}
onEditScript={editProducerScript}
{producerTriggers}
{selectionColumnGraph}
{selectionColumnLoading}
{selectionColumnTruncated}
{selectionColumnFailed}
{selectionDbt}
{schemaCanEvolve}
{selectionForkMaterialization}
{schemaContractContext}
{runsRefreshKey}
{runsPendingJobId}
{activeRunnable}
{downstreamSubscribers}
onStartBoundedRun={canBoundedRunOpenScript &&
editor.openScriptPath &&
onStartBoundedRunForOpen
? () => onStartBoundedRunForOpen?.(editor.openScriptPath!)
: undefined}
{onRunCompleted}
{onTestStateChange}
{requestRemoveSignal}
{requestRunSignal}
{requestRunCascadeSignal}
{focusUploadSignal}
draftScript={activeDraft?.script}
draftOutputAssets={activeDraft?.outputAssets}
draftInputAssets={activeDraft?.inputAssets}
{onDraftPathChange}
onDraftMetaChange={editor.setDraftMeta}
{pathPrefix}
workspace={ws}
onAnnotationsChange={editor.handleAnnotationsChange}
onAssetsChange={editor.handleAssetsChange}
onContentChange={editor.handleContentChange}
onDraftPersist={editor.handleDraftPersist}
onclose={() => {
closePanelModal()
onClose()
}}
onHide={onTogglePanelHidden &&
(() => {
closePanelModal()
onTogglePanelHidden()
})}
{onDiscard}
{onDraftSaved}
{onPersistedSaved}
{onScriptRenamed}
{onScriptRemoved}
/>
{/if}
{/snippet}
<div class="relative h-full w-full" bind:clientWidth={containerWidth}>
<Splitpanes class="!h-full" horizontal={stacked}>
<Pane bind:size={leftPaneSize}>
<div class="relative h-full">
<!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_static_element_interactions -->
<div
class="relative h-full"
ondblclick={panelMode === 'modal' ? openPanelModalFromGraph : undefined}
onpointerdowncapture={panelMode === 'modal' ? noteSelectionBeforeClick : undefined}
onclick={panelMode === 'modal' ? openPanelModalIfReselected : undefined}
>
<AssetGraphCanvas
graph={displayGraph}
selection={effectiveSelection}
@@ -470,10 +767,15 @@
{onOpenWebhook}
{onOpenDataUpload}
{readyDataUploadPaths}
onselect={onSelect}
onselect={handleCanvasSelect}
{onAddScriptForAsset}
{onAddPipelineScript}
{onRunnableMenuRemove}
{onDeleteAssetUpstream}
{nodeFixes}
{assetDeleteVerb}
{onEdgeAction}
{onAddAssetTrigger}
{onRunProducer}
{validStartPaths}
{onStartBoundedRun}
@@ -482,6 +784,8 @@
{panToNodeId}
showMinimap={!stacked}
viewportFitKey={viewportFitKey ?? folder}
assetsOnlyToggle
assetsOnlyFolder={workspace ? `${workspace}/${folder}` : undefined}
/>
{#if boundBar}{@render boundBar()}{/if}
{#if mode === 'edit'}
@@ -509,66 +813,47 @@
{/if}
</div></Pane
>
{#if detailsPaneOpen && workspace}
{#if dockedPaneOpen && workspace}
<Pane bind:size={rightPaneSize} minSize={25}>
{#if idleView && idlePane}
{@render idlePane()}
{:else}
<AssetGraphDetailsPane
{mode}
{onRequestEdit}
{canRunByPath}
{onRunByPath}
{onRunCascadeByPath}
{runFormInitialArgs}
{onRunFormArgsChange}
{resolveLocalScript}
{localScriptsVersion}
selection={activeDraft ? undefined : editor.selection}
selectionProducers={activeDraft ? [] : selectionProducers}
{selectionColumnGraph}
{selectionColumnLoading}
{selectionColumnTruncated}
{selectionColumnFailed}
{selectionDbt}
{schemaCanEvolve}
{selectionForkMaterialization}
{schemaContractContext}
{runsRefreshKey}
{runsPendingJobId}
{activeRunnable}
{downstreamSubscribers}
onStartBoundedRun={canBoundedRunOpenScript &&
editor.openScriptPath &&
onStartBoundedRunForOpen
? () => onStartBoundedRunForOpen?.(editor.openScriptPath!)
: undefined}
{onRunCompleted}
{onTestStateChange}
{requestRemoveSignal}
{requestRunSignal}
{requestRunCascadeSignal}
{focusUploadSignal}
draftScript={activeDraft?.script}
draftOutputAssets={activeDraft?.outputAssets}
draftInputAssets={activeDraft?.inputAssets}
{pathPrefix}
{onDraftPathChange}
{workspace}
onAnnotationsChange={editor.handleAnnotationsChange}
onAssetsChange={editor.handleAssetsChange}
onContentChange={editor.handleContentChange}
onDraftPersist={editor.handleDraftPersist}
onclose={onClose}
onHide={onTogglePanelHidden}
{onDiscard}
{onDraftSaved}
{onPersistedSaved}
{onScriptRenamed}
{onScriptRemoved}
/>
{/if}
{@render detailsBody(workspace)}
</Pane>
{/if}
</Splitpanes>
{#if showNodeHint}
<!-- Above the event log's bar, which owns the canvas's bottom edge. -->
<div
class="pointer-events-none absolute bottom-10 left-3 z-30 flex items-center gap-1.5 text-xs text-hint"
>
<MousePointerClick size={13} />
{nodeHintText}
</div>
{/if}
</div>
<!-- Portalled to the host's overlay anchor (a session preview tab) so the modal covers
that tab rather than the whole page; Disposable owns its place in the overlay stack
and its Escape. -->
<Disposable bind:open={panelModalOpen} bind:this={panelDisposable}>
{#snippet children({ zIndex })}
{#if panelMode === 'modal' && panelModalOpen && workspace}
<Portal target={modalHost ?? 'body'} class="contents">
<!-- svelte-ignore a11y_click_events_have_key_events -->
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="{modalHost ? 'absolute' : 'fixed'} inset-0 flex justify-center px-2 py-6"
style="z-index: {zIndex}"
role="dialog"
>
<div class="absolute inset-0 bg-black/20" onclick={() => (panelModalOpen = false)}></div>
<div
class="relative flex w-full max-w-4xl flex-col overflow-hidden rounded-md border bg-surface shadow-xl"
>
<div class="min-h-0 flex-1 overflow-auto">
{@render detailsBody(workspace)}
</div>
</div>
</div>
</Portal>
{/if}
{/snippet}
</Disposable>
frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte
+469 -32
View File
@@ -1,6 +1,7 @@
<script lang="ts" module>
import type { ComponentType } from 'svelte'
import type { SupportedLanguage } from '$lib/common'
import type { AssetKind, NewSchedule } from '$lib/gen'
export type PipelineInsertKind = {
// Machine-readable id; drives which right-column panel renders.
@@ -8,6 +9,9 @@
label: string
description?: string
icon?: ComponentType
// Its trigger is configured in its own editor right after the script is
// created, so the wizard's last button reads "Configure".
configuredAfterCreate?: boolean
}
export type PipelineInsertPick = {
@@ -22,7 +26,18 @@
// from this prompt via AI (using language + outputKind + the
// upstream input as context) instead of using the seeded template.
aiPrompt?: string
// Set by the schedule step when the picked kind is `schedule`.
schedule?: PipelineInsertSchedule
// Set by the asset step for the table output kinds, replacing the
// generated name.
outputAsset?: { kind: AssetKind; path: string }
}
export type PipelineInsertOptions = Pick<PipelineInsertPick, 'schedule' | 'outputAsset'>
// Everything but the target, which is the script being created. An empty
// `path` means the default `<script>_schedule`.
export type PipelineInsertSchedule = Omit<NewSchedule, 'script_path'>
</script>
<script lang="ts">
@@ -36,10 +51,28 @@
PIPELINE_OUTPUT_KINDS,
type PipelineOutputKind
} from './pipelineTemplates'
import CronInput from '$lib/components/CronInput.svelte'
import type { ScriptLang } from '$lib/gen'
import Label from '$lib/components/Label.svelte'
import TextInput from '$lib/components/text_input/TextInput.svelte'
import { CornerDownLeft, Sparkles } from 'lucide-svelte'
import TextInput, {
inputBaseClass,
inputBorderClass,
inputSizeClasses
} from '$lib/components/text_input/TextInput.svelte'
import DucklakePicker from '$lib/components/DucklakePicker.svelte'
import DatatablePicker from '$lib/components/DatatablePicker.svelte'
import Path from '$lib/components/Path.svelte'
import Section from '$lib/components/Section.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import DateTimeInput from '$lib/components/DateTimeInput.svelte'
import ScheduleAdvancedOptions, {
emptyScheduleAdvanced,
loadDefaultScheduleAdvanced,
scheduleAdvancedCfg
} from '$lib/components/triggers/schedules/ScheduleAdvancedOptions.svelte'
import { workspaceStore } from '$lib/stores'
import { ArrowLeft, ChevronDown, CornerDownLeft, Loader2, Sparkles } from 'lucide-svelte'
import { tick, untrack } from 'svelte'
import { arrowTabNav } from '$lib/attachments/arrowTabNav'
import { selectAndAdvanceTo } from '$lib/attachments/selectAndAdvanceTo'
@@ -49,8 +82,15 @@
pathPrefix?: string
defaultPathSuffix?: string
onPick: (pick: PipelineInsertPick) => void
trigger: import('svelte').Snippet
/** Receives whether the menu is open, e.g. to keep a hover state while it is. */
trigger: import('svelte').Snippet<[{ open: boolean }]>
placement?: 'bottom' | 'top' | 'left' | 'right'
/** Opens the menu each time it changes, for an entry point other than the trigger. */
openSignal?: number
/** With `openSignal`: only the schedule step, for this existing script; its
* confirm hands the schedule to `onSchedule` instead of creating a script. */
scheduleFor?: string
onSchedule?: (schedule: PipelineInsertSchedule) => void
}
let {
@@ -60,7 +100,10 @@
trigger: triggerSnippet,
placement = 'bottom',
defaultPathSuffix,
onPick
onPick,
openSignal,
scheduleFor,
onSchedule
}: Props = $props()
// When there's only one trigger kind, hide the Trigger column entirely
@@ -75,6 +118,107 @@
aiPrompt: undefined as undefined | string
})
let selected = $state(buildEmptySelected())
let menuOpen = $state(false)
$effect(() => {
if (!openSignal) return
untrack(() => {
if (scheduleFor) {
resetWizard()
selected.triggerId = 'schedule'
step = 1
config.schedulePath = `${scheduleFor}_schedule`
loadAdvancedDefaults()
void focusCurrentStep()
}
menuOpen = true
})
})
let selectedKind = $derived(kinds.find((k) => k.id === selected.triggerId))
// Steps after the first, each configuring something the picks call for.
type ConfigStep = 'schedule' | 'asset'
const NAMED_OUTPUT_KINDS: PipelineOutputKind[] = ['materialize', 'datatable', 'ducklake']
let configSteps = $derived.by<ConfigStep[]>(() => {
const steps: ConfigStep[] = []
if (selected.triggerId === 'schedule') steps.push('schedule')
if (selected.outputId && NAMED_OUTPUT_KINDS.includes(selected.outputId)) steps.push('asset')
return steps
})
// 0 is the trigger/language/output/path step; `n` is `configSteps[n - 1]`.
let step = $state(0)
let currentConfigStep = $derived(step > 0 ? configSteps[step - 1] : undefined)
let isLastStep = $derived(step >= configSteps.length)
const buildEmptyConfig = () => ({
schedule: '0 0 12 * *',
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone,
cronVersion: 'v2',
validCron: true,
// Follows the script path until the Path field is edited.
schedulePath: '',
schedulePathDirty: false,
schedulePathError: '',
summary: '',
description: '',
pauseUntil: false,
pausedUntil: undefined as string | undefined,
advanced: emptyScheduleAdvanced(),
// The workspace's default handlers, loaded when the schedule step opens.
advancedDefaults: 'unloaded' as 'unloaded' | 'loading' | 'loaded',
// `store` is the data table or DuckLake catalog name. The table name
// follows the script name until the user types in it.
asset: undefined as
| undefined
| { kind: 'ducklake' | 'datatable'; store: string | undefined; table: string },
tableEdited: false
})
let config = $state(buildEmptyConfig())
// A new schedule starts from the workspace's default handlers, as in the schedule
// drawer. The step cannot be confirmed until they are in.
function loadAdvancedDefaults() {
const ws = $workspaceStore
if (!ws || config.advancedDefaults !== 'unloaded') return
const target = config
target.advancedDefaults = 'loading'
loadDefaultScheduleAdvanced(ws)
.then((d) => (target.advanced = d))
.catch(() => {})
.finally(() => (target.advancedDefaults = 'loaded'))
}
function resetWizard() {
selected = buildEmptySelected()
config = buildEmptyConfig()
step = 0
}
const TABLE_NAME_RE = /^[^/\s]+$/
// The path grammar of `Path.svelte`: word segments, no empty or trailing ones.
const SCRIPT_NAME_RE = /^[\w-]+(\/[\w-]+)*$/
let scriptNameValid = $derived(SCRIPT_NAME_RE.test(selected.scriptPath.trim()))
let assetValid = $derived(!!config.asset?.store && TABLE_NAME_RE.test(config.asset.table.trim()))
let stepValid = $derived(
currentConfigStep === 'schedule'
? config.validCron &&
!!config.schedule.trim() &&
!config.schedulePathError &&
config.advancedDefaults === 'loaded'
: currentConfigStep === 'asset'
? assetValid
: scriptNameValid && !!selected.triggerId && !!selected.language && !!selected.outputId
)
function enterAssetStep() {
const kind = selected.outputId === 'datatable' ? 'datatable' : 'ducklake'
if (config.asset?.kind !== kind) {
config.asset = { kind, store: 'main', table: '' }
config.tableEdited = false
}
if (!config.tableEdited && config.asset) {
config.asset.table = selected.scriptPath.trim().split('/').pop() ?? ''
}
}
// Refs to the column containers so Enter inside one column can hand
// focus off to the first tabbable in the next column.
@@ -104,16 +248,73 @@
return out
}
// A step change re-renders the popover body, dropping focus to <body>, where
// keystrokes (Enter, Cmd+Enter) no longer reach the step's key handlers.
let contentEl: HTMLElement | undefined = $state()
async function focusCurrentStep() {
await tick()
const target =
step === 0
? pathEl?.querySelector('input')
: contentEl?.querySelector<HTMLElement>('[data-autofocus] input')
target?.focus()
}
function goBack() {
step -= 1
void focusCurrentStep()
}
function scheduleConfig(): PipelineInsertSchedule {
return {
path: config.schedulePath,
schedule: config.schedule.trim(),
timezone: config.timezone,
cron_version: config.cronVersion,
summary: config.summary.trim() || undefined,
description: config.description,
paused_until: config.pauseUntil ? config.pausedUntil : undefined,
is_flow: false,
args: {},
...scheduleAdvancedCfg($state.snapshot(config.advanced))
}
}
function confirm(close: () => void) {
if (scheduleFor) {
if (!stepValid) return
onSchedule?.(scheduleConfig())
close()
return
}
const suffix = selected.scriptPath.trim()
if (!suffix || !selected.triggerId || !selected.language || !selected.outputId) return
if (!stepValid) return
if (!isLastStep) {
step += 1
if (configSteps[step - 1] === 'asset') enterAssetStep()
if (configSteps[step - 1] === 'schedule') {
loadAdvancedDefaults()
if (!config.schedulePathDirty) config.schedulePath = `${pathPrefix}${suffix}_schedule`
}
void focusCurrentStep()
return
}
const trimmedPrompt = selected.aiPrompt?.trim()
onPick({
kindId: selected.triggerId,
language: selected.language,
path: pathPrefix + suffix,
outputKind: selected.outputId,
aiPrompt: trimmedPrompt && trimmedPrompt.length > 0 ? trimmedPrompt : undefined
aiPrompt: trimmedPrompt && trimmedPrompt.length > 0 ? trimmedPrompt : undefined,
schedule: configSteps.includes('schedule') ? scheduleConfig() : undefined,
outputAsset:
configSteps.includes('asset') && config.asset
? {
kind: config.asset.kind as AssetKind,
path: `${config.asset.store}/${config.asset.table.trim()}`
}
: undefined
})
close()
}
@@ -123,9 +324,18 @@
enableFlyTransition
contentClasses={twMerge(
'p-0 bg-surface overflow-hidden relative transition-height',
showBottomPanel ? 'h-[26rem]' : 'h-[22rem]'
// The first step's columns (w-56 + w-48 + w-80), held across steps.
singleKind ? 'w-[32rem]' : 'w-[46rem]',
currentConfigStep === 'schedule'
? 'h-[27rem]'
: currentConfigStep === 'asset'
? 'h-[14rem]'
: showBottomPanel
? 'h-[26rem]'
: 'h-[22rem]'
)}
class="inline-block"
bind:isOpen={menuOpen}
usePointerDownOutside
floatingConfig={{
placement,
@@ -136,24 +346,30 @@
fitViewport: true,
overlap: false
}}
onClose={() => (selected = buildEmptySelected())}
onClose={resetWizard}
>
{#snippet trigger()}
{@render triggerSnippet?.()}
{@render triggerSnippet?.({ open: menuOpen })}
{/snippet}
{#snippet content({ close })}
<div class="flex flex-col h-full">
<div class={'flex flex-row transition-height divide-x overflow-y-scroll'}>
{@render topSection()}
</div>
<div
class={twMerge(
'flex flex-col gap-5 grow transition-height px-4 border-t',
showBottomPanel ? 'h-[14rem] py-4' : 'h-0'
)}
>
{@render bottomSection(close)}
</div>
<div class="h-full" bind:this={contentEl}>
{#if currentConfigStep}
{@render configStepSection(currentConfigStep, close)}
{:else}
<div class="flex flex-col h-full">
<div class={'flex flex-row transition-height divide-x overflow-y-scroll'}>
{@render topSection()}
</div>
<div
class={twMerge(
'flex flex-col gap-5 grow transition-height px-4 border-t',
showBottomPanel ? 'h-[14rem] py-4' : 'h-0'
)}
>
{@render bottomSection(close)}
</div>
</div>
{/if}
</div>
{/snippet}
</Popover>
@@ -170,7 +386,10 @@
<Button
variant="subtle"
btnClasses={'text-left'}
onClick={() => (selected.triggerId = k.id)}
onClick={() => {
selected.triggerId = k.id
if (k.id === 'schedule') loadAdvancedDefaults()
}}
selected={isSelected}
>
{#if k.icon}
@@ -264,14 +483,30 @@
{#snippet bottomSection(close: () => void)}
<Label label="Path">
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div bind:this={pathEl} class="flex" onkeydown={selectAndAdvanceTo(() => aiPromptEl ?? saveEl)}>
<div
bind:this={pathEl}
class="flex"
onkeydown={(e) =>
(e.metaKey || e.ctrlKey) && e.key === 'Enter'
? (e.preventDefault(), e.stopPropagation(), confirm(close))
: selectAndAdvanceTo(() => aiPromptEl ?? saveEl)(e)}
>
<div
class="border rounded-md rounded-r-none border-r-0 text-xs w-fit shrink-0 whitespace-nowrap flex items-center px-2 text-secondary bg-surface-input"
>
{pathPrefix}
</div>
<TextInput bind:value={selected.scriptPath} class="rounded-l-none" />
<TextInput
bind:value={selected.scriptPath}
class="rounded-l-none"
error={!scriptNameValid && !!selected.scriptPath.trim()}
/>
</div>
{#if !scriptNameValid && selected.scriptPath.trim()}
<span class="text-2xs text-red-600 dark:text-red-400">
Letters, digits, <code>_</code> and <code>-</code>, in <code>/</code>-separated segments
</span>
{/if}
</Label>
{#if !$copilotInfo.workspaceDisabled}
<Label label="AI Prompt (optional)">
@@ -292,15 +527,217 @@
</div>
</Label>
{/if}
{@const hasAiPrompt = !!selected.aiPrompt?.trim()}
<div class="ml-auto" bind:this={saveEl}>
<Button
variant="accent"
btnClasses="w-fit"
disabled={!selected.scriptPath.trim()}
onClick={() => confirm(close)}
startIcon={hasAiPrompt ? { icon: Sparkles } : undefined}
shortCut={{ Icon: CornerDownLeft }}>{hasAiPrompt ? 'Generate' : 'Create'}</Button
>
{@render confirmButton(close)}
</div>
{/snippet}
{#snippet confirmButton(close: () => void)}
{@const hasAiPrompt = !!selected.aiPrompt?.trim()}
<Button
variant="accent"
btnClasses="w-fit"
disabled={!stepValid}
onClick={() => confirm(close)}
startIcon={isLastStep && hasAiPrompt ? { icon: Sparkles } : undefined}
shortCut={{ Icon: CornerDownLeft }}
>{scheduleFor
? 'Create schedule'
: !isLastStep
? step === 0
? 'Configure'
: 'Next'
: hasAiPrompt
? 'Generate'
: selectedKind?.configuredAfterCreate
? 'Configure'
: 'Create'}</Button
>
{/snippet}
{#snippet configStepSection(configStep: ConfigStep, close: () => void)}
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="flex flex-col h-full"
onkeydown={(e) => {
if (e.key !== 'Enter') return
const target = e.target as HTMLElement | null
// Cmd/Ctrl+Enter confirms from anywhere; bare Enter only from the step's own
// inputs, not from the Advanced fields (a textarea, pickers with their own Enter).
const bareEnterSubmits = target?.tagName === 'INPUT' && !target.closest('[data-advanced]')
if (e.metaKey || e.ctrlKey || bareEnterSubmits) {
e.preventDefault()
e.stopPropagation()
confirm(close)
}
}}
>
<div class="flex items-center gap-2 px-2 py-2 border-b">
{#if !scheduleFor}
<Button
variant="subtle"
unifiedSize="sm"
iconOnly
startIcon={{ icon: ArrowLeft }}
onClick={goBack}
title="Back"
/>
{/if}
<span class="text-xs font-semibold text-emphasis {scheduleFor ? 'pl-2' : ''}">
{scheduleFor
? `Schedule for ${scheduleFor}`
: configStep === 'schedule'
? 'Schedule'
: 'Output asset'}
</span>
{#if !scheduleFor}
<span class="text-2xs text-hint ml-auto pr-2"
>Step {step + 1} of {configSteps.length + 1}</span
>
{/if}
</div>
<div class="flex flex-col gap-2 grow overflow-auto p-4">
{#if configStep === 'schedule'}
<span class="text-2xs text-secondary">
Saved as a draft schedule and deployed along with the pipeline.
</span>
<div data-autofocus class="contents">
<CronInput
bind:schedule={config.schedule}
bind:timezone={config.timezone}
bind:validCRON={config.validCron}
bind:cronVersion={config.cronVersion}
/>
</div>
<div data-advanced class="mt-4">
<Section label="Advanced" collapsable initiallyCollapsed>
<div class="flex flex-col gap-6">
<Label label="Summary">
<TextInput
bind:value={config.summary}
inputProps={{ placeholder: 'Short summary to be displayed when listed' }}
/>
</Label>
<Label label="Path">
<Path
bind:path={config.schedulePath}
bind:dirty={config.schedulePathDirty}
bind:error={config.schedulePathError}
initialPath=""
checkInitialPathExistence
namePlaceholder="schedule"
kind="schedule"
/>
</Label>
<Label label="Description">
<TextInput
underlyingInputEl="textarea"
bind:value={config.description}
inputProps={{ placeholder: 'What this schedule does and how to use it' }}
/>
</Label>
<div class="flex flex-col gap-1">
<Toggle
options={{
right: 'Pause schedule until...',
rightTooltip:
'Pausing the schedule will program the next job to run as if the schedule starts at the time the pause is lifted, instead of now.'
}}
bind:checked={config.pauseUntil}
/>
{#if config.pauseUntil}
<DateTimeInput bind:value={config.pausedUntil} />
{/if}
</div>
<!-- Rendered once the workspace defaults are in, so they never
overwrite a handler picked while they load. -->
{#if config.advancedDefaults === 'loaded'}
<ScheduleAdvancedOptions
wsId={$workspaceStore}
itemKind="script"
canWrite
bind:errorHandlerSelected={config.advanced.errorHandlerSelected}
bind:errorHandlerPath={config.advanced.errorHandlerPath}
bind:errorHandleritemKind={config.advanced.errorHandleritemKind}
bind:errorHandlerExtraArgs={config.advanced.errorHandlerExtraArgs}
bind:wsErrorHandlerMuted={config.advanced.wsErrorHandlerMuted}
bind:failedTimes={config.advanced.failedTimes}
bind:failedExact={config.advanced.failedExact}
bind:recoveryHandlerSelected={config.advanced.recoveryHandlerSelected}
bind:recoveryHandlerPath={config.advanced.recoveryHandlerPath}
bind:recoveryHandlerItemKind={config.advanced.recoveryHandlerItemKind}
bind:recoveryHandlerExtraArgs={config.advanced.recoveryHandlerExtraArgs}
bind:recoveredTimes={config.advanced.recoveredTimes}
bind:successHandlerSelected={config.advanced.successHandlerSelected}
bind:successHandlerPath={config.advanced.successHandlerPath}
bind:successHandlerItemKind={config.advanced.successHandlerItemKind}
bind:successHandlerExtraArgs={config.advanced.successHandlerExtraArgs}
bind:retry={config.advanced.retry}
bind:dynamicSkipPath={config.advanced.dynamicSkipPath}
bind:tag={config.advanced.tag}
/>
{:else}
<div class="flex items-center gap-1.5 text-xs text-secondary">
<Loader2 size={12} class="animate-spin" />
Loading the workspace's default handlers…
</div>
{/if}
</div>
</Section>
</div>
{:else if configStep === 'asset' && config.asset}
<Label label="Name">
<div
class={twMerge(
inputBaseClass,
inputBorderClass({ error: !assetValid }),
inputSizeClasses.md,
'flex flex-row items-center gap-0 py-0 pl-2 pr-0'
)}
>
<span class="text-xs text-secondary shrink-0">{config.asset.kind}://</span>
{#if config.asset.kind === 'datatable'}
<DatatablePicker
bind:value={config.asset.store}
class="shrink-0"
selectInputClass="!border-none"
placeholder="data table"
useContentEditable
RightIcon={ChevronDown}
/>
{:else}
<DucklakePicker
bind:value={config.asset.store}
class="shrink-0"
selectInputClass="!border-none"
placeholder="catalog"
useContentEditable
RightIcon={ChevronDown}
/>
{/if}
<span class="text-sm text-secondary">/</span>
<div data-autofocus class="grow flex">
<TextInput
bind:value={config.asset.table}
class="!border-none grow"
inputProps={{
placeholder: 'table',
oninput: () => (config.tableEdited = true)
}}
/>
</div>
</div>
</Label>
<span class="text-2xs text-hint">
{config.asset.kind === 'datatable' ? 'Data table' : 'DuckLake catalog'}, then the table
name
</span>
{/if}
</div>
<div class="flex px-4 py-3 border-t">
<div class="ml-auto">
{@render confirmButton(close)}
</div>
</div>
</div>
{/snippet}
frontend/src/lib/components/assets/AssetGraph/PipelineNodeCard.svelte
+174
View File
@@ -0,0 +1,174 @@
<!--
@component
The body every pipeline graph node shares: icon, a kind line ("Script · draft", "Data table"),
a title, and trailing chips. Node components add only what is theirs — the icon, the chips,
and the controls hung around the card.
-->
<script lang="ts">
import { NODE } from '$lib/components/graph/util'
import {
PIPELINE_NODE_EXTRA_ROW,
PIPELINE_NODE_HEADER,
PIPELINE_NODE_HEADER_CARD_PAD,
PIPELINE_NODE_HEIGHT
} from './assetGraphLayout'
import { twMerge } from 'tailwind-merge'
import type { Snippet } from 'svelte'
interface Props {
icon: Snippet
kindLabel: string
title: string
/** Hover text; the path, when the title is a summary. */
tooltip?: string
/** Scripts sit on the main surface, assets and triggers a step down; in the
* assets-only view, which has no scripts, assets take the main surface. */
surface?: 'primary' | 'secondary'
selected?: boolean
/** Not deployed yet: dashed edge. */
draft?: boolean
/** Feedback states, which recolor the whole card. `error` is a misconfigured
* node, styled like a failed flow step. */
tone?: 'danger' | 'error' | 'success' | 'running'
/** Makes the whole card a button. */
onclick?: (e: MouseEvent) => void
/** Between the icon and the text: a live status the eye should find first. */
leading?: Snippet
trailing?: Snippet
/** Defaults to `NODE.width`. */
width?: number
/** A line under the title, in the text column. */
subtitle?: Snippet
/** A tab on the card's top edge. Only on a card without `onclick`, since it
* may hold buttons of its own. */
header?: Snippet
/** The header's script is selected: its edge takes the selected colour. */
headerSelected?: boolean
/** The header's script has an error: its edge goes red. */
headerTone?: 'error'
}
let {
icon,
kindLabel,
title,
tooltip,
surface = 'secondary',
selected = false,
draft = false,
tone,
onclick,
leading,
trailing,
subtitle,
header,
headerSelected = false,
headerTone,
width = NODE.width
}: Props = $props()
const cardClass = $derived(
twMerge(
'flex items-center w-full text-left font-normal rounded-md drop-shadow-sm overflow-hidden border transition-colors',
surface === 'primary' ? 'bg-surface' : 'bg-surface-secondary',
'border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500',
// Like a flow step: a card with a background needs no edge, only its shadow,
// until one of the states below draws one.
surface === 'primary' &&
!draft &&
!selected &&
!tone &&
'border-transparent dark:border-transparent hover:border-transparent dark:hover:border-transparent',
draft && 'border-dashed border-gray-400 dark:border-gray-500',
// Like a selected flow step: the fill and the edge colour change, the edge's
// width and dashes stay.
// Each dark: variant too: the base edge sets its own dark colour, which a plain
// border class does not override.
selected &&
'bg-surface-accent-selected border-border-selected dark:border-border-selected hover:border-border-selected dark:hover:border-border-selected',
tone === 'danger' &&
'bg-red-50 dark:bg-red-900/30 border-dashed border-red-400 dark:border-red-500 hover:bg-red-100 dark:hover:bg-red-900/40',
tone === 'error' &&
(selected
? 'bg-red-200 dark:bg-red-600 border-red-500 dark:border-red-500 hover:border-red-500 dark:hover:border-red-500'
: 'bg-red-100 dark:bg-red-700 border-transparent dark:border-transparent hover:border-transparent dark:hover:border-transparent'),
tone === 'success' &&
'bg-green-50 dark:bg-green-900/30 border-green-500 dark:border-green-600',
tone === 'running' &&
'bg-amber-50 dark:bg-amber-900/30 border-amber-400 dark:border-amber-600 animate-pulse'
)
)
const minHeight = $derived(PIPELINE_NODE_HEIGHT + (subtitle ? PIPELINE_NODE_EXTRA_ROW : 0))
const toneText = $derived(
tone === 'danger'
? 'text-red-700 dark:text-red-400'
: tone === 'error'
? 'text-red-700 dark:text-red-200'
: tone === 'success'
? 'text-green-700 dark:text-green-400'
: undefined
)
</script>
{#snippet body()}
<span class="shrink-0 ml-3 mr-2.5 flex items-center">{@render icon()}</span>
{@render leading?.()}
<span class="flex flex-col min-w-0 flex-1 pr-1 py-1 leading-tight">
<span class={twMerge('text-2xs truncate text-secondary', toneText)}>{kindLabel}</span>
<span class={twMerge('text-xs truncate text-emphasis', selected && 'text-accent', toneText)}
>{title}</span
>
{#if subtitle}
<span class="flex items-center min-w-0 mt-1">{@render subtitle()}</span>
{/if}
</span>
{#if trailing}
<span class="flex items-center shrink-0 pr-2">{@render trailing()}</span>
{/if}
{/snippet}
{#if onclick}
<button
type="button"
{onclick}
class={cardClass}
style="width: {width}px; min-height: {minHeight}px;"
title={tooltip}
>
{@render body()}
</button>
{:else if header}
<!-- The header is a tab on the card's top edge, inset from its sides. -->
<div class="flex flex-col" style="width: {width}px;">
<div
class={twMerge(
'mx-2 flex items-stretch min-w-0 rounded-t-md overflow-hidden bg-surface border border-b-0 border-gray-200 dark:border-gray-700',
// Like an errored card: red, and a stronger red when selected.
headerTone === 'error'
? headerSelected
? 'border-red-500 dark:border-red-500'
: 'border-red-300 dark:border-red-600'
: headerSelected && 'border-border-selected dark:border-border-selected'
)}
style="height: {PIPELINE_NODE_HEADER}px;"
>
{@render header()}
</div>
<div
class={twMerge(cardClass, !selected && !tone && 'bg-surface-tertiary')}
style="min-height: {minHeight + PIPELINE_NODE_HEADER_CARD_PAD}px; padding-block: {PIPELINE_NODE_HEADER_CARD_PAD /
2}px;"
title={tooltip}
>
{@render body()}
</div>
</div>
{:else}
<div
class={cardClass}
style="width: {width}px; min-height: {minHeight}px;"
title={tooltip}
>
{@render body()}
</div>
{/if}
frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte
+32 -9
View File
@@ -1,6 +1,6 @@
<script lang="ts">
import type { AssetKind, Script } from '$lib/gen'
import { CalendarClock, Loader2, Play, Upload, Zap } from 'lucide-svelte'
import { CalendarClock, Code2, Loader2, Play, Upload, Zap } from 'lucide-svelte'
import Button from '$lib/components/common/button/Button.svelte'
import HighlightCode from '$lib/components/HighlightCode.svelte'
import PipelineRunForm from './PipelineRunForm.svelte'
@@ -44,6 +44,12 @@
// "green / ready" state and seeds the whole-pipeline run. `isValid` is the
// full-schema validity (all required fields, not just the S3 file).
onArgsChange?: (path: string, args: Record<string, any>, isValid: boolean) => void
// Just the run form and the runs, without the source: what a data-upload
// click asks for. `onShowScript` switches to the script itself.
inputsOnly?: boolean
onShowScript?: () => void
// `onRun` previews a draft's own content, so a draft can run too.
runsDrafts?: boolean
}
let {
@@ -57,8 +63,12 @@
runsPendingJobId,
onRunCompleted,
initialArgs,
onArgsChange
onArgsChange,
inputsOnly = false,
onShowScript,
runsDrafts = false
}: Props = $props()
let runBlockedByDraft = $derived(isDraft && !runsDrafts)
// Seed once from the persisted args (see initialArgs). Cloned so the run
// form mutates its own copy, not the page's stored snapshot.
@@ -139,7 +149,7 @@
</script>
<div class="flex flex-col h-full">
{#if isDraft}
{#if runBlockedByDraft}
<div
class="shrink-0 flex items-center gap-2 px-3 py-1.5 text-2xs bg-amber-50 dark:bg-amber-900/30 border-b border-amber-200 dark:border-amber-900/60 text-amber-700 dark:text-amber-400"
>
@@ -168,13 +178,24 @@
{/if}
</span>
<div class="flex items-center gap-1.5">
{#if inputsOnly && onShowScript}
<Button
variant="subtle"
unifiedSize="sm"
startIcon={{ icon: Code2 }}
onclick={onShowScript}
title="Show the script instead of its inputs"
>
Open script
</Button>
{/if}
{#if hasCascade}
<Button
variant="accent-secondary"
unifiedSize="sm"
startIcon={{ icon: running ? Loader2 : Zap }}
onclick={() => run(true)}
disabled={isDraft || running || !runValid}
disabled={runBlockedByDraft || running || !runValid}
title={`Run this script with these inputs, then run its ${downstreamCount} downstream pipeline script${downstreamCount === 1 ? '' : 's'} in order`}
>
Run + downstream
@@ -185,8 +206,8 @@
unifiedSize="sm"
startIcon={{ icon: running ? Loader2 : Play }}
onclick={() => run(false)}
disabled={isDraft || running || !runValid || !onRun}
title={isDraft
disabled={runBlockedByDraft || running || !runValid || !onRun}
title={runBlockedByDraft
? 'Deploy this draft to run it for real'
: hasCascade
? 'Run just this step — downstream scripts are not triggered'
@@ -210,9 +231,11 @@
{/key}
{/if}
</div>
<div class="flex-1 min-h-0 overflow-auto text-xs p-3">
<HighlightCode code={script.content ?? ''} language={script.language} />
</div>
{#if !inputsOnly}
<div class="flex-1 min-h-0 overflow-auto text-xs p-3">
<HighlightCode code={script.content ?? ''} language={script.language} />
</div>
{/if}
</div>
</Pane>
<Pane size={45} minSize={20}>
frontend/src/lib/components/assets/AssetGraph/PipelineSetupSignpost.svelte
+40 -43
View File
@@ -3,6 +3,8 @@
import { WorkspaceService } from '$lib/gen'
import { base } from '$lib/base'
import { CircleCheck, CircleAlert, Database, HardDrive, ArrowRight } from 'lucide-svelte'
import Alert from '$lib/components/common/alert/Alert.svelte'
import { Button } from '$lib/components/common'
interface Props {
workspace: string
@@ -73,48 +75,43 @@
</script>
{#if show}
<div
class="flex flex-col gap-3 rounded-lg border border-amber-300 dark:border-amber-900/60 bg-amber-50 dark:bg-amber-950/30 p-4"
>
<div class="flex items-start gap-2">
<CircleAlert size={18} class="text-amber-600 dark:text-amber-400 mt-0.5 shrink-0" />
<div class="flex flex-col gap-0.5">
<h3 class="text-sm font-semibold text-emphasis">Finish setting up pipelines</h3>
<p class="text-xs text-tertiary">
Pipelines materialize data into DuckLake tables backed by object storage. Configure the
following before your first pipeline can run.
</p>
</div>
<Alert type="warning" title="Finish setting up pipelines" hideIcon>
<div class="flex flex-col gap-3">
<p>
Pipelines materialize data into DuckLake tables backed by object storage. Configure the
following before your first pipeline can run.
</p>
<ul class="flex flex-col gap-2">
{#each steps as step (step.title)}
{@const Icon = step.icon}
<li class="flex items-center gap-3 rounded-md border bg-surface-tertiary px-3 py-2">
{#if step.done === true}
<CircleCheck size={16} class="text-green-600 dark:text-green-400 shrink-0" />
{:else if step.done === false}
<CircleAlert size={16} class="text-yellow-600 dark:text-yellow-400 shrink-0" />
{:else}
<Icon size={16} class="text-secondary shrink-0" />
{/if}
<div class="flex flex-col min-w-0 flex-1">
<span class="text-xs font-semibold text-emphasis">{step.title}</span>
<span class="text-2xs text-secondary">{step.description}</span>
</div>
{#if step.done !== true}
<Button
variant="default"
unifiedSize="sm"
href={step.href}
endIcon={{ icon: ArrowRight }}
wrapperClasses="shrink-0"
>
{step.cta}
</Button>
{:else}
<span class="shrink-0 text-2xs text-green-700 dark:text-green-400">Configured</span>
{/if}
</li>
{/each}
</ul>
</div>
<ul class="flex flex-col gap-2">
{#each steps as step (step.title)}
{@const Icon = step.icon}
<li class="flex items-center gap-3 rounded-md border bg-surface px-3 py-2">
{#if step.done === true}
<CircleCheck size={16} class="text-green-600 dark:text-green-400 shrink-0" />
{:else if step.done === false}
<CircleAlert size={16} class="text-amber-600 dark:text-amber-400 shrink-0" />
{:else}
<Icon size={16} class="text-tertiary shrink-0" />
{/if}
<div class="flex flex-col min-w-0 flex-1">
<span class="text-xs font-semibold text-primary">{step.title}</span>
<span class="text-2xs text-tertiary">{step.description}</span>
</div>
{#if step.done !== true}
<a
href={step.href}
class="shrink-0 inline-flex items-center gap-1 text-xs text-blue-500 hover:underline whitespace-nowrap"
>
{step.cta}
<ArrowRight size={12} />
</a>
{:else}
<span class="shrink-0 text-2xs text-green-700 dark:text-green-400">Configured</span>
{/if}
</li>
{/each}
</ul>
</div>
</Alert>
{/if}
frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte
+59 -43
View File
@@ -1,18 +1,10 @@
<script lang="ts">
import { sendUserToast } from '$lib/utils'
import { workspaceStore } from '$lib/stores'
import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte'
import type { NativeTriggerKind } from './types'
import {
EmailTriggerService,
GcpTriggerService,
KafkaTriggerService,
MqttTriggerService,
AmqpTriggerService,
NatsTriggerService,
PostgresTriggerService,
ScheduleService,
SqsTriggerService
} from '$lib/gen'
import type { NativeTriggerKind, PipelineTriggerDraftKind } from './types'
import type { Schedule } from '$lib/gen'
import { deleteTriggerRow } from './pipelineTriggerDraftDeploy'
import KafkaTriggerEditor from '$lib/components/triggers/kafka/KafkaTriggerEditor.svelte'
import MqttTriggerEditor from '$lib/components/triggers/mqtt/MqttTriggerEditor.svelte'
import AmqpTriggerEditor from '$lib/components/triggers/amqp/AmqpTriggerEditor.svelte'
@@ -22,6 +14,10 @@
import GcpTriggerEditor from '$lib/components/triggers/gcp/GcpTriggerEditor.svelte'
import EmailTriggerEditor from '$lib/components/triggers/email/EmailTriggerEditor.svelte'
import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte'
import {
loadDefaultScheduleAdvanced,
scheduleAdvancedCfg
} from '$lib/components/triggers/schedules/ScheduleAdvancedOptions.svelte'
import WebhookEditor from '$lib/components/triggers/webhook/WebhookEditor.svelte'
import { setOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
@@ -110,6 +106,56 @@
}
}
/** The kind's full editor over a pipeline-local draft; Save hands the config back,
* and the drawer stays open when `onSave` refuses it.
* `saved` is the draft being edited (absent for a new one), applied over `defaults`. */
export function openTriggerDraft(
kind: PipelineTriggerDraftKind,
scriptPath: string,
defaults: Record<string, any>,
saved: Record<string, any> | undefined,
onSave: (cfg: Record<string, any>) => boolean
) {
switch (kind) {
case 'schedule': {
// A new draft starts from the workspace's default handlers, like a new
// schedule does; the drawer only applies them when it opens without one.
const open = (advanced: Record<string, any>) =>
scheduleEditor?.openDraft(
{
is_flow: false,
args: {},
...advanced,
...defaults,
...saved,
script_path: scriptPath
} as Schedule,
onSave
)
if (saved || !$workspaceStore) return open({})
return loadDefaultScheduleAdvanced($workspaceStore)
.then((a) => open(scheduleAdvancedCfg(a)))
.catch(() => open({}))
}
case 'kafka':
return kafkaEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'mqtt':
return mqttEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'amqp':
return amqpEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'nats':
return natsEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'postgres':
return postgresEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'sqs':
return sqsEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'gcp':
return gcpEditor?.openDraft(scriptPath, defaults, saved, onSave)
case 'email':
return emailEditor?.openDraft(scriptPath, defaults, saved, onSave)
}
}
// Open the webhook drawer (endpoint URLs + token flow). Caller owns the
// draft guard.
export function openWebhook(scriptPath: string) {
@@ -133,37 +179,7 @@
const { kind, path: triggerPath } = triggerDeleteTarget
triggerDeleteLoading = true
try {
switch (kind) {
case 'schedule':
await ScheduleService.deleteSchedule({ workspace, path: triggerPath })
break
case 'kafka':
await KafkaTriggerService.deleteKafkaTrigger({ workspace, path: triggerPath })
break
case 'mqtt':
await MqttTriggerService.deleteMqttTrigger({ workspace, path: triggerPath })
break
case 'amqp':
await AmqpTriggerService.deleteAmqpTrigger({ workspace, path: triggerPath })
break
case 'nats':
await NatsTriggerService.deleteNatsTrigger({ workspace, path: triggerPath })
break
case 'postgres':
await PostgresTriggerService.deletePostgresTrigger({ workspace, path: triggerPath })
break
case 'sqs':
await SqsTriggerService.deleteSqsTrigger({ workspace, path: triggerPath })
break
case 'gcp':
await GcpTriggerService.deleteGcpTrigger({ workspace, path: triggerPath })
break
case 'email':
await EmailTriggerService.deleteEmailTrigger({ workspace, path: triggerPath })
break
default:
return
}
if (!(await deleteTriggerRow(kind, triggerPath, workspace))) return
sendUserToast(`Deleted ${kind} trigger "${triggerPath}"`)
triggerDeleteTarget = undefined
onUpdate()
frontend/src/lib/components/assets/AssetGraph/RunStateChip.svelte
+37
View File
@@ -0,0 +1,37 @@
<script lang="ts">
import { twMerge } from 'tailwind-merge'
import { CheckCircle2, Loader2, XCircle } from 'lucide-svelte'
import type { RunnableRunState } from './activeRunnables.svelte'
let { runState: rs, class: className }: { runState: RunnableRunState; class?: string } = $props()
</script>
<div
class={twMerge(
'shrink-0 flex items-center gap-0.5 px-1 py-0.5 rounded-sm border',
rs.status === 'running'
? 'bg-blue-100 dark:bg-blue-900/40 text-blue-700 dark:text-blue-300 border-blue-300 dark:border-blue-700'
: rs.status === 'success'
? 'bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300 border-emerald-300 dark:border-emerald-700'
: 'bg-red-100 dark:bg-red-900/40 text-red-700 dark:text-red-300 border-red-300 dark:border-red-700',
className
)}
title={`${
rs.status === 'running'
? 'Running now'
: rs.status === 'success'
? 'Last run succeeded'
: 'Last run failed'
}${rs.runs > 0 ? ` — ran ${rs.runs}× this session` : ''}`}
>
{#if rs.status === 'running'}
<Loader2 size={10} class="animate-spin" />
{:else if rs.status === 'success'}
<CheckCircle2 size={10} />
{:else}
<XCircle size={10} />
{/if}
{#if rs.runs > 0}
<span class="text-3xs leading-none tabular-nums">×{rs.runs}</span>
{/if}
</div>
frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte
+121 -163
View File
@@ -1,10 +1,8 @@
<script lang="ts">
import { Handle, Position } from '@xyflow/svelte'
import {
CheckCircle2,
ChevronDown,
Code2,
EllipsisVertical,
GitBranch,
Layers,
Loader2,
@@ -15,17 +13,20 @@
Target,
Timer,
Trash2,
XCircle,
Zap
} from 'lucide-svelte'
import DbtIcon from '$lib/components/icons/DbtIcon.svelte'
import { twMerge } from 'tailwind-merge'
import { preventDefault, stopPropagation } from 'svelte/legacy'
import type { GraphUsageKind } from './types'
import type { RunnableRunState } from './activeRunnables.svelte'
import { parseDurationSecs } from './parsePipelineAnnotations'
import { NODE } from '$lib/components/graph/util'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import LanguageIcon from '$lib/components/common/languageIcons/LanguageIcon.svelte'
import PipelineNodeCard from './PipelineNodeCard.svelte'
import NodeActionsMenu from './NodeActionsMenu.svelte'
import NodeFixButton, { type NodeFix } from './NodeFixButton.svelte'
import NodeOnDot from './NodeOnDot.svelte'
import RunStateChip from './RunStateChip.svelte'
import type { ScriptLang } from '$lib/gen'
import Popover from '$lib/components/meltComponents/Popover.svelte'
import type { Item } from '$lib/utils'
import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte'
@@ -35,8 +36,16 @@
interface Props {
data: {
/** What is wrong with the node, and how to fix it: shows the Fix pill. */
fix?: NodeFix
/** Starts dragging the top dot onto an asset to subscribe the script to it. */
onStartOnDrag?: (e: PointerEvent) => void
runnable_kind: GraphUsageKind
path: string
summary?: string
language?: ScriptLang
// Why the script is misconfigured, when it is: renders it red.
error?: string
in_pipeline?: boolean
partition_kind?: 'daily' | 'hourly' | 'weekly' | 'monthly' | 'dynamic'
freshness?: string
@@ -101,10 +110,11 @@
}
let { data, selected = false }: Props = $props()
// The icon alone conveys "script" vs "flow"; the uppercase kind label was
// visually noisy and redundant. Tooltip on hover surfaces the path in
// full when truncated.
let Icon = $derived(data.runnable_kind === 'flow' ? GitBranch : Code2)
let kindLabel = $derived(
data.runnable_kind === 'flow' ? 'Flow' : data.dbt ? 'dbt project' : 'Script'
)
// The title shows the summary when there is one, so the tooltip keeps the path.
let nodeTooltip = $derived(
data.unsaved
? `${data.path} (unsaved draft)`
@@ -114,7 +124,6 @@
)
let hover = $state(false)
let menuOpen = $state(false)
let running = $state(false)
// Amber "computing now" surface, gated so only the replay player lights it up.
@@ -214,131 +223,106 @@
<!--
Mirrors the flow editor's step styling (getNodeColorClasses): muted
surface-tertiary fill with a quiet gray border, accent only for the
selected state, dashes for unsaved drafts. The 2px dashed stroke is
necessary for the dash gaps to be readable; 1px disappears into the
fill.
selected state, dashes for unsaved drafts — 1px, like the trigger
nodes' dashed outline, so every draft node reads the same.
-->
<div
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden border transition-colors',
'bg-surface border-gray-400 dark:border-gray-600 hover:border-gray-500 dark:hover:border-gray-500',
selected && 'bg-surface-accent-selected border-border-selected',
data.unsaved && 'border-2 border-dashed border-gray-400 dark:border-gray-500',
computingNow &&
'bg-amber-50 dark:bg-amber-900/30 border-amber-400 dark:border-amber-600 animate-pulse'
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={nodeTooltip}
>
<Icon size={14} class={`shrink-0 ml-2 mr-2 ${selected ? 'text-accent' : 'text-secondary'}`} />
<span class="flex-1 min-w-0 pr-1 py-0.5 text-2xs font-mono text-emphasis truncate">
{data.path}
</span>
<!-- Annotation chips share one neutral treatment — the icon carries
the meaning (colors are reserved for feedback, per the brand
guidelines). Only the freshness chip (when it has a verdict)
and the run-state chip below use semantic colors. -->
{#if data.partition_kind}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// partitioned ${data.partition_kind}`}
>
<Layers size={10} />
<span class="text-3xs leading-none">{data.partition_kind}</span>
</div>
{/if}
<!-- Freshness is the one annotation chip that carries feedback (a
fresh/stale verdict against real run history), so like the
run-state chip it uses semantic colors: emerald = within window,
amber = stale. Neutral when there's no verdict (drafts, bad
window value). -->
{#if data.freshness}
<div
class={twMerge(
'shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm',
freshnessState === 'fresh'
? 'bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300'
: freshnessState === 'stale'
? 'bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300'
: 'bg-surface-secondary text-secondary'
)}
title={freshnessTooltip}
>
<Timer size={10} />
<span class="text-3xs leading-none">{data.freshness}</span>
</div>
{/if}
{#if data.tag}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// tag ${data.tag}`}
>
<Tag size={10} />
<span class="text-3xs leading-none">{data.tag}</span>
</div>
{/if}
{#if data.retry}
{@const r = data.retry}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// retry ${r.count}${r.delay ? ` ${r.delay}` : ''}`}
>
<RotateCw size={10} />
<span class="text-3xs leading-none">×{r.count}</span>
</div>
{/if}
{#if data.macros && data.macros.length > 0}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// macros — defines ${data.macros.length} macro${data.macros.length > 1 ? 's' : ''}:\n${data.macros
.map((m) => `• ${m.name}(${m.params})${m.is_table ? ' → table' : ''}`)
.join('\n')}`}
>
<SquareFunction size={10} />
<span class="text-3xs leading-none">×{data.macros.length}</span>
</div>
{/if}
{#if data.dbt}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300"
title={`dbt project — materializes ${data.dbt.model_count} model${data.dbt.model_count === 1 ? '' : 's'}`}
>
<DbtIcon width={10} height={10} />
<span class="text-3xs leading-none">×{data.dbt.model_count}</span>
</div>
{/if}
{#if data.runState}
{@const rs = data.runState}
<div
class={twMerge(
'shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm',
rs.status === 'running'
? 'bg-blue-100 dark:bg-blue-900/40 text-blue-700 dark:text-blue-300'
: rs.status === 'success'
? 'bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300'
: 'bg-red-100 dark:bg-red-900/40 text-red-700 dark:text-red-300'
)}
title={`${
rs.status === 'running'
? 'Running now'
: rs.status === 'success'
? 'Last run succeeded'
: 'Last run failed'
}${rs.runs > 0 ? ` — ran ${rs.runs}× this session` : ''}`}
>
{#if rs.status === 'running'}
<Loader2 size={10} class="animate-spin" />
{:else if rs.status === 'success'}
<CheckCircle2 size={10} />
<NodeActionsMenu items={menuItems} {hover}>
<PipelineNodeCard
surface="primary"
kindLabel="{kindLabel}{data.unsaved ? ' · draft' : ''}"
title={data.summary || data.path}
tooltip={data.error ? `${data.path}: ${data.error}` : nodeTooltip}
{selected}
draft={data.unsaved}
tone={computingNow ? 'running' : data.error ? 'error' : undefined}
>
{#snippet icon()}
{#if data.language && data.runnable_kind === 'script'}
<LanguageIcon lang={data.language} width={14} height={14} />
{:else}
<XCircle size={10} />
<Icon size={14} class={selected ? 'text-accent' : 'text-secondary'} />
{/if}
{#if rs.runs > 0}
<span class="text-3xs leading-none tabular-nums">×{rs.runs}</span>
{/snippet}
{#snippet trailing()}
<!-- Annotation chips share one neutral treatment — the icon carries
the meaning (colors are reserved for feedback, per the brand
guidelines). Only the freshness chip (when it has a verdict)
and the run-state chip below use semantic colors. -->
{#if data.partition_kind}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// partitioned ${data.partition_kind}`}
>
<Layers size={10} />
<span class="text-3xs leading-none">{data.partition_kind}</span>
</div>
{/if}
</div>
{/if}
</div>
<!-- Freshness is the one annotation chip that carries feedback (a
fresh/stale verdict against real run history), so like the
run-state chip it uses semantic colors: emerald = within window,
amber = stale. Neutral when there's no verdict (drafts, bad
window value). -->
{#if data.freshness}
<div
class={twMerge(
'shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm',
freshnessState === 'fresh'
? 'bg-emerald-100 dark:bg-emerald-900/40 text-emerald-700 dark:text-emerald-300'
: freshnessState === 'stale'
? 'bg-amber-100 dark:bg-amber-900/40 text-amber-700 dark:text-amber-300'
: 'bg-surface-secondary text-secondary'
)}
title={freshnessTooltip}
>
<Timer size={10} />
<span class="text-3xs leading-none">{data.freshness}</span>
</div>
{/if}
{#if data.tag}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// tag ${data.tag}`}
>
<Tag size={10} />
<span class="text-3xs leading-none">{data.tag}</span>
</div>
{/if}
{#if data.retry}
{@const r = data.retry}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// retry ${r.count}${r.delay ? ` ${r.delay}` : ''}`}
>
<RotateCw size={10} />
<span class="text-3xs leading-none">×{r.count}</span>
</div>
{/if}
{#if data.macros && data.macros.length > 0}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-surface-secondary text-secondary"
title={`// macros — defines ${data.macros.length} macro${data.macros.length > 1 ? 's' : ''}:\n${data.macros
.map((m) => `• ${m.name}(${m.params})${m.is_table ? ' → table' : ''}`)
.join('\n')}`}
>
<SquareFunction size={10} />
<span class="text-3xs leading-none">×{data.macros.length}</span>
</div>
{/if}
{#if data.dbt}
<div
class="shrink-0 flex items-center gap-0.5 px-1 py-0.5 mr-1 rounded-sm bg-orange-100 dark:bg-orange-900/40 text-orange-700 dark:text-orange-300"
title={`dbt project — materializes ${data.dbt.model_count} model${data.dbt.model_count === 1 ? '' : 's'}`}
>
<DbtIcon width={10} height={10} />
<span class="text-3xs leading-none">×{data.dbt.model_count}</span>
</div>
{/if}
{#if data.runState}
<RunStateChip runState={data.runState} class="mr-1" />
{/if}
{/snippet}
</PipelineNodeCard>
</NodeActionsMenu>
{#if showRun}
<!-- Run button revealed on hover/select. Matches the placement, size,
and behaviour of AssetNode's run button so both nodes feel
@@ -463,38 +447,12 @@
{/if}
</div>
{/if}
{#if menuItems.length > 0}
<!--
Hover-revealed action menu, top-right of the node. Mirrors the
FlowModuleSchemaItem pattern: position the trigger button just
outside the node frame, only render it on hover (or while the
menu is open) so the canvas isn't visually cluttered when idle.
The pointerdown stop+preventDefault keeps xyflow from selecting
the node when the user is reaching for the menu.
-->
<div class="absolute -top-2 -right-2 h-7 p-1 min-w-7" style="will-change: transform;">
<DropdownV2
items={menuItems}
placement="bottom-end"
bind:open={menuOpen}
fixedHeight={false}
usePointerDownOutside
>
{#snippet buttonReplacement()}
<button
class={twMerge(
'center-center p-1 text-secondary shadow-sm bg-surface duration-0 hover:bg-surface-tertiary',
hover || menuOpen ? 'block' : '!hidden',
'shadow-md rounded-md'
)}
onpointerdown={stopPropagation(preventDefault(() => {}))}
title="Actions"
>
<EllipsisVertical size={12} />
</button>
{/snippet}
</DropdownV2>
{#if data.onStartOnDrag}
<NodeOnDot onStart={data.onStartOnDrag} visible={hover} />
{/if}
{#if data.fix}
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<NodeFixButton fix={data.fix} />
</div>
{/if}
</div>
frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte
+127 -225
View File
@@ -24,46 +24,36 @@
type Presentation = {
icon: ComponentType
label: string
// Tailwind class fragments for bg + border + text accent.
bg: string
border: string
borderUnsaved: string
iconText: string
}
// One muted treatment for every source kind — colors are meaningful, not
// One muted icon color for every source kind — colors are meaningful, not
// decorative (brand guidelines), so the kind is carried by the icon and
// label while bg/border stay on the surface/gray scale like the flow
// editor's nodes. Red stays reserved for the missing-trigger state.
const MUTED = {
bg: 'bg-surface-secondary',
border: 'outline-gray-400 dark:outline-gray-600',
borderUnsaved: 'outline-dashed outline-gray-400 dark:outline-gray-500',
iconText: 'text-secondary'
}
// label. Red stays reserved for the missing-trigger state.
const MUTED = { iconText: 'text-secondary' }
export const TRIGGER_NODE_STYLE: Record<TriggerNodeKind, Presentation> = {
schedule: { icon: Clock, label: 'schedule', ...MUTED },
webhook: { icon: Webhook, label: 'webhook', ...MUTED },
email: { icon: Mail, label: 'email', ...MUTED },
kafka: { icon: Zap, label: 'kafka', ...MUTED },
mqtt: { icon: Radio, label: 'mqtt', ...MUTED },
amqp: { icon: Radio, label: 'amqp', ...MUTED },
nats: { icon: MessageSquare, label: 'nats', ...MUTED },
postgres: { icon: Database, label: 'postgres', ...MUTED },
sqs: { icon: Send, label: 'sqs', ...MUTED },
gcp: { icon: CloudCog, label: 'gcp', ...MUTED },
data_upload: { icon: Upload, label: 'data upload', ...MUTED }
schedule: { icon: Clock, label: 'Schedule', ...MUTED },
webhook: { icon: Webhook, label: 'Webhook', ...MUTED },
email: { icon: Mail, label: 'Email', ...MUTED },
kafka: { icon: Zap, label: 'Kafka', ...MUTED },
mqtt: { icon: Radio, label: 'MQTT', ...MUTED },
amqp: { icon: Radio, label: 'AMQP', ...MUTED },
nats: { icon: MessageSquare, label: 'NATS', ...MUTED },
postgres: { icon: Database, label: 'Postgres', ...MUTED },
sqs: { icon: Send, label: 'SQS', ...MUTED },
gcp: { icon: CloudCog, label: 'GCP Pub/Sub', ...MUTED },
data_upload: { icon: Upload, label: 'Data upload', ...MUTED }
}
</script>
<script lang="ts">
import { Handle, Position } from '@xyflow/svelte'
import { NODE } from '$lib/components/graph/util'
import { twMerge } from 'tailwind-merge'
import { AlertTriangle, CheckCircle2, EllipsisVertical, Target, Trash2 } from 'lucide-svelte'
import DropdownV2 from '$lib/components/DropdownV2.svelte'
import { stopPropagation, preventDefault } from 'svelte/legacy'
import PipelineNodeCard from './PipelineNodeCard.svelte'
import NodeActionsMenu from './NodeActionsMenu.svelte'
import NodeFixButton, { type NodeFix } from './NodeFixButton.svelte'
import { describeSchedule } from '$lib/utils/describeCron'
import { AlertTriangle, CheckCircle2, Target, Trash2 } from 'lucide-svelte'
import type { Item } from '$lib/utils'
interface Props {
@@ -74,16 +64,22 @@
// owning script is in `runnable_path` (used by the title and the
// "+ Create trigger" drawer hook passed by the page).
data: {
/** What is wrong with the node, and how to fix it: shows the Fix pill. */
fix?: NodeFix
kind: TriggerNodeKind
ref: string
unsaved?: boolean
missing?: boolean
// A pipeline-local schedule draft: `ref` is the path it deploys to.
// Edit/delete go through the same callbacks, which the page routes
// to the draft instead of a trigger row.
draft?: boolean
// The cron, for a schedule: the node names its cadence instead of "Schedule".
schedule?: string
timezone?: string
// Shown as the title over the trigger's path, when the trigger has one.
summary?: string
runnable_path?: string
// True iff the target script is still a draft (no DB row yet).
// Drives the "(after draft save)" hint on the missing
// placeholder, since the page-level handler refuses to open the
// create drawer until the script is deployed.
runnable_unsaved?: boolean
// Page-supplied dispatcher that opens the matching native
// trigger drawer with `script_path` pre-filled. When absent
// (e.g. schedule, or a kind without an editor) the placeholder is
@@ -128,7 +124,6 @@
let { data }: Props = $props()
let hover = $state(false)
let menuOpen = $state(false)
let style = $derived(TRIGGER_NODE_STYLE[data.kind])
// Webhooks are never genuinely "missing" — every deployed runnable has an
@@ -143,7 +138,7 @@
let Icon = $derived(displayMissing ? AlertTriangle : style.icon)
let missingTitle = $derived(
displayMissing
? `Missing ${style.label} trigger: ${data.runnable_path ?? ''} declares \`// on ${style.label}\` but no ${style.label} trigger targets it. Click to create one, or remove the annotation.`
? `Missing ${style.label} trigger: ${data.runnable_path ?? ''} declares \`// on ${data.kind}\` but no ${style.label} trigger targets it. Click to create one, or remove the annotation.`
: undefined
)
// Webhook gets a drawer (URLs + webhook-specific token creation) rather
@@ -200,7 +195,7 @@
...(canDelete
? [
{
displayName: 'Delete…',
displayName: data.draft ? 'Discard draft' : 'Delete…',
icon: Trash2,
type: 'delete' as const,
action: () => {
@@ -212,6 +207,75 @@
: [])
])
// A schedule names its cadence ("Every day at 4:00", or the cron itself
// when it has no plain reading); every other kind names its kind.
let kindName = $derived(
data.kind === 'schedule' && data.schedule
? (describeSchedule(data.schedule, data.timezone) ?? `Schedule (${data.schedule})`)
: style.label
)
// One card for every state: which label, title and click the node gets.
let card = $derived.by(
(): {
kindLabel: string
title: string
tooltip?: string
draft: boolean
tone?: 'danger' | 'success'
onclick?: () => void
} => {
const unsaved = !!data.unsaved
if (canCreate)
return {
kindLabel: `${style.label} · missing`,
title: 'Click to create',
tooltip: missingTitle,
draft: false,
tone: 'danger',
onclick: handleMissingClick
}
if (canEdit)
return {
kindLabel: `${kindName}${data.draft ? ' · draft' : unsaved ? ' · unsaved' : ''}`,
title: data.summary || data.ref,
tooltip: data.draft
? `Draft ${style.label}: ${data.ref} — created when you save the pipeline. Click to edit.`
: `Edit ${style.label} trigger: ${data.ref}`,
draft: unsaved,
onclick: handleEditClick
}
if (canOpenWebhook)
return {
kindLabel: style.label,
title: 'URLs & token',
tooltip: `Webhook endpoint for ${data.runnable_path ?? ''} — click to view URLs and create a token`,
draft: unsaved,
onclick: handleWebhookClick
}
if (canOpenDataUpload)
return {
kindLabel: `${style.label}${dataUploadReady ? ' · ready' : ''}`,
title: dataUploadReady ? 'File staged' : 'Upload & run',
tooltip: dataUploadReady
? `Data upload for ${data.runnable_path ?? ''} — a file is staged; the pipeline is ready to run. Click to change it.`
: `Data upload for ${data.runnable_path ?? ''} — click to open the run form and upload a file`,
draft: !dataUploadReady && unsaved,
tone: dataUploadReady ? 'success' : undefined,
onclick: handleDataUploadClick
}
return {
kindLabel: `${displayMissing ? style.label : kindName}${displayMissing ? ' · missing' : unsaved ? ' · unsaved' : ''}`,
title: displayMissing ? 'no trigger row' : data.summary || data.ref,
tooltip:
missingTitle ??
(unsaved ? `Unsaved ${style.label}: ${data.ref}` : `${style.label}: ${data.ref}`),
draft: unsaved && !displayMissing,
tone: displayMissing ? 'danger' : undefined
}
}
)
function handleMissingClick() {
if (!canCreate || !data.runnable_path || !data.onCreateMissingTrigger) return
data.onCreateMissingTrigger(data.kind as NativeTriggerKind, data.runnable_path)
@@ -235,195 +299,33 @@
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div class="relative" onmouseenter={() => (hover = true)} onmouseleave={() => (hover = false)}>
{#if canCreate}
<!-- Same affordance as the asset's downstream + button: render the
whole node as a button so the cursor + hover + click is obvious
without depending on svelte-flow's wrapper-level handlers. -->
<button
type="button"
onclick={handleMissingClick}
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1 w-full text-left',
'bg-red-50 dark:bg-red-900/30 outline-dashed outline-red-400 dark:outline-red-500',
'hover:bg-red-100 dark:hover:bg-red-900/40 transition-colors'
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={missingTitle}
<NodeActionsMenu items={menuItems} {hover}>
<PipelineNodeCard
kindLabel={card.kindLabel}
title={card.title}
tooltip={card.tooltip}
draft={card.draft}
tone={card.tone}
onclick={card.onclick}
>
<Icon size={14} class="shrink-0 ml-2 mr-2 text-red-600 dark:text-red-400" />
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
<span class="text-3xs uppercase tracking-wide truncate text-red-700 dark:text-red-400">
{style.label} · missing
</span>
<span class="text-2xs font-mono truncate text-red-700 dark:text-red-400">
{data.runnable_unsaved ? 'Click to create (after draft save)' : 'Click to create'}
</span>
</div>
</button>
{:else if canEdit}
<!-- Mirrors the missing-trigger pattern: render the whole node as a
button so clicks open the editor drawer reliably (don't rely on
svelte-flow's onnodeclick wiring). -->
<button
type="button"
onclick={handleEditClick}
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1 w-full text-left',
style.bg,
data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border,
'hover:brightness-95 dark:hover:brightness-110 transition-[filter]'
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={`Edit ${style.label} trigger: ${data.ref}`}
>
<Icon size={14} class={`shrink-0 ml-2 mr-2 ${style.iconText}`} />
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
<span class="text-3xs uppercase tracking-wide truncate text-tertiary">
{style.label}{data.unsaved ? ' · unsaved' : ''}
</span>
<span class="text-2xs font-mono truncate text-emphasis">
{data.ref}
</span>
</div>
</button>
{:else if canOpenWebhook}
<!-- Webhook endpoint: implicit (no trigger row), so the node opens a
drawer with the URLs + webhook-specific token creation instead of
an editor. Styled like an attached trigger, never the red
"missing" state. -->
<button
type="button"
onclick={handleWebhookClick}
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1 w-full text-left',
style.bg,
data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border,
'hover:brightness-95 dark:hover:brightness-110 transition-[filter]'
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={`Webhook endpoint for ${data.runnable_path ?? ''} — click to view URLs and create a token`}
>
<Icon size={14} class={`shrink-0 ml-2 mr-2 ${style.iconText}`} />
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
<span class="text-3xs uppercase tracking-wide truncate text-tertiary">
{style.label}
</span>
<span class="text-2xs font-mono truncate text-emphasis"> URLs & token </span>
</div>
</button>
{:else if canOpenDataUpload}
<!-- Data upload: UI-first entry point (no trigger row). Clicking the
node opens the target script's run form, where the auto-generated
S3 picker lets the user upload a file and run the pipeline. Goes
green once a file is staged (ready), so "Run pipeline" can proceed;
until then it stays neutral with an "upload a file" prompt. Never
the red "missing" state. -->
<button
type="button"
onclick={handleDataUploadClick}
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1 w-full text-left',
dataUploadReady
? 'bg-green-50 dark:bg-green-900/30 outline-green-500 dark:outline-green-600'
: style.bg,
dataUploadReady ? '' : data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border,
'hover:brightness-95 dark:hover:brightness-110 transition-[filter]'
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={dataUploadReady
? `Data upload for ${data.runnable_path ?? ''} — a file is staged; the pipeline is ready to run. Click to change it.`
: `Data upload for ${data.runnable_path ?? ''} — click to open the run form and upload a file`}
>
<DataUploadIcon
size={14}
class={`shrink-0 ml-2 mr-2 ${dataUploadReady ? 'text-green-600 dark:text-green-400' : style.iconText}`}
/>
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
<span
class={twMerge(
'text-3xs uppercase tracking-wide truncate',
dataUploadReady ? 'text-green-700 dark:text-green-400' : 'text-tertiary'
)}
>
{style.label}{dataUploadReady ? ' · ready' : ''}
</span>
<span
class={twMerge(
'text-2xs font-mono truncate',
dataUploadReady ? 'text-green-700 dark:text-green-400' : 'text-emphasis'
)}
>
{dataUploadReady ? 'File staged' : 'Upload & run'}
</span>
</div>
</button>
{:else}
<div
class={twMerge(
'flex items-center rounded-md drop-shadow-sm overflow-hidden outline outline-1',
displayMissing
? 'bg-red-50 dark:bg-red-900/30 outline-dashed outline-red-400 dark:outline-red-500'
: style.bg,
displayMissing ? '' : data.unsaved ? `opacity-80 ${style.borderUnsaved}` : style.border
)}
style="width: {NODE.width}px; min-height: {NODE.height}px;"
title={missingTitle ??
(data.unsaved ? `Unsaved ${style.label}: ${data.ref}` : `${style.label}: ${data.ref}`)}
>
<Icon
size={14}
class={`shrink-0 ml-2 mr-2 ${displayMissing ? 'text-red-600 dark:text-red-400' : style.iconText}`}
/>
<div class="flex flex-col min-w-0 flex-1 pr-2 py-0.5 leading-tight">
<span
class={twMerge(
'text-3xs uppercase tracking-wide truncate',
displayMissing ? 'text-red-700 dark:text-red-400' : 'text-tertiary'
)}
>
{style.label}{displayMissing ? ' · missing' : data.unsaved ? ' · unsaved' : ''}
</span>
<span
class={twMerge(
'text-2xs font-mono truncate',
displayMissing ? 'text-red-700 dark:text-red-400' : 'text-emphasis'
)}
>
{displayMissing ? 'no trigger row' : data.ref}
</span>
</div>
</div>
{/if}
{#if menuItems.length > 0}
<!-- Hover-revealed kebab menu (Delete only for now). Mirrors the
RunnableNode pattern: positioned just outside the top-right of
the node, rendered only on hover or while the menu is open so
the canvas stays clean at rest. `pointerdown` is stopped so
svelte-flow doesn't kick off node selection / drag when the
user reaches for the menu. -->
<div class="absolute -top-2 -right-2 h-7 p-1 min-w-7" style="will-change: transform;">
<DropdownV2
items={menuItems}
placement="bottom-end"
bind:open={menuOpen}
fixedHeight={false}
usePointerDownOutside
>
{#snippet buttonReplacement()}
<button
class={twMerge(
'center-center p-1 text-secondary shadow-sm bg-surface duration-0 hover:bg-surface-tertiary',
hover || menuOpen ? 'block' : '!hidden',
'shadow-md rounded-md'
)}
onpointerdown={stopPropagation(preventDefault(() => {}))}
title="Actions"
>
<EllipsisVertical size={12} />
</button>
{/snippet}
</DropdownV2>
{#snippet icon()}
{#if canOpenDataUpload}
<DataUploadIcon
size={14}
class={dataUploadReady ? 'text-green-600 dark:text-green-400' : style.iconText}
/>
{:else}
<Icon
size={14}
class={displayMissing || canCreate ? 'text-red-600 dark:text-red-400' : style.iconText}
/>
{/if}
{/snippet}
</PipelineNodeCard>
</NodeActionsMenu>
{#if data.fix}
<div class="absolute left-1/2 -bottom-3 -translate-x-1/2 z-10">
<NodeFixButton fix={data.fix} />
</div>
{/if}
</div>
frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts
+14
View File
@@ -147,4 +147,18 @@ describe('layoutAssetGraph (tidy-tree with join breaks)', () => {
expect(pos.get(id)!.y).toBeGreaterThan(anchor.y)
}
})
it('packs narrow nodes by their own width', () => {
const pos = layoutAssetGraph({
nodes: [
{ ...n('p'), width: 100 },
{ ...n('l'), width: 100 },
{ ...n('r'), width: 180 }
],
edges: [e('p', 'l'), e('p', 'r')]
})
const gap = pos.get('r')!.x - 90 - (pos.get('l')!.x + 50)
expect(gap).toBe(NODE.gap.horizontal)
expect(pos.get('l')!.x).toBe(50)
})
})
frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts
+47 -21
View File
@@ -7,7 +7,15 @@ import { NODE } from '$lib/components/graph/util'
// breathing room.
const NODE_WIDTH = NODE.width
const NODE_HEIGHT = NODE.height + 30
const LAYER_GAP = NODE.gap.vertical
/** Rendered height of a pipeline node: a kind label over a title. Fits inside
* the `NODE_HEIGHT` row the layout reserves. */
export const PIPELINE_NODE_HEIGHT = 48
// A step more than the flow editor's gap: the + on a node's bottom edge sits in it.
const LAYER_GAP = NODE.gap.vertical + 16
/** Height of the "Add data source" pill the anchor draws, and its gap to the
* graph: tighter than LAYER_GAP, since no edge runs through it. */
const ANCHOR_HEIGHT = 32
const ANCHOR_GAP = 40
const SIBLING_GAP = NODE.gap.horizontal
// Horizontal gutter between two disjoint subgraphs. Wider than the
// within-subgraph sibling gap so visually-unrelated components (e.g. two
@@ -15,10 +23,16 @@ const SIBLING_GAP = NODE.gap.horizontal
// into each other's columns — but only ~2×, so they don't drift far apart.
const COMPONENT_GAP = SIBLING_GAP * 2
const LAYER_H = NODE_HEIGHT + LAYER_GAP
/** Extra height of an assets-only node: the trigger chip under its title. */
export const PIPELINE_NODE_EXTRA_ROW = 20
/** Height of the header strip naming an assets-only node's producing script. */
export const PIPELINE_NODE_HEADER = 22
/** Extra vertical padding of a card under a header, top plus bottom. */
export const PIPELINE_NODE_HEADER_CARD_PAD = 12
interface GraphInput {
nodes: Array<{ id: string; data: AssetGraphNodeData }>
/** `width` defaults to `NODE.width`. */
nodes: Array<{ id: string; data: AssetGraphNodeData; width?: number }>
edges: Array<{ source: string; target: string }>
}
@@ -52,15 +66,18 @@ interface Band {
// intersects theirs.
//
// y comes from longest-path layering (same top-down orientation as before:
// producers above, assets in the middle, consumers below). Returns positions
// (band centers) normalized so the component's min x,y = 0. Cyclic input is
// handled by dropping feedback edges (see the Kahn step below).
// producers above, assets in the middle, consumers below). Returns node
// centers, normalized so the component's leftmost edge and top are at 0.
// Cyclic input is handled by dropping feedback edges (see the Kahn step below).
function layoutComponent(
nodes: GraphInput['nodes'],
edges: GraphInput['edges']
edges: GraphInput['edges'],
rowH: number
): Map<string, Positioned> {
const layerH = rowH + LAYER_GAP
const out = new Map<string, Positioned>()
const ids = new Set(nodes.map((n) => n.id))
const nodeW = new Map(nodes.map((n) => [n.id, n.width ?? NODE_WIDTH]))
const parents = new Map<string, string[]>()
const children = new Map<string, string[]>()
@@ -137,7 +154,7 @@ function layoutComponent(
const kids = treeChildren.get(id)!
const kidsW =
kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * Math.max(0, kids.length - 1)
W.set(id, Math.max(NODE_WIDTH, kidsW))
W.set(id, Math.max(nodeW.get(id)!, kidsW))
}
// Vertical (pixel) extent of a subtree, for band collision checks.
@@ -152,14 +169,14 @@ function layoutComponent(
if (l > hi) hi = l
for (const k of treeChildren.get(cur)!) stack.push(k)
}
return { top: lo * LAYER_H, bottom: hi * LAYER_H + NODE_HEIGHT }
return { top: lo * layerH, bottom: hi * layerH + rowH }
}
// Recursive placement: node centered over its band, children packed
// side-by-side and centered within it.
function placeTree(id: string, left: number) {
const w = W.get(id)!
out.set(id, { x: left + w / 2, y: layer.get(id)! * LAYER_H })
out.set(id, { x: left + w / 2, y: layer.get(id)! * layerH })
const kids = treeChildren.get(id)!
if (kids.length === 0) return
const kidsW = kids.reduce((acc, k) => acc + W.get(k)!, 0) + SIBLING_GAP * (kids.length - 1)
@@ -211,11 +228,11 @@ function layoutComponent(
placeAndRecord(id, center - w / 2)
}
// Normalize so the component's min x,y = 0.
// Normalize so the component's leftmost edge and top are at 0.
let minX = Infinity
let minY = Infinity
for (const p of out.values()) {
if (p.x < minX) minX = p.x
for (const [id, p] of out) {
if (p.x - nodeW.get(id)! / 2 < minX) minX = p.x - nodeW.get(id)! / 2
if (p.y < minY) minY = p.y
}
if (isFinite(minX) && isFinite(minY)) {
@@ -243,7 +260,13 @@ function layoutComponent(
//
// Falls back to a stable grid if the component layout throws (defensive —
// cycles are already absorbed by feedback-edge dropping in layoutComponent).
export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map<string, Positioned> {
export function layoutAssetGraph(
graph: GraphInput,
anchorId?: string,
extraRowHeight = 0
): Map<string, Positioned> {
const rowH = NODE_HEIGHT + extraRowHeight
const layerH = rowH + LAYER_GAP
const byId = new Map<string, Positioned>()
if (graph.nodes.length === 0) return byId
@@ -297,19 +320,20 @@ export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map<stri
// 3. Lay out each component and pack side-by-side. xOffset advances by
// the laid-out width of each component (max node center + a node width,
// since positions are node centers) plus the gutter.
const widthOf = new Map(nodes.map((n) => [n.id, n.width ?? NODE_WIDTH]))
let xOffset = 0
for (let c = 0; c < nComp; c++) {
const positions = layoutComponent(compNodes[c], compEdges[c])
let maxX = 0
for (const p of positions.values()) if (p.x > maxX) maxX = p.x
const positions = layoutComponent(compNodes[c], compEdges[c], rowH)
let maxRight = 0
for (const [id, p] of positions) {
maxRight = Math.max(maxRight, p.x + widthOf.get(id)! / 2)
byId.set(id, { x: p.x + xOffset, y: p.y })
}
xOffset += maxX + NODE_WIDTH + COMPONENT_GAP
xOffset += maxRight + COMPONENT_GAP
}
// 4. Re-place the anchor centered horizontally over the whole packed
// graph, one layer above it; then renormalize so the anchor sits at the
// graph, just above it; then renormalize so the anchor sits at the
// top (y = 0), pushing the components down a layer to make room (mirrors
// the previous "anchor is the parent of every root" behaviour).
if (anchorId && graph.nodes.some((n) => n.id === anchorId)) {
@@ -324,7 +348,9 @@ export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map<stri
if (p.x > maxX) maxX = p.x
if (p.y < minY) minY = p.y
}
byId.set(anchorId, { x: (minX + maxX) / 2, y: minY - LAYER_H })
// One gap above the graph, sized to the pill rather than a full node row:
// a row tall enough for an assets-only card leaves it floating high.
byId.set(anchorId, { x: (minX + maxX) / 2, y: minY - ANCHOR_HEIGHT - ANCHOR_GAP })
let nMinY = Infinity
for (const p of byId.values()) if (p.y < nMinY) nMinY = p.y
for (const p of byId.values()) p.y -= nMinY
@@ -336,7 +362,7 @@ export function layoutAssetGraph(graph: GraphInput, anchorId?: string): Map<stri
graph.nodes.forEach((n, i) => {
byId.set(n.id, {
x: (i % cols) * (NODE_WIDTH + SIBLING_GAP),
y: Math.floor(i / cols) * LAYER_H
y: Math.floor(i / cols) * layerH
})
})
return byId
frontend/src/lib/components/assets/AssetGraph/assetNodeWidth.ts
+66
View File
@@ -0,0 +1,66 @@
import { NODE } from '$lib/components/graph/util'
const MIN_WIDTH = 160
let ctx: CanvasRenderingContext2D | null | undefined
/** Rendered width of `text` at `rem` in the page font, or a rough guess without a DOM. */
function textWidth(text: string, rem: number): number {
if (typeof document === 'undefined') return text.length * rem * 16 * 0.6
ctx ??= document.createElement('canvas').getContext('2d')
const root = parseFloat(getComputedStyle(document.documentElement).fontSize) || 16
if (!ctx) return text.length * rem * root * 0.6
const body = getComputedStyle(document.body)
ctx.font = `${rem * root}px ${body.fontFamily}`
const spacing = parseFloat(body.letterSpacing) || 0
return ctx.measureText(text).width + spacing * text.length
}
/**
* Width of an assets-only asset card, from the pieces `PipelineNodeCard` and
* `AssetNode` lay out: icon and the widest of kind / title / trigger chip, or the
* header naming the producing script, if wider. The numbers mirror their Tailwind spacing; change one
* and the other drifts. Clamped to the regular node width, past which the title
* truncates as usual.
*/
export function assetsOnlyNodeWidth(n: {
kind: string
title: string
chip: string
/** The chip leads with an icon (the folded-errors Fix chip). */
chipIcon?: boolean
/** The chip ends in the edit icon (an editable trigger). */
chipEdit?: boolean
/** The header tab: its text (a script's summary or path, "N scripts"), icon
* count, and whether it ends in the menu chevron (else the edit icon) or has a "draft" mark;
* undefined without a header. */
header?: { label: string; icons: number; chevron?: boolean; draft?: boolean }
}): number {
const icon = 12 + 14 + 10
const text = Math.max(
textWidth(n.kind, 0.7),
textWidth(n.title, 0.75),
textWidth(n.chip, 0.65) + 12 + 2 + (n.chipIcon ? 14 : 0) + (n.chipEdit ? 4 + 9 : 0)
)
const body = Math.ceil(icon + text + 4 + 8 + 2 + 8)
// Header tab: mx-2 and a 1px border around px-2, then gap-1.5 between its
// items: the icons, the label, a "draft" mark, and the chevron or edit icon.
// The run chip is left out: it comes and goes with runs, and a width that
// followed it would re-lay the graph on a click; the label truncates instead.
const h = n.header
const header = h
? Math.ceil(
16 +
2 +
16 +
h.icons * (11 + 6) +
textWidth(h.label, 0.65) +
(h.draft ? 6 + textWidth('draft', 0.65) : 0) +
// The menu chevron, or the edit icon of a single script.
6 +
(h.chevron ? 11 : 10) +
4
)
: 0
return Math.min(NODE.width, Math.max(MIN_WIDTH, body, header))
}
frontend/src/lib/components/assets/AssetGraph/assetsOnlyFolders.ts
+25
View File
@@ -0,0 +1,25 @@
// Pipelines open in the assets-only view. This remembers the ones last left
// showing their scripts and triggers, as `<workspace>/<folder>`, so reopening one
// shows it the same way. Browser-local: a missing or unreadable list reads as
// "none", and storage that throws (private mode, blocked site data) is ignored.
const KEY = 'pipeline-full-view-folders'
function read(): string[] {
try {
const parsed = JSON.parse(localStorage.getItem(KEY) ?? '[]')
return Array.isArray(parsed) ? parsed.filter((f) => typeof f === 'string') : []
} catch {
return []
}
}
export function isAssetsOnlyFolder(folder: string): boolean {
return !read().includes(folder)
}
export function setAssetsOnlyFolder(folder: string, on: boolean): void {
const rest = read().filter((f) => f !== folder)
try {
localStorage.setItem(KEY, JSON.stringify(on ? rest : [...rest, folder]))
} catch {}
}

Some files were not shown because too many files have changed in this diff Show More