starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-25 00:01:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into di/flow-input-redesign

Browse Source
This commit is contained in:
Diego Imbert
2025-09-27 13:30:18 +02:00
parent 496af49a9f 15731e7726
commit 6358bf3e44
120 changed files with 2245 additions and 1000 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+35
View File
@@ -1,5 +1,40 @@
# Changelog
## [1.549.1](https://github.com/windmill-labs/windmill/compare/v1.549.0...v1.549.1) (2025-09-26)
### Bug Fixes
* do not request unecessarily get_scheduled_for ([0269211](https://github.com/windmill-labs/windmill/commit/02692111a1a8eefb2675b14d53f109a66c1b9a78))
* fix agent_workers completed job back-compatibility deserialization ([db4bc7e](https://github.com/windmill-labs/windmill/commit/db4bc7ee6963955abc7e290bd67ea913b0f5e2ad))
## [1.549.0](https://github.com/windmill-labs/windmill/compare/v1.548.3...v1.549.0) (2025-09-26)
### Features
* **backend:** job result stream optimization ([#6673](https://github.com/windmill-labs/windmill/issues/6673)) ([8f4fef9](https://github.com/windmill-labs/windmill/commit/8f4fef98042c49346c89bdf5e0b9b1f2d52e371f))
### Bug Fixes
* scim group handling when deleting instance user + conversion ([#6677](https://github.com/windmill-labs/windmill/issues/6677)) ([4205e83](https://github.com/windmill-labs/windmill/commit/4205e83cfde453827eab23c31e76a0f0490d31b7))
## [1.548.3](https://github.com/windmill-labs/windmill/compare/v1.548.2...v1.548.3) (2025-09-25)
### Bug Fixes
* fix job loader token initialization ([f5d238e](https://github.com/windmill-labs/windmill/commit/f5d238edcfed6b0f066d459cdc718679a7b51187))
* websocket runnable [#6675](https://github.com/windmill-labs/windmill/issues/6675) ([a308782](https://github.com/windmill-labs/windmill/commit/a308782bcf7ef9913887521d74796b490619d0c8))
## [1.548.2](https://github.com/windmill-labs/windmill/compare/v1.548.1...v1.548.2) (2025-09-24)
### Bug Fixes
* **ui:** workers button on navbar require a single click only ([afa8104](https://github.com/windmill-labs/windmill/commit/afa8104cb0c1a8f1a6fe124a6e01c1d32f049afa))
## [1.548.1](https://github.com/windmill-labs/windmill/compare/v1.548.0...v1.548.1) (2025-09-24)
backend/.gitignore
+2 -1
View File
@@ -9,4 +9,5 @@ windmill-api/openapi-*.*
.duckdb/*
*ee.rs
generate_mcp_endpoints_tools/venv
bacon.toml
bacon.toml
libwindmill_duckdb_ffi_internal.so
backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n eig.igroup as group_name,\n ws.auto_add_instance_groups_roles\n FROM email_to_igroup eig\n INNER JOIN workspace_settings ws ON ws.workspace_id = $1\n WHERE eig.email = $2\n AND eig.igroup = ANY(ws.auto_add_instance_groups)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "group_name",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "auto_add_instance_groups_roles",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true
]
},
"hash": "1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608"
}
backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a"
}
backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d"
}
backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json
Generated
-101
View File
@@ -1,101 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n SUBSTR(rs.stream, $8) AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option<String>\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "completed",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "running",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "new_result_stream",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "stream_job",
"type_info": "Uuid"
},
{
"ordinal": 7,
"name": "workflow_as_code_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "log_offset",
"type_info": "Int4"
},
{
"ordinal": 9,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 10,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "progress",
"type_info": "Int4"
},
{
"ordinal": 12,
"name": "result_stream: Option<String>",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid",
"Bool",
"Bool",
"TextArray",
"Bool",
"Int4"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
false,
null,
false
]
},
"hash": "40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773"
}
backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json
Generated
+49
View File
@@ -0,0 +1,49 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json<Box<RawValue>>\",\n rs.stream AS \"result_stream: Option<String>\",\n rs.offset AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Int4",
"Bool"
]
},
"nullable": [
null,
null,
null,
null,
null
]
},
"hash": "4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d"
}
backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_result_stream (workspace_id, job_id, stream)\n VALUES ($1, $2, $3)\n ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b"
}
backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"UuidArray"
]
},
"nullable": []
},
"hash": "5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148"
}
backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json
Generated
+101
View File
@@ -0,0 +1,101 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $3 AND idx >= $8\n GROUP BY job_id\n )\n SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n rs.stream AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n rs.offset AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option<String>\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "completed",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "running",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "logs",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "new_result_stream",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "mem_peak",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 6,
"name": "stream_job",
"type_info": "Uuid"
},
{
"ordinal": 7,
"name": "workflow_as_code_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "log_offset",
"type_info": "Int4"
},
{
"ordinal": 9,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 10,
"name": "created_by!",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "progress",
"type_info": "Int4"
},
{
"ordinal": 12,
"name": "result_stream: Option<String>",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int4",
"Text",
"Uuid",
"Bool",
"Bool",
"TextArray",
"Bool",
"Int4"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
false,
null,
null
]
},
"hash": "7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882"
}
backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa"
}
backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json
Generated
-49
View File
@@ -1,49 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json<Box<RawValue>>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json<Box<RawValue>>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "flow_status: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Int4",
"Bool"
]
},
"nullable": [
null,
null,
null,
null,
null
]
},
"hash": "80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7"
}
backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json
Generated
-55
View File
@@ -1,55 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT \n jc.result as \"result: sqlx::types::Json<Box<RawValue>>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option<bool>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",\n CHAR_LENGTH(rs.stream) AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "running: Option<bool>",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Int4",
"Bool"
]
},
"nullable": [
true,
false,
false,
null,
null,
null
]
},
"hash": "8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34"
}
backend/.sqlx/query-9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx)\n VALUES (\n $1, \n $2,\n $3, \n $4\n )\n ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid",
"Text",
"Int4"
]
},
"nullable": []
},
"hash": "9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde"
}
backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json
Generated
+55
View File
@@ -0,0 +1,55 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT \n jc.result as \"result: sqlx::types::Json<Box<RawValue>>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option<bool>\",\n rs.stream AS \"result_stream: Option<String>\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "running: Option<bool>",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Int4",
"Bool"
]
},
"nullable": [
true,
false,
false,
null,
null,
null
]
},
"hash": "9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35"
}
backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json → backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o \n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) \n AND workspace_id = $1 and archived = false",
"query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o\n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false)\n AND workspace_id = $1 and archived = false",
"describe": {
"columns": [
{
@@ -42,5 +42,5 @@
true
]
},
"hash": "2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1"
"hash": "9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339"
}
backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json
Generated
-49
View File
@@ -1,49 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json<Box<RawValue>>\",\n jq.running as \"running: Option<bool>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "running: Option<bool>",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Bool"
]
},
"nullable": [
null,
false,
null,
null,
null
]
},
"hash": "a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05"
}
backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json
Generated
+49
View File
@@ -0,0 +1,49 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $1 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json<Box<RawValue>>\",\n jq.running as \"running: Option<bool>\",\n rs.stream AS \"result_stream: Option<String>\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "result: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "running: Option<bool>",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "result_stream: Option<String>",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "stream_offset",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "stream_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Bool"
]
},
"nullable": [
null,
false,
null,
null,
null
]
},
"hash": "a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49"
}
backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM job_result_stream_v2 WHERE job_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090"
}
backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "stream",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "offset",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid"
]
},
"nullable": [
null,
null
]
},
"hash": "dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261"
}
backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json
Generated
+41
View File
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_admin",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "operator",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "added_via",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
true
]
},
"hash": "eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d"
}
backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "new_result_stream",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "stream_offset",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid"
]
},
"nullable": [
null,
null
]
},
"hash": "f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a"
}
backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics",
"describe": {
"columns": [],
"parameters": {
"Left": []
},
"nullable": []
},
"hash": "f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571"
}
backend/Cargo.lock
Generated
+82 -82
View File
@@ -14,9 +14,9 @@ dependencies = [
[[package]]
name = "addr2line"
version = "0.24.2"
version = "0.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
dependencies = [
"gimli",
]
@@ -620,7 +620,7 @@ dependencies = [
"thiserror 1.0.69",
"time",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tokio-util",
"tokio-websockets",
"tracing",
@@ -1245,9 +1245,9 @@ dependencies = [
[[package]]
name = "backtrace"
version = "0.3.75"
version = "0.3.76"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002"
checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
dependencies = [
"addr2line",
"cfg-if",
@@ -1255,7 +1255,7 @@ dependencies = [
"miniz_oxide 0.8.9",
"object",
"rustc-demangle",
"windows-targets 0.52.6",
"windows-link 0.2.0",
]
[[package]]
@@ -1935,9 +1935,9 @@ dependencies = [
[[package]]
name = "cc"
version = "1.2.38"
version = "1.2.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9"
checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -3625,7 +3625,7 @@ dependencies = [
"serde_json",
"thiserror 2.0.16",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tokio-socks",
"tokio-util",
"tower 0.5.2",
@@ -4673,7 +4673,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -5065,7 +5065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -5893,9 +5893,9 @@ dependencies = [
[[package]]
name = "gimli"
version = "0.31.1"
version = "0.32.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
[[package]]
name = "git-version"
@@ -6606,7 +6606,7 @@ dependencies = [
"pin-project-lite",
"rustls-native-certs 0.7.3",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tower-service",
]
@@ -6655,7 +6655,7 @@ dependencies = [
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tower-service",
"webpki-roots 1.0.2",
]
@@ -6764,7 +6764,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.0",
"windows-core 0.62.1",
]
[[package]]
@@ -7888,7 +7888,7 @@ dependencies = [
"rustls-pki-types",
"smtp-proto",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"webpki-roots 0.26.11",
]
@@ -8027,9 +8027,9 @@ dependencies = [
[[package]]
name = "memchr"
version = "2.7.5"
version = "2.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
[[package]]
name = "memmap2"
@@ -8887,9 +8887,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.36.7"
version = "0.37.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
dependencies = [
"memchr",
]
@@ -9794,9 +9794,9 @@ dependencies = [
[[package]]
name = "postgres-protocol"
version = "0.6.8"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54"
checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4"
dependencies = [
"base64 0.22.1",
"byteorder",
@@ -9831,7 +9831,7 @@ dependencies = [
"bytes",
"chrono",
"fallible-iterator 0.2.0",
"postgres-protocol 0.6.8",
"postgres-protocol 0.6.9",
"serde",
"serde_json",
"uuid",
@@ -10569,9 +10569,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.11.2"
version = "1.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912"
checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c"
dependencies = [
"aho-corasick",
"memchr",
@@ -10581,9 +10581,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.10"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6"
checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad"
dependencies = [
"aho-corasick",
"memchr",
@@ -10696,7 +10696,7 @@ dependencies = [
"sync_wrapper 1.0.2",
"tokio",
"tokio-native-tls",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tokio-util",
"tower 0.5.2",
"tower-http",
@@ -11077,7 +11077,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.11.0",
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -11406,7 +11406,7 @@ version = "0.1.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
dependencies = [
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -13296,7 +13296,7 @@ dependencies = [
"getrandom 0.3.3",
"once_cell",
"rustix 1.1.2",
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -13687,7 +13687,7 @@ dependencies = [
"percent-encoding",
"phf 0.11.3",
"pin-project-lite",
"postgres-protocol 0.6.8",
"postgres-protocol 0.6.9",
"postgres-types 0.2.9",
"rand 0.9.0",
"socket2 0.5.10",
@@ -13729,9 +13729,9 @@ dependencies = [
[[package]]
name = "tokio-rustls"
version = "0.26.3"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [
"rustls 0.23.29",
"tokio",
@@ -13823,7 +13823,7 @@ dependencies = [
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tokio-util",
]
@@ -13917,7 +13917,7 @@ dependencies = [
"rustls-pemfile 2.2.0",
"socket2 0.5.10",
"tokio",
"tokio-rustls 0.26.3",
"tokio-rustls 0.26.4",
"tokio-stream",
"tower 0.4.13",
"tower-layer",
@@ -15108,7 +15108,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.0",
"windows-sys 0.61.1",
]
[[package]]
@@ -15119,7 +15119,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"aws-sdk-config",
@@ -15179,7 +15179,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"argon2",
@@ -15299,7 +15299,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -15314,7 +15314,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"chrono",
"serde",
@@ -15327,7 +15327,7 @@ dependencies = [
[[package]]
name = "windmill-autoscaling"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"axum",
@@ -15346,7 +15346,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15430,7 +15430,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"regex",
"serde",
@@ -15445,7 +15445,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"bytes",
@@ -15469,7 +15469,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"itertools 0.14.0",
"lazy_static",
@@ -15481,7 +15481,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -15490,7 +15490,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15502,7 +15502,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"serde_json",
@@ -15514,7 +15514,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"gosyn",
@@ -15526,7 +15526,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15538,7 +15538,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"serde_json",
@@ -15550,7 +15550,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -15561,7 +15561,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15572,7 +15572,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -15584,7 +15584,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15607,7 +15607,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15621,7 +15621,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -15638,7 +15638,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15652,7 +15652,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -15670,7 +15670,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"getrandom 0.2.16",
@@ -15695,7 +15695,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"serde_json",
@@ -15705,7 +15705,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -15738,7 +15738,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -15748,7 +15748,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.548.1"
version = "1.549.1"
dependencies = [
"anyhow",
"async-once-cell",
@@ -15919,8 +15919,8 @@ version = "0.61.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3"
dependencies = [
"windows-implement 0.60.0",
"windows-interface 0.59.1",
"windows-implement 0.60.1",
"windows-interface 0.59.2",
"windows-link 0.1.3",
"windows-result 0.3.4",
"windows-strings 0.4.2",
@@ -15928,12 +15928,12 @@ dependencies = [
[[package]]
name = "windows-core"
version = "0.62.0"
version = "0.62.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c"
checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9"
dependencies = [
"windows-implement 0.60.0",
"windows-interface 0.59.1",
"windows-implement 0.60.1",
"windows-interface 0.59.2",
"windows-link 0.2.0",
"windows-result 0.4.0",
"windows-strings 0.5.0",
@@ -15974,9 +15974,9 @@ dependencies = [
[[package]]
name = "windows-implement"
version = "0.60.0"
version = "0.60.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
checksum = "edb307e42a74fb6de9bf3a02d9712678b22399c87e6fa869d6dfcd8c1b7754e0"
dependencies = [
"proc-macro2",
"quote",
@@ -16007,9 +16007,9 @@ dependencies = [
[[package]]
name = "windows-interface"
version = "0.59.1"
version = "0.59.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
checksum = "c0abd1ddbc6964ac14db11c7213d6532ef34bd9aa042c2e5935f59d7908b46a5"
dependencies = [
"proc-macro2",
"quote",
@@ -16127,14 +16127,14 @@ version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.3",
"windows-targets 0.53.4",
]
[[package]]
name = "windows-sys"
version = "0.61.0"
version = "0.61.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa"
checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f"
dependencies = [
"windows-link 0.2.0",
]
@@ -16172,11 +16172,11 @@ dependencies = [
[[package]]
name = "windows-targets"
version = "0.53.3"
version = "0.53.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b"
dependencies = [
"windows-link 0.1.3",
"windows-link 0.2.0",
"windows_aarch64_gnullvm 0.53.0",
"windows_aarch64_msvc 0.53.0",
"windows_i686_gnu 0.53.0",
backend/Cargo.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.548.1"
version = "1.549.1"
authors.workspace = true
edition.workspace = true
@@ -34,7 +34,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal"]
[workspace.package]
version = "1.548.1"
version = "1.549.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
aa4c7936b99d0de2f7af31cde79dfc7b1b482c29
4011644daea63877e43ebc20cbe6704f381b4b42
backend/migrations/20250924150953_new_job_result_stream_table.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add down migration script here
DROP TABLE job_result_stream_v2;
ALTER TABLE job_result_stream ADD CONSTRAINT fk_job_result_stream_job_id FOREIGN KEY (job_id) REFERENCES v2_job_queue(id) ON DELETE CASCADE;
backend/migrations/20250924150953_new_job_result_stream_table.up.sql
+13
View File
@@ -0,0 +1,13 @@
-- Add up migration script here
CREATE TABLE job_result_stream_v2 (
job_id UUID NOT NULL,
workspace_id TEXT NOT NULL,
stream TEXT NOT NULL,
idx INT NOT NULL,
PRIMARY KEY (job_id, idx)
);
GRANT ALL ON TABLE job_result_stream_v2 TO windmill_admin;
GRANT ALL ON TABLE job_result_stream_v2 TO windmill_user;
ALTER TABLE job_result_stream DROP CONSTRAINT fk_job_result_stream_job_id;
backend/src/monitor.rs
+12 -1
View File
@@ -950,6 +950,17 @@ pub async fn delete_expired_items(db: &DB) -> () {
{
tracing::error!("Error deleting job: {:?}", e);
}
// should already be deleted but just in case
if let Err(e) = sqlx::query!(
"DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)",
&deleted_jobs
)
.execute(&mut *tx)
.await
{
tracing::error!("Error deleting job result stream: {:?}", e);
}
}
}
Err(e) => {
@@ -1654,7 +1665,7 @@ pub async fn monitor_db(
}
async fn vacuuming_tables(db: &Pool<Postgres>) -> error::Result<()> {
sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics")
sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics")
.execute(db)
.await?;
Ok(())
backend/update_sqlx.sh
+9 -1
View File
@@ -21,9 +21,17 @@ if [[ "$(uname)" == "Darwin" ]]; then
sed -i '' 's/^samael = { version="0.0.14", features = \["xmlsec"\] }/#samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml
# Uncomment the git-based samael dependency
sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml
# Run cargo sqlx prepare with deno_core_mac
echo "Running cargo sqlx prepare with deno_core_mac..."
cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private,deno_core_mac
else
# Run cargo sqlx prepare
echo "Running cargo sqlx prepare..."
cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private
fi
cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private
# Undo the samael changes on macOS
if [[ "$(uname)" == "Darwin" ]]; then
backend/windmill-api/openapi.yaml
+25 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.548.1
version: 1.549.1
title: Windmill API
contact:
@@ -575,6 +575,27 @@ paths:
text/plain:
schema:
type: string
/w/{workspace}/users/convert_to_group/{username}:
post:
summary: convert manual user to group user (require admin privilege)
operationId: convertUserToGroup
tags:
- user
- admin
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: username
in: path
required: true
schema:
type: string
responses:
"200":
description: convert user to group user
content:
text/plain:
schema:
type: string
/github_app/connected_repositories:
get:
@@ -15353,6 +15374,8 @@ components:
AuditLog:
type: object
properties:
workspace_id:
type: string
id:
type: integer
timestamp:
@@ -15458,6 +15481,7 @@ components:
span:
type: string
required:
- workspace_id
- id
- timestamp
- username
backend/windmill-api/src/flows.rs
+1 -1
View File
@@ -937,7 +937,7 @@ async fn update_flow(
clear_schedule(&mut tx, &schedule.path, &w_id).await?;
if schedule.enabled {
tx = push_scheduled_job(&db, tx, &schedule, None).await?;
tx = push_scheduled_job(&db, tx, &schedule, None, None).await?;
}
}
backend/windmill-api/src/jobs.rs
+122 -39
View File
@@ -175,6 +175,7 @@ pub fn workspaced_service() -> Router {
.route(
"/run_and_stream/f/*script_path",
get(stream_flow_by_path)
.post(stream_flow_by_path)
.head(|| async { "" })
.layer(cors.clone())
.layer(ce_headers.clone()),
@@ -182,6 +183,7 @@ pub fn workspaced_service() -> Router {
.route(
"/run_and_stream/p/*script_path",
get(stream_script_by_path)
.post(stream_script_by_path)
.head(|| async { "" })
.layer(cors.clone())
.layer(ce_headers.clone()),
@@ -189,6 +191,7 @@ pub fn workspaced_service() -> Router {
.route(
"/run_and_stream/h/:hash",
get(stream_script_by_hash)
.post(stream_script_by_hash)
.head(|| async { "" })
.layer(cors.clone())
.layer(ce_headers.clone()),
@@ -3552,7 +3555,7 @@ pub enum DynamicSelectRunnableRef {
#[serde(rename = "deployed")]
Deployed { path: String, runnable_kind: RunnableKind },
#[serde(rename = "inline")]
Inline { code: String, language: Option<ScriptLang> },
Inline { code: String, lang: Option<ScriptLang> },
}
pub struct QueryOrBody<D>(pub Option<D>);
@@ -5220,6 +5223,7 @@ pub async fn stream_flow_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
method: hyper::http::Method,
args: RawWebhookArgs,
) -> error::Result<Response> {
stream_job(
@@ -5230,6 +5234,7 @@ pub async fn stream_flow_by_path(
RunnableId::from_flow_path(flow_path.to_path()),
args,
run_query,
method == http::Method::GET,
)
.await
}
@@ -5240,6 +5245,7 @@ pub async fn stream_script_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
method: hyper::http::Method,
args: RawWebhookArgs,
) -> error::Result<Response> {
stream_job(
@@ -5250,6 +5256,7 @@ pub async fn stream_script_by_path(
RunnableId::from_script_path(script_path.to_path()),
args,
run_query,
method == http::Method::GET,
)
.await
}
@@ -5260,6 +5267,7 @@ pub async fn stream_script_by_hash(
Extension(user_db): Extension<UserDB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
method: hyper::http::Method,
args: RawWebhookArgs,
) -> error::Result<Response> {
stream_job(
@@ -5270,6 +5278,7 @@ pub async fn stream_script_by_hash(
RunnableId::from_script_hash(script_hash),
args,
run_query,
method == http::Method::GET,
)
.await
}
@@ -5282,24 +5291,39 @@ pub async fn stream_job(
runnable_id: RunnableId,
args: RawWebhookArgs,
run_query: RunJobQuery,
is_get: bool,
) -> error::Result<Response> {
let payload_r = run_query.payload.clone().map(decode_payload).map(|x| {
x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}")))
});
let args = if is_get {
let payload_r = run_query.payload.clone().map(decode_payload).map(|x| {
x.map_err(|e| {
Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))
})
});
let payload_args = if let Some(payload) = payload_r {
payload?
let payload_args = if let Some(payload) = payload_r {
payload?
} else {
HashMap::new()
};
let mut args = args.process_args(&authed, &db, &w_id, None).await?;
args.body = args::Body::HashMap(payload_args);
let args = args
.to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor)
.await?;
args
} else {
HashMap::new()
args.to_args_from_runnable(
&authed,
&db,
&w_id,
runnable_id.clone(),
run_query.skip_preprocessor,
)
.await?
};
let mut args = args.process_args(&authed, &db, &w_id, None).await?;
args.body = args::Body::HashMap(payload_args);
let args = args
.to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor)
.await?;
let poll_delay_ms = run_query.poll_delay_ms;
let uuid = match runnable_id {
RunnableId::ScriptId(ScriptId::ScriptPath(script_path))
@@ -6351,7 +6375,7 @@ async fn run_dynamic_select(
dynamic_input = dynamic_input_res;
}
},
DynamicSelectRunnableRef::Inline { code, language } => {
DynamicSelectRunnableRef::Inline { code, lang: language } => {
dynamic_input = DynamicInput {
x_windmill_dyn_select_code: code,
x_windmill_dyn_select_lang: language.unwrap_or_default(),
@@ -6678,7 +6702,7 @@ async fn get_job_update(
&job_id,
log_offset,
stream_offset,
get_progress,
get_progress.unwrap_or(false),
running,
true,
false,
@@ -6782,6 +6806,7 @@ fn start_job_update_sse_stream(
// Send initial update immediately
let mut running = running;
let mut mem_peak = 0;
match get_job_update_data(
&opt_authed,
&opt_tokened,
@@ -6790,7 +6815,7 @@ fn start_job_update_sse_stream(
&job_id,
log_offset,
stream_offset,
get_progress,
false,
running,
true,
true,
@@ -6847,11 +6872,12 @@ fn start_job_update_sse_stream(
}
}
let mut get_progress_m: bool = false;
// Poll for updates every 1 second
let mut i = 0;
let start = Instant::now();
let mut last_ping = Instant::now();
let mut last_progress_check = Instant::now();
loop {
i += 1;
@@ -6894,6 +6920,10 @@ fn start_job_update_sse_stream(
}
tokio::time::sleep(std::time::Duration::from_millis(ms_duration)).await;
// Check progress if the user requested it, and check periodically if the job has progress
// Once it has progress, we always check progress
let check_progress = get_progress.unwrap_or(false)
&& (get_progress_m || last_progress_check.elapsed().as_secs() > 5);
match get_job_update_data(
&opt_authed,
&opt_tokened,
@@ -6902,7 +6932,7 @@ fn start_job_update_sse_stream(
&job_id,
log_offset,
stream_offset,
get_progress,
check_progress,
running,
false,
true,
@@ -6923,6 +6953,13 @@ fn start_job_update_sse_stream(
if update.new_logs.as_ref().is_some_and(|x| x.is_empty()) {
update.new_logs = None;
}
if check_progress {
if update.progress.is_some() {
get_progress_m = true;
} else {
last_progress_check = Instant::now();
}
}
// if !only_result.unwrap_or(false) {
// tracing::error!("update {:?}", update);
@@ -6990,14 +7027,20 @@ async fn get_flow_stream_delta(
) -> error::Result<Option<(Option<String>, Option<i32>)>> {
if let Some(job_id) = flow_stream_job_id {
let record = sqlx::query!(
"SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2",
"
SELECT
string_agg(stream, '' order by idx asc) as stream,
max(idx) + 1 as offset
FROM job_result_stream_v2
WHERE job_id = $2 AND idx >= $1
",
stream_offset.unwrap_or(0),
job_id,
)
.fetch_optional(db)
.await?;
if let Some(record) = record {
Ok(Some((record.new_result_stream, record.stream_offset)))
Ok(Some((record.stream, record.offset)))
} else {
Ok(None)
}
@@ -7014,7 +7057,7 @@ async fn get_job_update_data(
job_id: &Uuid,
log_offset: Option<i32>,
stream_offset: Option<i32>,
get_progress: Option<bool>,
get_progress: bool,
running: Option<bool>,
log_view: bool,
get_full_job_on_completion: bool,
@@ -7046,18 +7089,28 @@ async fn get_job_update_data(
let (result, running, mut result_stream, mut new_stream_offset, new_flow_stream_job_id) =
if let Some(tags) = tags {
let r = sqlx::query!(
"SELECT
"
WITH result_stream AS (
SELECT
string_agg(stream, '' order by idx asc) as stream,
job_id,
max(idx) + 1 as offset
FROM job_result_stream_v2
WHERE job_id = $2 AND idx >= $3
GROUP BY job_id
)
SELECT
jc.result as \"result: sqlx::types::Json<Box<RawValue>>\",
v2_job.tag,
v2_job_queue.running as \"running: Option<bool>\",
SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",
CHAR_LENGTH(rs.stream) AS stream_offset,
rs.stream AS \"result_stream: Option<String>\",
rs.offset AS stream_offset,
CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job
FROM v2_job
LEFT JOIN v2_job_queue USING (id)
LEFT JOIN v2_job_completed jc USING (id)
LEFT JOIN v2_job_status js USING (id)
LEFT JOIN job_result_stream rs ON rs.job_id = $2
LEFT JOIN result_stream rs ON rs.job_id = $2
WHERE v2_job.id = $2 AND v2_job.workspace_id = $1",
w_id,
job_id,
@@ -7086,11 +7139,21 @@ async fn get_job_update_data(
} else {
if running.is_some_and(|x| !x) {
let r = sqlx::query!(
"SELECT
"
WITH result_stream AS (
SELECT
string_agg(stream, '' order by idx asc) as stream,
job_id,
max(idx) + 1 as offset
FROM job_result_stream_v2
WHERE job_id = $1 AND idx >= $3
GROUP BY job_id
)
SELECT
COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json<Box<RawValue>>\",
jq.running as \"running: Option<bool>\",
SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",
CHAR_LENGTH(rs.stream) + 1 AS stream_offset,
rs.stream AS \"result_stream: Option<String>\",
rs.offset AS stream_offset,
CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job
FROM (
SELECT $1::uuid as job_id, $2::text as workspace_id
@@ -7098,7 +7161,7 @@ async fn get_job_update_data(
LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id
LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id
LEFT JOIN v2_job_status js ON js.id = base.job_id
LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id
LEFT JOIN result_stream rs ON rs.job_id = base.job_id
WHERE base.job_id = $1",
job_id,
w_id,
@@ -7119,10 +7182,20 @@ async fn get_job_update_data(
}
} else {
let q = sqlx::query!(
"SELECT
"
WITH result_stream AS (
SELECT
string_agg(stream, '' order by idx asc) as stream,
job_id,
max(idx) + 1 as offset
FROM job_result_stream_v2
WHERE job_id = $2 AND idx >= $3
GROUP BY job_id
)
SELECT
COALESCE(jc.result, NULL) as \"result: sqlx::types::Json<Box<RawValue>>\",
SUBSTR(rs.stream, $3) AS \"result_stream: Option<String>\",
CHAR_LENGTH(rs.stream) + 1 AS stream_offset,
rs.stream AS \"result_stream: Option<String>\",
rs.offset AS stream_offset,
COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json<Box<RawValue>>\",
CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job
FROM (
@@ -7130,7 +7203,7 @@ async fn get_job_update_data(
) base
LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id
LEFT JOIN v2_job_status js ON js.id = base.job_id
LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id
LEFT JOIN result_stream rs ON rs.job_id = base.job_id
WHERE base.job_id = $2",
w_id,
job_id,
@@ -7180,20 +7253,30 @@ async fn get_job_update_data(
})
} else {
let mut record = sqlx::query!(
"SELECT
"
WITH result_stream AS (
SELECT
string_agg(stream, '' order by idx asc) as stream,
job_id,
max(idx) + 1 as offset
FROM job_result_stream_v2
WHERE job_id = $3 AND idx >= $8
GROUP BY job_id
)
SELECT
c.id IS NOT NULL AS completed,
CASE
WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)
ELSE false
END AS running,
CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,
SUBSTR(rs.stream, $8) AS new_result_stream,
rs.stream AS new_result_stream,
COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,
COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json<Box<RawValue>>\",
(COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,
COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json<Box<RawValue>>\",
CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,
CHAR_LENGTH(rs.stream) + 1 AS stream_offset,
rs.offset AS stream_offset,
created_by AS \"created_by!\",
CASE WHEN $4::BOOLEAN THEN (
SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'
@@ -7204,14 +7287,14 @@ async fn get_job_update_data(
LEFT JOIN v2_job_runtime r USING (id)
LEFT JOIN v2_job_status f USING (id)
LEFT JOIN v2_job_completed c USING (id)
LEFT JOIN job_result_stream rs ON rs.job_id = $3
LEFT JOIN result_stream rs ON rs.job_id = $3
LEFT JOIN job_logs ON job_logs.job_id = $3
WHERE j.workspace_id = $2 AND j.id = $3
AND ($6::text[] IS NULL OR j.tag = ANY($6))",
log_offset,
w_id,
job_id,
get_progress.unwrap_or(false),
get_progress,
running,
tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>,
no_logs.unwrap_or(false),
backend/windmill-api/src/schedule.rs
+3 -3
View File
@@ -277,7 +277,7 @@ async fn create_schedule(
.await?;
if ns.enabled.unwrap_or(true) {
tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into())).await?
tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into()), None).await?
}
tx.commit().await?;
@@ -419,7 +419,7 @@ async fn edit_schedule(
.await?;
if schedule.enabled {
tx = push_scheduled_job(&db, tx, &schedule, None).await?;
tx = push_scheduled_job(&db, tx, &schedule, None, None).await?;
}
tx.commit().await?;
@@ -683,7 +683,7 @@ pub async fn set_enabled(
.await?;
if payload.enabled {
tx = push_scheduled_job(&db, tx, &schedule, None).await?;
tx = push_scheduled_job(&db, tx, &schedule, None, None).await?;
}
tx.commit().await?;
backend/windmill-api/src/scripts.rs
+1 -1
View File
@@ -900,7 +900,7 @@ async fn create_script_internal<'c>(
clear_schedule(&mut tx, &schedule.path, &w_id).await?;
if schedule.enabled {
tx = push_scheduled_job(&db, tx, &schedule, None).await?;
tx = push_scheduled_job(&db, tx, &schedule, None, None).await?;
}
}
} else {
backend/windmill-api/src/triggers/websocket/handler.rs
+2 -1
View File
@@ -12,6 +12,7 @@ use tokio_tungstenite::connect_async;
use windmill_common::{
db::UserDB,
error::{Error, Result},
worker::to_raw_value,
};
use windmill_git_sync::DeployedObject;
@@ -236,7 +237,7 @@ impl TriggerCrud for WebsocketTrigger {
url.starts_with("$flow:"),
&db,
authed.clone(),
config.url_runnable_args.as_ref(),
config.url_runnable_args.as_ref().map(to_raw_value).as_ref(),
&workspace_id,
)
.await?,
backend/windmill-api/src/triggers/websocket/listener.rs
+7 -40
View File
@@ -5,7 +5,7 @@ use crate::triggers::{
trigger_runnable, trigger_runnable_and_wait_for_raw_result,
trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs,
},
websocket::WebsocketConfig,
websocket::{get_url_from_runnable_value, WebsocketConfig},
Listener,
};
use anyhow::Context;
@@ -31,43 +31,6 @@ use windmill_common::{
use windmill_queue::PushArgsOwned;
impl ListeningTrigger<WebsocketConfig> {
async fn get_url_from_runnable(&self, db: &DB) -> Result<String> {
let runnable_kind = if self.is_flow { "Flow" } else { "Script" };
tracing::info!(
"Running {} {} to get WebSocket URL",
runnable_kind.to_lowercase(),
self.path
);
let authed = self.authed(db, "ws").await?;
let args = raw_value_to_args_hashmap(
self.trigger_config.url_runnable_args.as_ref().map(|r| &r.0),
)?;
let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx(
db,
None,
authed,
&self.workspace_id,
&self.script_path,
self.is_flow,
PushArgsOwned { args, extra: None },
None,
None,
None,
"".to_string(), // doesn't matter as no retry/error handler
)
.await?;
serde_json::from_str::<String>(result.get()).map_err(|_| {
Error::BadConfig(format!(
"{} {} did not return a string",
runnable_kind, self.path,
))
})
}
async fn send_initial_messages(
&self,
writer: &mut SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
@@ -189,8 +152,12 @@ impl Listener for WebsocketTrigger {
)) => {
return Ok(None);
},
url_result = listening_trigger.get_url_from_runnable(&db) => match url_result {
url_result = {
let authed = listening_trigger.authed(db, "ws").await?;
let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0);
let path = url.splitn(2, ':').nth(1).unwrap();
get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id)
} => match url_result {
Ok(url) => Cow::Owned(url),
Err(err) => {
return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into());
backend/windmill-api/src/triggers/websocket/mod.rs
+3 -3
View File
@@ -61,11 +61,11 @@ pub struct TestWebsocketConfig {
}
pub fn value_to_args_hashmap(
args: Option<&serde_json::Value>,
args: Option<&Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>> {
let args = if let Some(args) = args {
let args_map: Option<HashMap<String, serde_json::Value>> =
serde_json::from_value(args.clone())
serde_json::from_str(args.get())
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?;
args_map
@@ -89,7 +89,7 @@ pub async fn get_url_from_runnable_value(
is_flow: bool,
db: &DB,
authed: ApiAuthed,
args: Option<&serde_json::Value>,
args: Option<&Box<RawValue>>,
workspace_id: &str,
) -> Result<String> {
tracing::info!(
backend/windmill-api/src/users.rs
+141 -2
View File
@@ -71,6 +71,7 @@ pub fn workspaced_service() -> Router {
.route("/get/:user", get(get_workspace_user))
.route("/update/:user", post(update_workspace_user))
.route("/delete/:user", delete(delete_workspace_user))
.route("/convert_to_group/:user", post(convert_user_to_group))
.route("/is_owner/*path", get(is_owner_of_path))
.route("/whois/:username", get(whois))
.route("/whoami", get(whoami))
@@ -1362,6 +1363,138 @@ async fn update_workspace_user(
Ok(format!("user {} updated", user_email))
}
async fn convert_user_to_group(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((w_id, username_to_convert)): Path<(String, String)>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = db.begin().await?;
// Get user email and current status
let user_info = sqlx::query!(
"SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2",
username_to_convert,
&w_id
)
.fetch_optional(&mut *tx)
.await?;
let user_info = not_found_if_none(user_info, "User", &username_to_convert)?;
// Check if user is already a group user
if let Some(added_via) = &user_info.added_via {
if added_via.get("source").and_then(|v| v.as_str()) == Some("instance_group") {
return Err(Error::BadRequest("User is already a group user".to_string()));
}
}
// Find which instance groups this user belongs to that are configured for auto-add in this workspace
let eligible_groups = sqlx::query!(
r#"
SELECT
eig.igroup as group_name,
ws.auto_add_instance_groups_roles
FROM email_to_igroup eig
INNER JOIN workspace_settings ws ON ws.workspace_id = $1
WHERE eig.email = $2
AND eig.igroup = ANY(ws.auto_add_instance_groups)
"#,
&w_id,
&user_info.email
)
.fetch_all(&mut *tx)
.await?;
if eligible_groups.is_empty() {
return Err(Error::BadRequest(
"User is not a member of any instance groups configured for auto-add in this workspace".to_string()
));
}
// Determine the group with highest precedence (same logic as process_instance_group_auto_adds)
let roles: std::collections::HashMap<String, String> = if let Some(roles_json) = &eligible_groups[0].auto_add_instance_groups_roles {
serde_json::from_value(roles_json.clone()).unwrap_or_default()
} else {
std::collections::HashMap::new()
};
let mut best_group = &eligible_groups[0].group_name;
let mut best_precedence = 0u8;
for group in &eligible_groups {
let default_role = "developer".to_string();
let role = roles.get(&group.group_name).unwrap_or(&default_role);
let precedence = match role.as_str() {
"admin" => 3,
"developer" => 2,
"operator" => 1,
_ => 2,
};
if precedence > best_precedence {
best_precedence = precedence;
best_group = &group.group_name;
}
}
let primary_group_name = best_group;
// Determine role from group configuration using the selected primary group
let default_role = "developer".to_string();
let role = roles.get(primary_group_name).unwrap_or(&default_role).as_str();
let (is_admin, is_operator) = match role {
"admin" => (true, false),
"operator" => (false, true),
_ => (false, false),
};
// Update user with instance group information
let instance_group_source = serde_json::json!({
"source": "instance_group",
"group": primary_group_name
});
sqlx::query!(
"UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5",
instance_group_source,
is_admin,
is_operator,
username_to_convert,
&w_id
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed,
"users.convert_to_group",
ActionKind::Update,
&w_id,
Some(&username_to_convert),
Some([("group", primary_group_name.as_str()), ("role", role)].into()),
)
.await?;
tx.commit().await?;
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
windmill_git_sync::DeployedObject::User { email: user_info.email.clone() },
Some(format!("Converted user '{}' to group user (group: {}, role: {})", &user_info.email, primary_group_name, role)),
true,
)
.await?;
Ok(format!("User {} converted to group user (group: {}, role: {})", username_to_convert, primary_group_name, role))
}
async fn update_user(
authed: ApiAuthed,
Path(email_to_update): Path<String>,
@@ -1459,6 +1592,12 @@ async fn delete_user(
.execute(&mut *tx)
.await?;
}
// Remove user from all instance groups email_to_igroup
sqlx::query!("DELETE FROM email_to_igroup WHERE email = $1", &email_to_delete)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed,
@@ -2191,8 +2330,8 @@ async fn get_all_runnables(
.collect::<Vec<_>>(),
);
let scripts = sqlx::query!(
"SELECT workspace_id as workspace, path, summary, description, schema FROM script as o
WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false)
"SELECT workspace_id as workspace, path, summary, description, schema FROM script as o
WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false)
AND workspace_id = $1 and archived = false", workspace
)
.fetch_all(&mut *tx)
backend/windmill-common/src/result_stream.rs
+10 -3
View File
@@ -19,17 +19,24 @@ pub async fn append_result_stream_db(
workspace_id: &str,
job_id: &Uuid,
nstream: &str,
offset: i32,
) -> error::Result<()> {
if !nstream.is_empty() {
sqlx::query!(
r#"
INSERT INTO job_result_stream (workspace_id, job_id, stream)
VALUES ($1, $2, $3)
ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3
INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx)
VALUES (
$1,
$2,
$3,
$4
)
ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || $3
"#,
workspace_id,
job_id,
nstream,
offset
)
.execute(db)
.await?;
backend/windmill-duckdb-ffi-internal/src/lib.rs
+15 -4
View File
@@ -16,6 +16,17 @@ pub struct Arg {
pub json_value: serde_json::Value,
}
// Freeing from the caller side crashes the runtime with jemalloc enabled (EXIT CODE 11 SEGFAULT)
#[unsafe(no_mangle)]
pub extern "C" fn free_cstr(string: *mut c_char) -> () {
if string.is_null() {
return;
}
unsafe {
let _ = CString::from_raw(string);
}
}
#[unsafe(no_mangle)]
pub extern "C" fn run_duckdb_ffi(
query_block_list: *const *const c_char,
@@ -289,11 +300,11 @@ fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result<Box<RawValue>,
duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()),
duckdb::types::Value::Float(f) => serde_json::Value::Number(
serde_json::Number::from_f64(f as f64)
.ok_or_else(|| ("Could not convert to f64".to_string()))?,
.ok_or_else(|| "Could not convert to f64".to_string())?,
),
duckdb::types::Value::Double(f) => serde_json::Value::Number(
serde_json::Number::from_f64(f)
.ok_or_else(|| ("Could not convert to f64".to_string()))?,
.ok_or_else(|| "Could not convert to f64".to_string())?,
),
duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()),
duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()),
@@ -401,7 +412,7 @@ fn json_value_to_duckdb_value(
"double" | "float8" => duckdb::types::Value::Double(v),
"decimal" | "numeric" => duckdb::types::Value::Decimal(
Decimal::from_f64(v)
.ok_or_else(|| ("Could not convert f64 to Decimal".to_string()))?,
.ok_or_else(|| "Could not convert f64 to Decimal".to_string())?,
),
_ => duckdb::types::Value::Double(v), // default fallback
}
@@ -436,7 +447,7 @@ fn string_to_duckdb_timestamp(s: &str) -> Result<duckdb::types::Value, String> {
fn string_to_duckdb_date(s: &str) -> Result<duckdb::types::Value, String> {
use chrono::Datelike;
let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d")
.map_err(|e| (format!("Invalid date format: {}", e)))?;
.map_err(|e| format!("Invalid date format: {}", e))?;
Ok(duckdb::types::Value::Date32(date.num_days_from_ce()))
}
backend/windmill-queue/src/jobs.rs
+45 -7
View File
@@ -147,6 +147,7 @@ pub struct JobCompleted {
pub token: String,
pub canceled_by: Option<CanceledBy>,
pub duration: Option<i64>,
pub has_stream: Option<bool>,
}
pub async fn cancel_single_job<'c>(
@@ -737,6 +738,7 @@ pub async fn add_completed_job_error(
canceled_by,
flow_is_done,
duration,
false,
)
.await?;
Ok(result)
@@ -758,6 +760,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
canceled_by: Option<CanceledBy>,
flow_is_done: bool,
duration: Option<i64>,
has_stream: bool,
) -> Result<(Uuid, i64), Error> {
// tracing::error!("Start");
// let start = tokio::time::Instant::now();
@@ -782,6 +785,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
&canceled_by,
flow_is_done,
duration,
has_stream,
)
})
.retry(
@@ -790,7 +794,11 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.with_max_times(5)
.build(),
)
.when(|err| !matches!(err, Error::QuotaExceeded(_)) && !matches!(err, Error::ResultTooLarge(_)))
.when(|err| {
!matches!(err, Error::QuotaExceeded(_))
&& !matches!(err, Error::ResultTooLarge(_))
&& !matches!(err, Error::AlreadyCompleted(_))
})
.notify(|err, dur| {
tracing::error!("Could not insert completed job, retrying in {dur:#?}, err: {err:#?}");
})
@@ -834,6 +842,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
canceled_by: &Option<CanceledBy>,
flow_is_done: bool,
duration: Option<i64>,
has_stream: bool,
) -> windmill_common::error::Result<(Option<Uuid>, i64, bool)> {
// let start = std::time::Instant::now();
@@ -855,7 +864,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
return value;
}
let _duration = sqlx::query_scalar!(
let duration = sqlx::query_scalar!(
"INSERT INTO v2_job_completed AS cj
( workspace_id
, id
@@ -891,10 +900,33 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
/* $9 */ duration,
/* $10 */ result_columns as Option<&Vec<String>>,
)
.fetch_one(&mut *tx)
.fetch_optional(&mut *tx)
.await
.map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))?;
let duration = if let Some(duration) = duration {
duration
} else {
let already_inserted = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)",
job_id
)
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))?
.unwrap_or(false);
if already_inserted {
return Err(Error::AlreadyCompleted(format!(
"The queued job {job_id} is already completed."
)));
} else {
return Err(Error::AlreadyCompleted(format!(
"There is no queued job anymore for {job_id} but there is no completed job either."
)));
}
};
if let Some(labels) = result.wm_labels() {
sqlx::query!(
"UPDATE v2_job SET labels = (
@@ -924,7 +956,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
)
WHERE id = $3",
&queued_job.id.to_string(),
_duration,
duration,
parent_job
)
.execute(&mut *tx)
@@ -1116,6 +1148,12 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.execute(&mut *tx)
.await?;
if !success || has_stream {
sqlx::query!("DELETE FROM job_result_stream_v2 WHERE job_id = $1", job_id)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
tracing::info!(
@@ -1124,7 +1162,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
path = &queued_job.runnable_path(),
job_kind = ?queued_job.kind,
started_at = ?queued_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()),
duration = ?_duration,
duration = ?duration,
permissioned_as = ?queued_job.permissioned_as,
email = ?queued_job.permissioned_as_email,
created_by = queued_job.created_by,
@@ -1137,7 +1175,7 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
queued_job.id
);
// tracing::info!("completed job: {:?}", start.elapsed().as_micros());
Ok((None, _duration, _skip_downstream_error_handlers))
Ok((None, duration, _skip_downstream_error_handlers))
}
async fn check_result_size<T: ValidableJson>(
@@ -1655,7 +1693,7 @@ pub async fn handle_maybe_scheduled_job<'c>(
let push_next_job_future = (|| {
tokio::time::timeout(std::time::Duration::from_secs(5), async {
let mut tx = db.begin().await?;
tx = push_scheduled_job(db, tx, &schedule, None).await?;
tx = push_scheduled_job(db, tx, &schedule, None, Some(job.scheduled_for)).await?;
tx.commit().await?;
Ok::<(), Error>(())
})
backend/windmill-queue/src/schedule.rs
+16
View File
@@ -9,6 +9,8 @@
use crate::push;
use crate::PushIsolationLevel;
use anyhow::Context;
use chrono::DateTime;
use chrono::Utc;
use sqlx::{PgExecutor, Postgres, Transaction};
use std::collections::HashMap;
use std::str::FromStr;
@@ -34,6 +36,7 @@ pub async fn push_scheduled_job<'c>(
mut tx: Transaction<'c, Postgres>,
schedule: &Schedule,
authed: Option<&Authed>,
now_cutoff: Option<DateTime<Utc>>,
) -> Result<Transaction<'c, Postgres>> {
if !*LICENSE_KEY_VALID.read().await {
return Err(error::Error::BadRequest(
@@ -50,6 +53,19 @@ pub async fn push_scheduled_job<'c>(
let now = now_from_db(&mut *tx).await?;
let now = match now_cutoff {
Some(now_cutoff) if now_cutoff >= now => {
tracing::error!(
"now_cutoff ({:?}) is after now ({:?}) for schedule {}. Using now_cutoff + 1s. This likely means the pg clock was shifted backwards.",
now_cutoff,
now,
&schedule.path
);
now_cutoff + chrono::Duration::seconds(1)
}
_ => now,
};
let starting_from = match schedule.paused_until {
Some(paused_until) if paused_until > now => paused_until.with_timezone(&tz),
paused_until_o => {
backend/windmill-worker/src/ai/mod.rs
+1
View File
@@ -4,4 +4,5 @@
pub mod image_handler;
pub mod providers;
pub mod query_builder;
pub mod sse;
pub mod types;
backend/windmill-worker/src/ai/providers/google_ai.rs
+19 -2
View File
@@ -5,7 +5,7 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err
use crate::ai::{
image_handler::download_and_encode_s3_image,
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
types::*,
};
@@ -157,18 +157,24 @@ impl QueryBuilder for GoogleAIQueryBuilder {
matches!(output_type, OutputType::Text)
}
fn supports_streaming(&self) -> bool {
// Google AI supports streaming for text output
true
}
async fn build_request(
&self,
args: &BuildRequestArgs<'_>,
client: &AuthedClient,
workspace_id: &str,
stream: bool,
) -> Result<String, Error> {
match args.output_type {
OutputType::Text => {
// For text output, use OpenAI-compatible format
let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI);
openai_builder
.build_request(args, client, workspace_id)
.build_request(args, client, workspace_id, stream)
.await
}
OutputType::Image => self.build_image_request(args, client, workspace_id).await,
@@ -238,6 +244,17 @@ impl QueryBuilder for GoogleAIQueryBuilder {
}
}
async fn parse_streaming_response(
&self,
response: reqwest::Response,
stream_event_processor: StreamEventProcessor,
) -> Result<ParsedResponse, Error> {
let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI);
openai_builder
.parse_streaming_response(response, stream_event_processor)
.await
}
fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String {
match output_type {
OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint
backend/windmill-worker/src/ai/providers/openai.rs
+53 -2
View File
@@ -5,7 +5,8 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err
use crate::ai::{
image_handler::download_and_encode_s3_image,
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
sse::{OpenAISSEParser, SSEParser},
types::*,
};
@@ -90,6 +91,7 @@ pub struct OpenAIRequest<'a> {
pub max_completion_tokens: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub response_format: Option<ResponseFormat>,
pub stream: bool,
}
pub struct OpenAIQueryBuilder {
@@ -162,6 +164,7 @@ impl OpenAIQueryBuilder {
args: &BuildRequestArgs<'_>,
client: &AuthedClient,
workspace_id: &str,
stream: bool,
) -> Result<String, Error> {
let prepared_messages = self
.prepare_messages_for_api(args.messages, client, workspace_id)
@@ -196,6 +199,7 @@ impl OpenAIQueryBuilder {
temperature: args.temperature,
max_completion_tokens: args.max_tokens,
response_format,
stream,
};
serde_json::to_string(&request)
@@ -255,14 +259,23 @@ impl QueryBuilder for OpenAIQueryBuilder {
true
}
fn supports_streaming(&self) -> bool {
// OpenAI supports streaming for text output
true
}
async fn build_request(
&self,
args: &BuildRequestArgs<'_>,
client: &AuthedClient,
workspace_id: &str,
stream: bool,
) -> Result<String, Error> {
match args.output_type {
OutputType::Text => self.build_text_request(args, client, workspace_id).await,
OutputType::Text => {
self.build_text_request(args, client, workspace_id, stream)
.await
}
OutputType::Image => self.build_image_request(args, client, workspace_id).await,
}
}
@@ -330,10 +343,48 @@ impl QueryBuilder for OpenAIQueryBuilder {
}
}),
tool_calls: first_choice.message.tool_calls.unwrap_or_default(),
events_str: None,
})
}
}
async fn parse_streaming_response(
&self,
response: reqwest::Response,
stream_event_processor: StreamEventProcessor,
) -> Result<ParsedResponse, Error> {
let mut openai_sse_parser = OpenAISSEParser::new(stream_event_processor);
openai_sse_parser.parse_events(response).await?;
let OpenAISSEParser {
accumulated_content,
accumulated_tool_calls,
mut events_str,
stream_event_processor,
} = openai_sse_parser;
// Process streaming events with error handling
for tool_call in accumulated_tool_calls.values() {
let event = StreamingEvent::ToolCallArguments {
call_id: tool_call.id.clone(),
function_name: tool_call.function.name.clone(),
arguments: tool_call.function.arguments.clone(),
};
stream_event_processor.send(event, &mut events_str).await?;
}
Ok(ParsedResponse::Text {
content: if accumulated_content.is_empty() {
None
} else {
Some(accumulated_content)
},
tool_calls: accumulated_tool_calls.into_values().collect(),
events_str: Some(events_str),
})
}
fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String {
let path = match output_type {
OutputType::Text => "chat/completions",
backend/windmill-worker/src/ai/providers/openrouter.rs
+19 -2
View File
@@ -5,7 +5,7 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err
use crate::ai::{
providers::openai::{OpenAIQueryBuilder, OpenAIResponse},
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder},
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor},
types::*,
};
@@ -70,17 +70,23 @@ impl QueryBuilder for OpenRouterQueryBuilder {
true
}
fn supports_streaming(&self) -> bool {
// OpenRouter supports streaming for text output
true
}
async fn build_request(
&self,
args: &BuildRequestArgs<'_>,
client: &AuthedClient,
workspace_id: &str,
stream: bool,
) -> Result<String, Error> {
match args.output_type {
OutputType::Text => {
// For text, use standard OpenAI format without modalities
self.openai_builder
.build_request(args, client, workspace_id)
.build_request(args, client, workspace_id, stream)
.await
}
OutputType::Image => {
@@ -184,9 +190,20 @@ impl QueryBuilder for OpenRouterQueryBuilder {
.join(" "),
}),
tool_calls: first_choice.message.tool_calls.unwrap_or_default(),
events_str: None,
})
}
async fn parse_streaming_response(
&self,
response: reqwest::Response,
stream_event_processor: StreamEventProcessor,
) -> Result<ParsedResponse, Error> {
self.openai_builder
.parse_streaming_response(response, stream_event_processor)
.await
}
fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String {
// OpenRouter uses the same endpoint for both text and image generation
format!("{}/chat/completions", base_url)
backend/windmill-worker/src/ai/query_builder.rs
+103 -8
View File
@@ -1,13 +1,19 @@
use async_trait::async_trait;
use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object};
use windmill_common::{
client::AuthedClient, error::Error, s3_helpers::S3Object, worker::Connection,
};
use windmill_queue::MiniPulledJob;
use crate::ai::{
providers::{
google_ai::GoogleAIQueryBuilder,
openai::{OpenAIQueryBuilder, OpenAIToolCall},
openrouter::OpenRouterQueryBuilder,
use crate::{
ai::{
providers::{
google_ai::GoogleAIQueryBuilder,
openai::{OpenAIQueryBuilder, OpenAIToolCall},
openrouter::OpenRouterQueryBuilder,
},
types::*,
},
types::*,
job_logger::append_result_stream,
};
/// Arguments for building an AI request
@@ -26,7 +32,7 @@ pub struct BuildRequestArgs<'a> {
/// Response from AI provider
pub enum ParsedResponse {
Text { content: Option<String>, tool_calls: Vec<OpenAIToolCall> },
Text { content: Option<String>, tool_calls: Vec<OpenAIToolCall>, events_str: Option<String> },
Image { base64_data: String },
}
@@ -36,17 +42,32 @@ pub trait QueryBuilder: Send + Sync {
/// Check if this provider supports tools with the given output type
fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool;
/// Check if this provider supports streaming
fn supports_streaming(&self) -> bool;
/// Build the request body for the provider
async fn build_request(
&self,
args: &BuildRequestArgs<'_>,
client: &AuthedClient,
workspace_id: &str,
stream: bool,
) -> Result<String, Error>;
/// Parse the response from the provider
async fn parse_response(&self, response: reqwest::Response) -> Result<ParsedResponse, Error>;
/// Parse streaming response from the provider
async fn parse_streaming_response(
&self,
_response: reqwest::Response,
_stream_event_processor: StreamEventProcessor,
) -> Result<ParsedResponse, Error> {
return Err(Error::internal_err(
"Missing implementation for parse_streaming_response for this provider".to_string(),
));
}
/// Get the API endpoint for this provider
fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String;
@@ -69,3 +90,77 @@ pub fn create_query_builder(provider: &ProviderWithResource) -> Box<dyn QueryBui
_ => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Pass provider kind for Azure handling
}
}
pub struct StreamEventProcessor {
tx: tokio::sync::mpsc::Sender<String>,
pub handle: Option<tokio::task::JoinHandle<()>>,
}
impl Clone for StreamEventProcessor {
fn clone(&self) -> Self {
Self { tx: self.tx.clone(), handle: None }
}
}
impl StreamEventProcessor {
pub fn new(conn: &Connection, job: &MiniPulledJob) -> Self {
let (tx, mut rx) = tokio::sync::mpsc::channel::<String>(100);
let conn = conn.clone();
let job_id = job.id.clone();
let workspace_id = job.workspace_id.clone();
let handle = tokio::spawn(async move {
let mut offset = -1;
while let Some(event) = rx.recv().await {
offset += 1;
match tokio::time::timeout(
std::time::Duration::from_secs(20),
append_result_stream(&conn, &workspace_id, &job_id, &event, offset),
)
.await
{
Ok(res) => {
if let Err(err) = res {
tracing::error!("Failed to save stream event: {}", err);
}
}
Err(err) => {
tracing::error!("Did not manage to save stream event after 20 seconds, stopping stream event processor: {}", err);
break;
}
}
}
});
Self { tx, handle: Some(handle) }
}
pub async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> {
match serde_json::to_string(&event) {
Ok(event_json) => {
let event_json = format!("{}\n", event_json);
events_str.push_str(&event_json);
if let Err(err) = self
.tx
.send(event_json.clone())
.await
.map_err(|e| Error::internal_err(format!("Failed to send event: {}", e)))
{
tracing::error!(
"Failed to send event to stream event processor, skiping event: {}",
err
);
}
Ok(())
}
Err(e) => Err(Error::internal_err(format!(
"Failed to serialize streaming event {:#?}, error is: {}",
event, e
))),
}
}
pub fn to_handle(self) -> Option<tokio::task::JoinHandle<()>> {
self.handle
}
}
backend/windmill-worker/src/ai/sse.rs
+158
View File
@@ -0,0 +1,158 @@
use std::collections::HashMap;
use reqwest::Response;
use serde::Deserialize;
use serde_json;
use tokio_stream::StreamExt;
use windmill_common::{error::Error, utils::rd_string};
use crate::ai::{
providers::openai::{OpenAIFunction, OpenAIToolCall},
query_builder::StreamEventProcessor,
types::StreamingEvent,
};
#[derive(Deserialize)]
pub struct OpenAIChoiceDeltaToolCallFunction {
pub name: Option<String>,
pub arguments: Option<String>,
}
#[derive(Deserialize)]
pub struct OpenAIChoiceDeltaToolCall {
pub index: Option<i64>,
pub id: Option<String>,
pub function: Option<OpenAIChoiceDeltaToolCallFunction>,
}
#[derive(Deserialize)]
pub struct OpenAIChoiceDelta {
pub content: Option<String>,
pub tool_calls: Option<Vec<OpenAIChoiceDeltaToolCall>>,
}
#[derive(Deserialize)]
pub struct OpenAIChoice {
pub delta: Option<OpenAIChoiceDelta>,
}
#[derive(Deserialize)]
pub struct OpenAISSEEvent {
pub choices: Option<Vec<OpenAIChoice>>,
}
pub trait SSEParser {
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>;
async fn parse_events(&mut self, response: Response) -> Result<(), Error> {
let mut stream = response.bytes_stream();
let mut buffer = String::new();
while let Some(chunk_result) = stream.next().await {
let chunk = chunk_result
.map_err(|e| Error::internal_err(format!("Failed to read chunk: {}", e)))?;
// Convert chunk to string and add to buffer
let chunk_str = String::from_utf8_lossy(&chunk);
buffer.push_str(&chunk_str);
// Process complete lines from buffer
while let Some(newline_pos) = buffer.find("\n\n") {
let line = buffer.drain(..newline_pos + 2).collect::<String>();
let line = line.trim_end_matches('\n');
// Skip empty lines and comments
if line.is_empty() || line.starts_with(':') {
continue;
}
// Parse SSE data field
if let Some(data) = line.strip_prefix("data: ") {
if data == "[DONE]" {
// OpenAI sends [DONE] to indicate end of stream
return Ok(());
}
self.parse_event_data(data).await?;
}
}
}
Ok(())
}
}
pub struct OpenAISSEParser {
pub accumulated_content: String,
pub accumulated_tool_calls: HashMap<i64, OpenAIToolCall>,
pub events_str: String,
pub stream_event_processor: StreamEventProcessor,
}
impl OpenAISSEParser {
pub fn new(stream_event_processor: StreamEventProcessor) -> Self {
Self {
accumulated_content: String::new(),
accumulated_tool_calls: HashMap::new(),
events_str: String::new(),
stream_event_processor,
}
}
}
impl SSEParser for OpenAISSEParser {
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
let event: OpenAISSEEvent = serde_json::from_str(data).map_err(|e| {
Error::internal_err(format!("Failed to parse SSE chunk {}: {}", data, e))
})?;
if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) {
if let Some(delta) = choices.remove(0).delta {
if let Some(content) = delta.content.filter(|s| !s.is_empty()) {
self.accumulated_content.push_str(&content);
let event = StreamingEvent::TokenDelta { content };
self.stream_event_processor
.send(event, &mut self.events_str)
.await?;
}
if let Some(tool_calls) = delta.tool_calls {
for (idx, tool_call) in tool_calls.into_iter().enumerate() {
let idx = tool_call.index.unwrap_or_else(|| idx as i64);
if let Some(function) = tool_call.function {
if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) {
if let Some(arguments) = function.arguments {
tool_call.function.arguments += &arguments;
}
} else {
let fun_name = function.name.unwrap_or_default();
let call_id = tool_call.id.unwrap_or_else(|| rd_string(24));
let event = StreamingEvent::ToolCall {
call_id: call_id.clone(),
function_name: fun_name.clone(),
};
self.stream_event_processor
.send(event, &mut self.events_str)
.await?;
self.accumulated_tool_calls.insert(
idx,
OpenAIToolCall {
id: call_id,
function: OpenAIFunction {
name: fun_name,
arguments: function.arguments.unwrap_or_default(),
},
r#type: "function".to_string(),
},
);
}
}
}
}
}
}
Ok(())
}
}
backend/windmill-worker/src/ai/types.rs
+19
View File
@@ -116,6 +116,7 @@ pub struct AIAgentArgs {
pub output_schema: Option<OpenAPISchema>,
pub output_type: Option<OutputType>,
pub user_images: Option<Vec<S3Object>>,
pub streaming: Option<bool>,
}
#[derive(Deserialize, Debug)]
@@ -153,6 +154,24 @@ impl ProviderWithResource {
pub struct AIAgentResult<'a> {
pub output: Box<RawValue>,
pub messages: Vec<Message<'a>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub wm_stream: Option<String>,
}
/// Events for streaming AI responses
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum StreamingEvent {
/// Individual token from the AI response
TokenDelta { content: String },
/// Tool call has started
ToolCall { call_id: String, function_name: String },
/// Tool call arguments are complete
ToolCallArguments { call_id: String, function_name: String, arguments: String },
/// Tool execution has started
ToolExecution { call_id: String, function_name: String },
/// Tool execution result
ToolResult { call_id: String, function_name: String, result: String, success: bool },
}
#[derive(Serialize, Deserialize, Clone, Debug)]
backend/windmill-worker/src/ai_executor.rs
+130 -18
View File
@@ -1,3 +1,4 @@
use anyhow::Context;
use async_recursion::async_recursion;
use regex::Regex;
use serde_json::value::RawValue;
@@ -5,8 +6,7 @@ use std::{collections::HashMap, sync::Arc};
use ulid;
use uuid::Uuid;
use windmill_common::{
ai_providers::AIProvider,
ai_providers::AZURE_API_VERSION,
ai_providers::{AIProvider, AZURE_API_VERSION},
cache,
client::AuthedClient,
db::DB,
@@ -27,10 +27,14 @@ use windmill_queue::{
use crate::{
ai::{
image_handler::upload_image_to_s3,
query_builder::{create_query_builder, BuildRequestArgs, ParsedResponse},
query_builder::{
create_query_builder, BuildRequestArgs, ParsedResponse, StreamEventProcessor,
},
types::*,
},
common::{build_args_map, error_to_value, OccupancyMetrics},
common::{
build_args_map, error_to_value, resolve_job_timeout, OccupancyMetrics, StreamNotifier,
},
create_job_dir,
handle_child::run_future_with_polling_update_job_poller,
handle_queued_job, parse_sig_of_lang,
@@ -44,7 +48,6 @@ lazy_static::lazy_static! {
}
const MAX_AGENT_ITERATIONS: usize = 10;
const REQUEST_TIMEOUT_SECONDS: u64 = 120;
fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result<Box<RawValue>, Error> {
let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some
@@ -114,6 +117,7 @@ pub async fn handle_ai_agent_job(
worker_name: &str,
hostname: &str,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
has_stream: &mut bool,
) -> Result<Box<RawValue>, Error> {
let args = build_args_map(job, client, conn).await?;
@@ -264,6 +268,12 @@ pub async fn handle_ai_agent_job(
let mut inner_occupancy_metrics = occupancy_metrics.clone();
let stream_notifier = StreamNotifier::new(conn, job);
if let Some(stream_notifier) = stream_notifier {
stream_notifier.update_flow_status_with_stream_job();
}
let agent_fut = run_agent(
db,
conn,
@@ -279,6 +289,7 @@ pub async fn handle_ai_agent_job(
worker_name,
hostname,
killpill_rx,
has_stream,
);
let result = run_future_with_polling_update_job_poller(
@@ -412,6 +423,7 @@ pub async fn run_agent(
worker_name: &str,
hostname: &str,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text);
let base_url = args.provider.get_base_url(db).await?;
@@ -501,6 +513,21 @@ pub async fn run_agent(
// For non-Anthropic providers, response_format is handled by the query builder
}
// Check if streaming is enabled and supported
let should_stream = args.streaming.unwrap_or(false)
&& query_builder.supports_streaming()
&& output_type == &OutputType::Text;
*has_stream = should_stream;
let mut final_events_str = String::new();
let stream_event_processor = if should_stream {
Some(StreamEventProcessor::new(conn, job))
} else {
None
};
// Main agent loop
for i in 0..MAX_AGENT_ITERATIONS {
if used_structured_output_tool {
@@ -522,21 +549,25 @@ pub async fn run_agent(
};
let request_body = query_builder
.build_request(&build_args, client, &job.workspace_id)
.build_request(&build_args, client, &job.workspace_id, should_stream)
.await?;
let endpoint =
query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type);
let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type);
let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout)
.await
.0;
let mut request = HTTP_CLIENT
.post(&endpoint)
.timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECONDS))
.timeout(timeout)
.header("Content-Type", "application/json");
// Apply authentication headers
for (header_name, header_value) in auth_headers {
request = request.header(header_name, header_value);
for (header_name, header_value) in &auth_headers {
request = request.header(*header_name, header_value.clone());
}
if args.provider.kind.is_azure_openai(&base_url) {
@@ -551,10 +582,21 @@ pub async fn run_agent(
match resp.error_for_status_ref() {
Ok(_) => {
let parsed = query_builder.parse_response(resp).await?;
let parsed = if let Some(stream_event_processor) = stream_event_processor.clone() {
query_builder
.parse_streaming_response(resp, stream_event_processor)
.await?
} else {
// Handle non-streaming response
query_builder.parse_response(resp).await?
};
match parsed {
ParsedResponse::Text { content: response_content, tool_calls } => {
ParsedResponse::Text { content: response_content, tool_calls, events_str } => {
if let Some(events_str) = events_str {
final_events_str.push_str(&events_str);
}
if let Some(ref response_content) = response_content {
actions.push(AgentAction::Message {});
messages.push(OpenAIMessage {
@@ -589,6 +631,17 @@ pub async fn run_agent(
// Handle tool calls (keeping existing tool execution logic)
for tool_call in tool_calls.iter() {
// Stream tool call progress
if let Some(ref stream_event_processor) = stream_event_processor {
let event = StreamingEvent::ToolExecution {
call_id: tool_call.id.clone(),
function_name: tool_call.function.name.clone(),
};
stream_event_processor
.send(event, &mut final_events_str)
.await?;
}
// Check if this is the structured output tool
if structured_output_tool_name
.as_ref()
@@ -631,10 +684,22 @@ pub async fn run_agent(
update_flow_status_module_with_actions(db, parent_job, &actions)
.await?;
let raw_tool_call_args = if tool_call.function.arguments.is_empty()
{
"{}".to_string()
} else {
tool_call.function.arguments.clone()
};
let tool_call_args =
serde_json::from_str::<HashMap<String, Box<RawValue>>>(
&tool_call.function.arguments,
)?;
&raw_tool_call_args,
)
.with_context(|| {
format!(
"Failed to parse tool call arguments for tool call {}: {}",
tool_call.function.name, tool_call.function.arguments
)
})?;
let job_payload = match tool.module.get_value()? {
FlowModuleValue::Script {
@@ -806,12 +871,13 @@ pub async fn run_agent(
worker_name,
)
.await;
let error_message =
format!("Error running tool: {}", err_string);
messages.push(OpenAIMessage {
role: "tool".to_string(),
content: Some(OpenAIContent::Text(format!(
"Error running tool: {}",
err_string
))),
content: Some(OpenAIContent::Text(
error_message.clone(),
)),
tool_call_id: Some(tool_call.id.clone()),
agent_action: Some(AgentAction::ToolCall {
job_id,
@@ -820,6 +886,21 @@ pub async fn run_agent(
}),
..Default::default()
});
// Stream tool result (error case)
if let Some(ref stream_event_processor) =
stream_event_processor
{
let tool_result_event = StreamingEvent::ToolResult {
call_id: tool_call.id.clone(),
function_name: tool_call.function.name.clone(),
result: error_message,
success: false,
};
stream_event_processor
.send(tool_result_event, &mut final_events_str)
.await?;
}
update_flow_status_module_with_actions_success(
db, parent_job, false,
)
@@ -857,7 +938,6 @@ pub async fn run_agent(
"Tool job completed but no result".to_string(),
));
};
messages.push(OpenAIMessage {
role: "tool".to_string(),
content: Some(OpenAIContent::Text(
@@ -871,6 +951,22 @@ pub async fn run_agent(
}),
..Default::default()
});
// Stream tool result (success case)
if let Some(ref stream_event_processor) =
stream_event_processor
{
let tool_result_event = StreamingEvent::ToolResult {
call_id: tool_call.id.clone(),
function_name: tool_call.function.name.clone(),
result: result.get().to_string(),
success: true,
};
stream_event_processor
.send(tool_result_event, &mut final_events_str)
.await?;
}
update_flow_status_module_with_actions_success(
db, parent_job, success,
)
@@ -930,8 +1026,24 @@ pub async fn run_agent(
None => to_raw_value(&""),
};
if let Some(stream_event_processor) = stream_event_processor {
if let Some(handle) = stream_event_processor.to_handle() {
if let Err(e) = handle.await {
return Err(Error::internal_err(format!(
"Error waiting for stream event processor: {}",
e
)));
}
}
}
Ok(to_raw_value(&AIAgentResult {
output: output_value,
messages: final_messages,
wm_stream: if !final_events_str.is_empty() {
Some(final_events_str)
} else {
None
},
}))
}
backend/windmill-worker/src/bun_executor.rs
+5 -1
View File
@@ -858,6 +858,7 @@ pub async fn handle_bun_job(
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
@@ -1115,7 +1116,7 @@ async function run() {{
let res = await Main.{main_name}(...argsArr);
if (isAsyncIterable(res)) {{
for await (const chunk of res) {{
console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n'));
}}
res = null;
}}
@@ -1331,6 +1332,7 @@ try {{
false,
occupancy_metrics,
stream_notifier,
has_stream,
)
.await?;
tracing::info!(
@@ -1493,6 +1495,8 @@ try {{
)
.await?;
*has_stream = handle_result.result_stream.is_some();
if apply_preprocessor {
let args = read_file(&format!("{job_dir}/args.json"))
.await
backend/windmill-worker/src/dedicated_worker.rs
+3 -3
View File
@@ -186,14 +186,14 @@ pub async fn handle_dedicated_process(
let result = Arc::new(result);
append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await;
if line.starts_with("wm_res[success]:") {
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap()
} else {
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap()
}
},
Err(e) => {
tracing::error!("Could not deserialize job result `{line}`: {e:?}");
job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap();
job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap();
},
};
logs = init_log.clone();
backend/windmill-worker/src/deno_executor.rs
+5 -1
View File
@@ -194,6 +194,7 @@ pub async fn handle_deno_job(
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
// let mut start = Instant::now();
let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
@@ -298,7 +299,7 @@ async function run() {{
let res: any = await {main_name}(...argsArr);
if (isAsyncIterable(res)) {{
for await (const chunk of res) {{
console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n'));
}}
res = null;
}}
@@ -440,6 +441,9 @@ try {{
stream_notifier,
)
.await?;
*has_stream = handle_result.result_stream.is_some();
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await
{
backend/windmill-worker/src/duckdb_executor.rs
+11 -16
View File
@@ -1,6 +1,6 @@
use std::cell::RefCell;
use std::env;
use std::ffi::{c_char, CString};
use std::ffi::{c_char, CStr, CString};
use std::ptr::NonNull;
use std::sync::{Arc, Mutex};
@@ -193,6 +193,7 @@ struct DuckDbFfiLib {
column_order_ptr: *mut *mut c_char,
) -> *mut c_char,
>,
free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>,
}
impl DuckDbFfiLib {
@@ -232,6 +233,7 @@ impl DuckDbFfiLib {
let lib = Box::leak(Box::new(lib));
Ok(DuckDbFfiLib {
run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? },
free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? },
})
}
}
@@ -264,8 +266,9 @@ fn run_duckdb_ffi_safe<'a>(
let w_id = CString::new(w_id).map_err(to_anyhow)?;
let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi;
let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr;
let mut column_order: *mut c_char = std::ptr::null_mut();
let result_cstr = unsafe {
let result_str = unsafe {
let ptr = run_duckdb_ffi(
query_block_list.as_ptr(),
query_block_list_count,
@@ -275,27 +278,19 @@ fn run_duckdb_ffi_safe<'a>(
w_id.as_ptr(),
&mut column_order,
);
CString::from_raw(ptr) // Using from_raw to take ownership and ensure it gets freed
let str = CStr::from_ptr(ptr).to_string_lossy().to_string();
free_cstr(ptr);
str
};
let column_order = if column_order.is_null() {
None
} else {
Some(unsafe {
serde_json::from_str::<Vec<String>>(&CString::from_raw(column_order).to_string_lossy())?
})
let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() };
unsafe { free_cstr(column_order) };
Some(serde_json::from_str::<Vec<String>>(&str)?)
};
let result_str = result_cstr
.to_str()
.map_err(|e| {
Error::ExecutionErr(format!(
"Failed to convert result C string to Rust string: {}",
e.to_string()
))
})?
.to_string();
if result_str.starts_with("ERROR") {
Err(Error::ExecutionErr(result_str[6..].to_string()))
} else {
backend/windmill-worker/src/handle_child.rs
+13 -3
View File
@@ -29,7 +29,7 @@ use windmill_queue::{append_logs, CanceledBy};
use std::os::unix::process::ExitStatusExt;
use std::process::ExitStatus;
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering};
use std::sync::Arc;
use std::{io, panic, time::Duration};
@@ -388,6 +388,7 @@ pub async fn write_lines(
let mut pipe_stdout = pipe_stdout;
let is_stream = Arc::new(AtomicBool::new(false));
let offset = Arc::new(AtomicI32::new(0));
while let Some(line) = output.by_ref().next().await {
let do_write_ = do_write.shared();
@@ -486,6 +487,7 @@ pub async fn write_lines(
let pg_log_total_size = pg_log_total_size.clone();
let stream_notifier = stream_notifier.clone();
let is_stream = is_stream.clone();
let offset = offset.clone();
(do_write, write_result) = tokio::spawn(async move {
if !nstream.is_empty() {
if let Some(stream_notifier) = stream_notifier {
@@ -493,9 +495,17 @@ pub async fn write_lines(
is_stream.store(true, Ordering::SeqCst);
stream_notifier.update_flow_status_with_stream_job();
}
}
};
if let Err(err) = append_result_stream(&conn, &w_id, &job_id, &nstream).await {
if let Err(err) = append_result_stream(
&conn,
&w_id,
&job_id,
&nstream,
offset.fetch_add(1, Ordering::SeqCst),
)
.await
{
tracing::error!(
"Unable to send result stream for job {job_id}. Error was: {:?}",
err
backend/windmill-worker/src/job_logger.rs
+15 -3
View File
@@ -8,6 +8,7 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED};
use windmill_common::{error, DB};
use windmill_queue::append_logs;
use serde::Serialize;
use std::sync::atomic::AtomicU32;
use std::sync::Arc;
@@ -62,22 +63,33 @@ pub async fn append_job_logs(
}
}
#[derive(Serialize)]
struct ResultStreamBody<'a> {
result_stream: &'a str,
offset: i32,
}
pub async fn append_result_stream(
conn: &Connection,
workspace_id: &str,
job_id: &Uuid,
nstream: &str,
offset: i32,
) -> error::Result<()> {
match conn {
Connection::Sql(db) => {
append_result_stream_db(db, workspace_id, job_id, nstream).await?;
append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?;
}
Connection::Http(client) => {
let body = ResultStreamBody { result_stream: nstream, offset };
if let Err(e) = client
.post::<_, String>(
&format!("/api/w/{}/agent_workers/push_logs/{}", workspace_id, job_id),
&format!(
"/api/w/{}/agent_workers/push_result_stream/{}",
workspace_id, job_id
),
None,
&nstream,
&body,
)
.await
{
backend/windmill-worker/src/js_eval.rs
+14 -7
View File
@@ -770,6 +770,7 @@ pub async fn eval_fetch_timeout(
_load_client: bool,
_occupation_metrics: &mut OccupancyMetrics,
_stream_notifier: Option<StreamNotifier>,
_has_stream: &mut bool,
) -> anyhow::Result<Box<RawValue>> {
use serde_json::value::to_raw_value;
Ok(to_raw_value("require deno_core").unwrap())
@@ -792,6 +793,7 @@ pub async fn eval_fetch_timeout(
load_client: bool,
occupation_metrics: &mut OccupancyMetrics,
stream_notifier: Option<StreamNotifier>,
has_stream: &mut bool,
) -> windmill_common::error::Result<Box<RawValue>> {
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::<String>();
@@ -808,9 +810,11 @@ pub async fn eval_fetch_timeout(
let conn_ = conn.clone();
let w_id_ = w_id.to_string();
tokio::spawn(async move {
let mut offset = -1;
while let Some(stream) = result_stream_receiver.recv().await {
use crate::job_logger::append_result_stream;
if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream).await {
offset += 1;
if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream, offset).await {
tracing::error!("failed to append result stream: {e}");
}
}
@@ -978,22 +982,24 @@ pub async fn eval_fetch_timeout(
drop(js_runtime);
if let Ok(r) = r {
match handle.await {
Ok(Some(logs)) => Ok(merge_result_stream(r, Some(logs)).await),
Ok(None) => Ok(r),
Ok(Some(logs)) => {
Ok(merge_result_stream(r, Some(logs)).await.map(|r| (r, true)))
}
Ok(None) => Ok(r.map(|r| (r, false))),
Err(e) => Err(Error::ExecutionErr(e.to_string())),
}
} else {
r
r.map(|r| r.map(|r| (r, false)))
}
// r
};
let r = runtime.block_on(future)?;
// tracing::info!("total: {:?}", instant.elapsed());
r as windmill_common::error::Result<Box<RawValue>>
r as windmill_common::error::Result<(Box<RawValue>, bool)>
});
let res = run_future_with_polling_update_job_poller(
let (res, new_has_stream) = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
conn,
@@ -1012,6 +1018,7 @@ pub async fn eval_fetch_timeout(
}
e
})?;
*has_stream = new_has_stream;
*mem_peak = (res.get().len() / 1000) as i32;
Ok(res)
}
@@ -1109,7 +1116,7 @@ function processStreamIterative(res) {{
iterator.next().then(function(result) {{
if (!result.done) {{
const chunk = result.value;
console.log("WM_STREAM: " + chunk.replace('\n', '\\n'));
console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n'));
// Continue the loop
step();
}} else {{
backend/windmill-worker/src/python_executor.rs
+3
View File
@@ -544,6 +544,7 @@ pub async fn handle_python_job(
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
has_stream: &mut bool,
) -> windmill_common::error::Result<Box<RawValue>> {
let script_path = crate::common::use_flow_root_path(job.runnable_path());
@@ -885,6 +886,8 @@ mount {{
)
.await?;
*has_stream = handle_result.result_stream.is_some();
if apply_preprocessor {
let args = read_file(&format!("{job_dir}/args.json"))
.await
backend/windmill-worker/src/result_processor.rs
+17 -2
View File
@@ -17,7 +17,14 @@ use windmill_common::otel_oss::FutureExt;
use uuid::Uuid;
use windmill_common::{
add_time, error::{self, Error}, flow_status::{FlowJobDuration}, jobs::JobKind, utils::WarnAfterExt, worker::{to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, KillpillSender, DB
add_time,
error::{self, Error},
flow_status::FlowJobDuration,
jobs::JobKind,
utils::WarnAfterExt,
worker::{to_raw_value, Connection, WORKER_GROUP},
worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap},
KillpillSender, DB,
};
#[cfg(feature = "benchmark")]
@@ -405,6 +412,7 @@ pub async fn process_result(
preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
conn: &Connection,
duration: Option<i64>,
has_stream: bool,
) -> error::Result<bool> {
match result {
Ok(result) => {
@@ -421,6 +429,7 @@ pub async fn process_result(
cached_res_path,
token: token.to_string(),
duration,
has_stream: Some(has_stream),
},
)
.with_context(windmill_common::otel_oss::otel_ctx())
@@ -483,6 +492,7 @@ pub async fn process_result(
cached_res_path,
token: token.to_string(),
duration,
has_stream: Some(has_stream),
},
)
.with_context(windmill_common::otel_oss::otel_ctx())
@@ -557,6 +567,7 @@ pub async fn process_completed_job(
duration,
result_columns,
preprocessed_args,
has_stream,
..
}: JobCompleted,
client: &AuthedClient,
@@ -621,6 +632,7 @@ pub async fn process_completed_job(
canceled_by,
false,
duration,
has_stream.unwrap_or(false),
)
.await?;
drop(job);
@@ -679,7 +691,10 @@ pub async fn process_completed_job(
&job.workspace_id,
false,
Arc::new(serde_json::value::to_raw_value(&result).unwrap()),
duration.map(|x| FlowJobDuration { started_at: job.started_at.unwrap(), duration_ms: x }),
duration.map(|x| FlowJobDuration {
started_at: job.started_at.unwrap(),
duration_ms: x,
}),
false,
&same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(),
&worker_dir,
backend/windmill-worker/src/worker.rs
+14
View File
@@ -765,6 +765,7 @@ pub async fn handle_all_job_kind_error(
cached_res_path: None,
token: authed_client.token.clone(),
duration: None,
has_stream: Some(false),
},
false,
)
@@ -1687,6 +1688,7 @@ pub async fn run_worker(
token: "".to_string(),
canceled_by: None,
duration: None,
has_stream: Some(false),
},
true,
)
@@ -2229,6 +2231,7 @@ async fn do_nativets(
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
has_stream: &mut bool,
) -> windmill_common::error::Result<Box<RawValue>> {
let args = build_args_map(job, client, conn).await?.map(Json);
let job_args = if args.is_some() {
@@ -2255,6 +2258,7 @@ async fn do_nativets(
true,
occupancy_metrics,
stream_notifier,
has_stream,
)
.await?)
}
@@ -2408,6 +2412,7 @@ pub async fn handle_queued_job(
cached_res_path: None,
token: client.token.clone(),
duration: None,
has_stream: Some(false),
},
true,
)
@@ -2491,6 +2496,7 @@ pub async fn handle_queued_job(
let mut column_order: Option<Vec<String>> = None;
let mut new_args: Option<HashMap<String, Box<RawValue>>> = None;
let mut has_stream = false;
let result = match job.kind {
JobKind::Dependencies => match conn {
Connection::Sql(db) => {
@@ -2582,6 +2588,7 @@ pub async fn handle_queued_job(
worker_name,
hostname,
killpill_rx,
&mut has_stream,
)
.await
}
@@ -2614,6 +2621,7 @@ pub async fn handle_queued_job(
occupancy_metrics,
killpill_rx,
precomputed_agent_info,
&mut has_stream,
)
.await;
occupancy_metrics.total_duration_of_running_jobs +=
@@ -2646,6 +2654,7 @@ pub async fn handle_queued_job(
new_args,
conn,
Some(started.elapsed().as_millis() as i64),
has_stream,
)
.await
}
@@ -2833,6 +2842,7 @@ async fn handle_code_execution_job(
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
has_stream: &mut bool,
) -> error::Result<Box<RawValue>> {
let script_hash = || {
job.runnable_id
@@ -3172,6 +3182,7 @@ async fn handle_code_execution_job(
canceled_by,
worker_name,
occupancy_metrics,
has_stream,
)
.await?;
return Ok(result);
@@ -3245,6 +3256,7 @@ mount {{
new_args,
occupancy_metrics,
precomputed_agent_info,
has_stream,
)
.await
}
@@ -3264,6 +3276,7 @@ mount {{
envs,
new_args,
occupancy_metrics,
has_stream,
)
.await
}
@@ -3286,6 +3299,7 @@ mount {{
new_args,
occupancy_metrics,
precomputed_agent_info,
has_stream,
)
.await
}
backend/windmill-worker/src/worker_flow.rs
+2
View File
@@ -1473,6 +1473,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None,
true,
None,
false,
)
.await?;
duration
@@ -1492,6 +1493,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None,
true,
None,
false,
)
.await?;
duration
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.548.1";
export const VERSION = "v1.549.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/src/main.ts
+1 -1
View File
@@ -68,7 +68,7 @@ export {
// }
// });
export const VERSION = "1.548.1";
export const VERSION = "1.549.1";
export const WM_FORK_PREFIX = "wm-fork";
cli/windmill-utils-internal/package.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-utils-internal",
"version": "1.3.0",
"version": "1.3.1",
"description": "Internal utility functions for Windmill",
"main": "dist/index.js",
"types": "dist/index.d.ts",
@@ -21,4 +21,4 @@
"files": [
"dist/**/*"
]
}
}
cli/windmill-utils-internal/src/config/index.ts
+1 -1
View File
@@ -1 +1 @@
export * from "./config.ts";
export * from "./config";
cli/windmill-utils-internal/src/index.ts
+5 -5
View File
@@ -8,8 +8,8 @@
* - Cross-platform path constants
*/
export * from "./inline-scripts.ts";
export * from "./path-utils.ts";
export * from "./parse.ts";
export * from "./config.ts";
export { SEP, DELIMITER } from "./constants.ts";
export * from "./inline-scripts";
export * from "./path-utils";
export * from "./parse";
export * from "./config";
export { SEP, DELIMITER } from "./constants";
cli/windmill-utils-internal/src/inline-scripts/extractor.ts
+2 -2
View File
@@ -1,5 +1,5 @@
import { newPathAssigner } from "../path-utils/path-assigner.ts";
import { FlowModule } from "../gen/types.gen.ts";
import { newPathAssigner } from "../path-utils/path-assigner";
import { FlowModule } from "../gen/types.gen";
/**
* Represents an inline script extracted from a flow module
cli/windmill-utils-internal/src/inline-scripts/index.ts
+2 -2
View File
@@ -1,2 +1,2 @@
export * from "./replacer.ts";
export * from "./extractor.ts";
export * from "./replacer";
export * from "./extractor";
cli/windmill-utils-internal/src/inline-scripts/replacer.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { FlowModule } from "../gen/types.gen.ts";
import { FlowModule } from "../gen/types.gen";
/**
* Replaces inline script references with actual file content from the filesystem.
cli/windmill-utils-internal/src/parse/index.ts
+1 -1
View File
@@ -1 +1 @@
export * from "./parse-schema.ts";
export * from "./parse-schema";
cli/windmill-utils-internal/src/parse/parse-schema.ts
+13 -13
View File
@@ -1,7 +1,7 @@
/**
* Type alias for enum values - can be an array of strings or undefined
*/
export type EnumType = string[] | undefined;
export type EnumType = string[] | { label: string; value: string }[] | undefined;
/**
* Represents a property in a JSON schema with various validation and display options
@@ -17,7 +17,7 @@ export interface SchemaProperty {
items?: {
type?: "string" | "number" | "bytes" | "object" | "resource";
contentEncoding?: "base64";
enum?: string[];
enum?: EnumType;
resourceType?: string;
properties?: { [name: string]: SchemaProperty };
};
@@ -54,22 +54,22 @@ export function argSigToJsonSchemaType(
| string
| { resource: string | null }
| {
list:
| (string | { name?: string; props?: { key: string; typ: any }[] })
| { str: any }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| null;
}
list:
| (string | { name?: string; props?: { key: string; typ: any }[] })
| { str: any }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| null;
}
| { dynselect: string }
| { dynmultiselect: string }
| { str: string[] | null }
| { object: { name?: string; props?: { key: string; typ: any }[] } }
| {
oneof: {
label: string;
properties: { key: string; typ: any }[];
}[];
},
oneof: {
label: string;
properties: { key: string; typ: any }[];
}[];
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
cli/windmill-utils-internal/src/path-utils/index.ts
+1 -1
View File
@@ -1 +1 @@
export * from "./path-assigner.ts";
export * from "./path-assigner";
cli/windmill-utils-internal/src/path-utils/path-assigner.ts
+1 -1
View File
@@ -1,4 +1,4 @@
import { RawScript } from "../gen/types.gen.ts";
import { RawScript } from "../gen/types.gen";
const INLINE_SCRIPT_PREFIX = "inline_script";
frontend/package-lock.json
Generated
+6 -6
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.548.1",
"version": "1.549.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.548.1",
"version": "1.549.1",
"hasInstallScript": true,
"license": "AGPL-3.0",
"dependencies": {
@@ -86,7 +86,7 @@
"windmill-parser-wasm-ts": "1.538.0",
"windmill-parser-wasm-yaml": "1.510.1",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.0",
"windmill-utils-internal": "^1.3.1",
"xterm": "^5.3.0",
"xterm-readline": "^1.1.2",
"y-monaco": "^0.1.4",
@@ -13542,9 +13542,9 @@
"integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g=="
},
"node_modules/windmill-utils-internal": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.0.tgz",
"integrity": "sha512-UH7G+NVODkhm4o3BbjaOrSE2Qu+J6ro7+vpsIs+GvjDZM4ogSN7aJfnQNHW7Ke0VY74BKZVClTKROe/N6I5Reg==",
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.1.tgz",
"integrity": "sha512-afRGUDcvaUfGu7FA6DD0xWECQiKnXADs0N4WyQQ+OvaloxZ4oQzdEpLnVab/m3T02hhs29ru4Ilrdxu3ozyT5Q==",
"license": "Apache 2.0"
},
"node_modules/word-wrap": {
frontend/package.json
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.548.1",
"version": "1.549.1",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -151,7 +151,7 @@
"windmill-parser-wasm-ts": "1.538.0",
"windmill-parser-wasm-yaml": "1.510.1",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.0",
"windmill-utils-internal": "^1.3.1",
"xterm": "^5.3.0",
"xterm-readline": "^1.1.2",
"y-monaco": "^0.1.4",
frontend/src/lib/common.ts
+1 -1
View File
@@ -15,7 +15,7 @@ export interface PropertyDisplayInfo {
propertiesNumber: number
}
export type EnumType = string[] | undefined
export type EnumType = string[] | { value: string; label: string }[] | undefined
export interface SchemaProperty {
type: string | undefined
frontend/src/lib/components/AIAgentLogViewer.svelte
+2 -2
View File
@@ -13,14 +13,14 @@
import { onMount } from 'svelte'
type AgentActionWithContent = NonNullable<FlowStatusModule['agent_actions']>[number] & {
content: string
content?: unknown
}
const resultSchema = z.object({
messages: z.array(
z.object({
role: z.string(),
content: z.string().optional(),
content: z.unknown(),
agent_action: z
.union([
z.object({
frontend/src/lib/components/ArgEnum.svelte
+13 -5
View File
@@ -32,11 +32,19 @@
let customItems: string[] = $state([])
let items = $derived.by(() => {
const l = [...(enum_ ? enum_ : []), ...customItems].map((item) => ({
value: item,
label: enumLabels?.[item] ?? item
}))
if (create && filterText && l.every((i) => i.value !== filterText)) {
const l = [...(enum_ ? enum_ : []), ...customItems]
.map((item) => {
if (typeof item === 'string') {
return {
value: item,
label: enumLabels?.[item] ?? item
}
} else if (typeof item === 'object') {
return item
}
})
.filter((i) => i != undefined)
if (create && filterText && l.every((i) => i?.value !== filterText)) {
l.push({ value: filterText, label: `Add new: ${filterText}` })
}
return l
frontend/src/lib/components/ArgInput.svelte
+6 -1
View File
@@ -255,7 +255,12 @@
if (inputCat === 'string') {
nvalue = nullable ? null : ''
} else if (inputCat == 'enum' && required) {
nvalue = enum_?.[0]
let firstV = enum_?.[0]
if (typeof firstV === 'string') {
nvalue = firstV
} else if (firstV && typeof firstV === 'object') {
nvalue = firstV.value
}
} else if (inputCat == 'boolean') {
nvalue = false
} else if (inputCat == 'list') {
frontend/src/lib/components/Dev.svelte
+1 -1
View File
@@ -574,7 +574,7 @@
let themeDarkRaw = $derived($page.url.searchParams.get('activeColorTheme'))
let themeDark = $derived(themeDarkRaw == '2' || themeDarkRaw == '4')
$effect(() => {
$effect.pre(() => {
setContext<{ token?: string }>('AuthToken', { token })
})
$effect.pre(() => {
frontend/src/lib/components/DisplayResult.svelte
+38 -32
View File
@@ -36,7 +36,7 @@
import Tooltip from './Tooltip.svelte'
import HighlightTheme from './HighlightTheme.svelte'
import type { DisplayResultUi } from './custom_ui'
import { getContext, hasContext, createEventDispatcher, onDestroy } from 'svelte'
import { getContext, hasContext, createEventDispatcher, onDestroy, untrack } from 'svelte'
import { toJsonStr } from '$lib/utils'
import { userStore } from '$lib/stores'
import ResultStreamDisplay from './ResultStreamDisplay.svelte'
@@ -139,24 +139,32 @@
function isTableRowObject(json) {
// check array of objects (with possible a first row of headers)
const hasHeaders =
Array.isArray(json[0]) &&
json[0].length > 0 &&
json[0].length <= 50 &&
json[0].every((item) => typeof item === 'string')
return isTableRowObjectInner(json, hasHeaders)
}
function isTableRowObjectInner(json: any, hasHeaders: boolean) {
return (
Array.isArray(json) &&
json.length > 0 &&
(json.every(
(item) =>
item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item)
) ||
(Array.isArray(json[0]) &&
json[0].every((item) => typeof item === 'string') &&
json
.slice(1)
.every(
(item) =>
item &&
typeof item === 'object' &&
Object.keys(item).length > 0 &&
!Array.isArray(item)
)))
json.length > (hasHeaders ? 1 : 0) &&
json.every((item, index) => {
if (hasHeaders && index === 0) {
return true
}
if (item && typeof item === 'object') {
let keys = Object.keys(item)
if (keys.length > 0 && !Array.isArray(item)) {
if (hasHeaders || keys.length <= 50) {
return true
}
}
}
return false
})
)
}
@@ -200,6 +208,7 @@
}
let size = roughSizeOfObject(result)
console.debug('size of object', size)
// Otherwise, check if the result is too large (10kb) for json
if (size > TABLE_MAX_SIZE) {
@@ -363,28 +372,23 @@
json.length > 0 &&
Array.isArray(json[0]) &&
json[0].length > 0 &&
json[0].every((item) => typeof item === 'string') &&
json
.slice(1)
.every(
(item) =>
item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item)
)
json[0].every((item) => typeof item === 'string')
) {
const headers = json[0]
const rows = json.slice(1)
const rows: { [key: string]: string }[] = new Array(json.length - 1)
const result = rows.map((row) => {
for (let i = 1; i < json.length; i++) {
const obj: { [key: string]: string } = {}
const row = json[i]
for (const header of headers) {
obj[header] = row[header]
for (let j = 0; j < headers.length; j++) {
obj[headers[j]] = row[headers[j]]
}
return obj
})
rows[i - 1] = obj
}
return result
return rows
}
return json
@@ -479,7 +483,9 @@
$effect(() => {
;[result]
resultKind = inferResultKind(result)
untrack(() => {
resultKind = inferResultKind(result)
})
})
$effect(() => {
chooseToolbarLocation(
frontend/src/lib/components/FlowLoopIterationPreview.svelte
+3 -3
View File
@@ -67,7 +67,7 @@
let selectedJobStep: string | undefined = $state(undefined)
let isRunning: boolean = $state(false)
let jobProgressReset: (() => void) | undefined = $state(undefined)
let progressBar: FlowProgressBar | undefined = $state(undefined)
export function test() {
runPreview(previewArgs, undefined)
@@ -80,7 +80,7 @@
args: Record<string, any>,
restartedFrom: RestartedFrom | undefined
) {
jobProgressReset?.()
progressBar?.reset()
const newFlow = { value: { modules }, summary: '' }
jobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom)
isRunning = true
@@ -161,7 +161,7 @@
<div></div>
</div>
<div class="w-full flex flex-col gap-y-1">
<FlowProgressBar {job} bind:reset={jobProgressReset} />
<FlowProgressBar {job} bind:this={progressBar} />
</div>
<div class="overflow-y-auto grow pr-4">
<div class="max-h-1/2 overflow-auto border-b">
frontend/src/lib/components/FlowMetadata.svelte
+8 -4
View File
@@ -10,9 +10,13 @@
import { Clock, MemoryStick, Calendar, Bot, User, Code2 } from 'lucide-svelte'
import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte'
export let job: Job
const SMALL_ICON_SIZE = 14
export let scheduleEditor: ScheduleEditor
interface Props {
job: Job
scheduleEditor: ScheduleEditor
}
let { job, scheduleEditor }: Props = $props()
</script>
<div
@@ -74,11 +78,11 @@
<Calendar size={SMALL_ICON_SIZE} class="text-secondary min-w-3.5" />
<span class="whitespace-nowrap">
Schedule:
<!-- svelte-ignore a11y-invalid-attribute -->
<!-- svelte-ignore a11y_invalid_attribute -->
<a
href="#"
class="break-words text-blue-600 font-normal"
on:click={() =>
onclick={() =>
scheduleEditor?.openEdit(job.schedule_path ?? '', job.job_kind == 'flow')}
>
{truncateRev(job.schedule_path, 40)}
frontend/src/lib/components/FlowStatusViewer.svelte
+1 -1
View File
@@ -88,7 +88,7 @@
retryStatus.val = {}
suspendStatus.val = {}
globalRefreshes = {}
flowState.val = {}
flowState = {}
localDurationStatuses = {}
localModuleStates = {}
}
frontend/src/lib/components/JobArgs.svelte
+20 -16
View File
@@ -14,14 +14,18 @@
import { deepEqual } from 'fast-equals'
import { isWindmillTooBigObject } from './job_args'
export let id: string | undefined = undefined
export let args: any
export let argLabel: string | undefined = undefined
export let workspace: string | undefined = undefined
interface Props {
id?: string | undefined
args: any
argLabel?: string | undefined
workspace?: string | undefined
}
let jsonViewer: Drawer
let runLocally: Drawer
let jsonStr = ''
let { id = undefined, args, argLabel = undefined, workspace = undefined }: Props = $props()
let jsonViewer: Drawer | undefined = $state()
let runLocally: Drawer | undefined = $state()
let jsonStr = $state('')
function pythonCode() {
return `
@@ -53,9 +57,9 @@ ${Object.entries(args)
}
</script>
{#if args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'PREPROCESSOR_ARGS_ARE_DISCARDED'}
{#if args && typeof args === 'object' && deepEqual( Object.keys(args ?? {}), ['reason'] ) && args['reason'] == 'PREPROCESSOR_ARGS_ARE_DISCARDED'}
Preprocessor args are discarded
{:else if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'}
{:else if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args ?? {}), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'}
The args are too big in size to be able to fetch alongside job. Please <a
href="/api/w/{workspace}/jobs_u/get_args/{id}"
target="_blank">download the JSON file to view them</a
@@ -68,21 +72,21 @@ ${Object.entries(args)
<Cell head first>{argLabel ?? 'Arg'}</Cell>
<Cell head last>Value</Cell>
</tr>
<svelte:fragment slot="headerAction">
{#snippet headerAction()}
<button
on:click={() => {
onclick={() => {
jsonStr = JSON.stringify(args, null, 4)
jsonViewer.openDrawer()
jsonViewer?.openDrawer()
}}
>
<Expand size={18} />
</button>
</svelte:fragment>
{/snippet}
</Head>
<tbody class="divide-y w-full">
{#if args && typeof args === 'object' && Object.keys(args).length > 0}
{#each Object.entries(args).sort((a, b) => a[0].localeCompare(b[0])) as [arg, value]}
{#if args && typeof args === 'object' && Object.keys(args ?? {}).length > 0}
{#each Object.entries(args ?? {}).sort( (a, b) => a?.[0]?.localeCompare(b?.[0]) ) as [arg, value]}
<Row>
<Cell first>{arg}</Cell>
<Cell><ArgInfo {value} /></Cell>
@@ -124,7 +128,7 @@ ${Object.entries(args)
Download
</Button>
<Button
on:click={runLocally.openDrawer}
on:click={() => runLocally?.openDrawer()}
color="light"
size="xs"
startIcon={{ icon: ChevronRightSquare }}
frontend/src/lib/components/JobLoader.svelte
+10 -47
View File
@@ -68,15 +68,6 @@
children
}: Props = $props()
/// Last time asked for job progress
let lastTimeCheckedProgress: number | undefined = undefined
/// Will try to poll progress every 5s and if once progress returned was not undefined, will be ignored
/// and getProgressRate will be used instead
const getProgressRetryRate: number = 5000
/// How often loader poll progress
const getProgressRate: number = 1000
let workspace = $derived(workspaceOverride ?? $workspaceStore)
let syncIteration: number = 0
@@ -96,6 +87,8 @@
let lastNoLogs = $state(noLogs)
let lastCompletedJobId = $state<string | undefined>(undefined)
let token = getContext<{ token?: string }>('AuthToken')
$effect(() => {
let newIsLoading = currentId !== undefined
untrack(() => {
@@ -144,6 +137,7 @@
export async function abstractRun(fn: () => Promise<string>, callbacks?: Callbacks) {
try {
isLoading = true
scriptProgress = undefined
lastCompletedJobId = undefined
clearCurrentJob()
lastCallbacks = callbacks
@@ -252,9 +246,6 @@
if (logOffset == 0) {
logOffset = job?.logs?.length ? job.logs?.length + 1 : 0
}
if (resultStreamOffset == 0) {
resultStreamOffset = job?.result_stream?.length ? job.result_stream?.length + 1 : 0
}
}
export async function getLogs() {
if (job) {
@@ -299,10 +290,6 @@
hash?: string,
callbacks?: Callbacks
): Promise<string> {
// Reset in case we rerun job without reloading
scriptProgress = undefined
lastTimeCheckedProgress = undefined
return abstractRun(
() =>
JobService.runScriptPreview({
@@ -364,6 +351,7 @@
syncIteration = 0
errorIteration = 0
currentId = testId
scriptProgress = undefined
if (loadPlaceholderJobOnStart) {
job = structuredClone(loadPlaceholderJobOnStart)
} else {
@@ -383,33 +371,6 @@
}
}
function setJobProgress(job: Job) {
let getProgress: boolean | undefined = undefined
// We only pull individual job progress this way
// Flow's progress we are getting from FlowStatusModule of flow job
if (job.job_kind == 'script' || isScriptPreview(job.job_kind)) {
// First time, before running job, lastTimeCheckedProgress is always undefined
if (lastTimeCheckedProgress) {
const lastTimeCheckedMs = Date.now() - lastTimeCheckedProgress
// Ask for progress if the last time we asked is >5s OR the progress was once not undefined
if (
lastTimeCheckedMs > getProgressRetryRate ||
(scriptProgress != undefined && lastTimeCheckedMs > getProgressRate)
) {
lastTimeCheckedProgress = Date.now()
getProgress = true
}
} else {
// Make it think we asked for progress, but in reality we didnt. First 5s we want to wait without putting extra work on db
// 99.99% of the jobs won't have progress be set so we have to do a balance between having low-latency for jobs that use it and job that don't
// we would usually not care to have progress the first 5s and jobs that are less than 5s
lastTimeCheckedProgress = Date.now()
}
}
return getProgress
}
const clamp = (num: number, min: number, max: number) => Math.min(Math.max(num, min), max)
function updateJobFromProgress(
@@ -425,6 +386,7 @@
}
}
if (previewJobUpdates.progress) {
console.log('progress', previewJobUpdates.progress)
// Progress cannot go back and cannot be set to 100
scriptProgress = clamp(previewJobUpdates.progress, scriptProgress ?? 0, 99)
}
@@ -485,7 +447,6 @@
try {
if (job && `running` in job) {
callbacks?.running?.({ id })
let getProgress: boolean | undefined = setJobProgress(job)
refreshLogOffset()
@@ -494,7 +455,8 @@
id,
running: job.running,
logOffset: logOffset,
getProgress: getProgress
streamOffset: resultStreamOffset,
getProgress: false
})
if ((previewJobUpdates.running ?? false) || (previewJobUpdates.completed ?? false)) {
@@ -640,7 +602,9 @@
}
let getProgress: boolean | undefined =
onlyResult || !job ? undefined : setJobProgress(job)
onlyResult || !job
? undefined
: job.job_kind == 'script' || isScriptPreview(job.job_kind)
refreshLogOffset()
// Build SSE URL with query parameters
@@ -676,7 +640,6 @@
)
}
let token = getContext<{ token?: string }>('AuthToken')
if (token?.token && token.token != '') {
params.set('token', token.token)
}
frontend/src/lib/components/JobStatus.svelte
+5 -1
View File
@@ -10,7 +10,11 @@
const SMALL_ICON_SIZE = 12
export let job: QueuedJob | CompletedJob | undefined
interface Props {
job: QueuedJob | CompletedJob | undefined
}
let { job }: Props = $props()
</script>
{#if job && 'success' in job && job.success}
frontend/src/lib/components/ModulePreviewResultViewer.svelte
+1 -8
View File
@@ -46,8 +46,6 @@
const { stepsInputArgs } = getContext<FlowEditorContext>('FlowEditorContext')
let jobProgressReset: () => void = $state(() => {})
let outputPickerInner: OutputPickerInner | undefined = $state(undefined)
export function getOutputPickerInner() {
return outputPickerInner
@@ -63,12 +61,7 @@
<Splitpanes horizontal>
<Pane size={65} minSize={10} class="text-sm text-tertiary">
{#if scriptProgress}
<JobProgressBar
job={testJob}
bind:scriptProgress
bind:reset={jobProgressReset}
compact={true}
/>
<JobProgressBar job={testJob} {scriptProgress} compact={true} />
{/if}
<OutputPickerInner
frontend/src/lib/components/QueuePosition.svelte
+1 -1
View File
@@ -61,8 +61,8 @@
}
return () => {
scheduledForTimeout && clearTimeout(scheduledForTimeout)
if (queuePositionInterval) {
scheduledForTimeout && clearTimeout(scheduledForTimeout)
clearInterval(queuePositionInterval)
}
}
frontend/src/lib/components/ScriptEditor.svelte
+4 -4
View File
@@ -124,7 +124,7 @@
})
let showHistoryDrawer = $state(false)
let jobProgressReset: (() => void) | undefined = $state(undefined)
let jobProgressBar: JobProgressBar | undefined = $state(undefined)
let diffMode = $state(false)
let websocketAlive = $state({
@@ -201,7 +201,7 @@
export async function runTest() {
// Not defined if JobProgressBar not loaded
if (jobProgressReset) jobProgressReset()
jobProgressBar?.reset()
//@ts-ignore
let job = await jobLoader.runPreview(
path,
@@ -784,8 +784,8 @@
<!-- Put to the slot in logpanel -->
<JobProgressBar
job={testJob}
bind:scriptProgress
bind:reset={jobProgressReset}
{scriptProgress}
bind:this={jobProgressBar}
compact={true}
/>
{/if}
frontend/src/lib/components/StringTypeNarrowing.svelte
+41 -26
View File
@@ -99,12 +99,12 @@
}
let choice = `choice ${enum_?.length ? enum_?.length + 1 : 1}`
enum_ = enum_ ? enum_.concat(choice) : [choice]
enum_ = enum_ ? (enum_.concat(choice) as EnumType) : [choice]
}
function remove(item: string) {
enum_ = (enum_ || []).filter((el) => el !== item)
if (enum_.length == 0) {
enum_ = (enum_ || []).filter((el) => el !== item) as EnumType
if (enum_?.length == 0) {
enum_ = undefined
}
@@ -231,34 +231,49 @@
<div class="flex flex-col gap-1">
{#if enum_}
{#each enum_ as _, i}
<div class="flex flex-row w-full gap-2 pt-2">
<input
id="input"
type="text"
bind:value={enum_[i]}
oninput={(event) => enum_ && onEnumKeyChange(event?.currentTarget.value, enum_[i])}
/>
{#if enumLabels !== undefined}
{#if typeof enum_[i] === 'string'}
<div class="flex flex-row w-full gap-2 pt-2">
<input
id="input"
type="text"
bind:value={enumLabels[enum_[i]]}
placeholder="Optional title..."
oninput={(event) => {
if (event?.currentTarget.value === '') {
if (enumLabels === undefined) {
enumLabels = {}
}
enum_ && delete enumLabels[enum_[i]]
}
}}
bind:value={enum_[i]}
oninput={(event) =>
enum_ &&
typeof enum_[i] === 'string' &&
onEnumKeyChange(event?.currentTarget.value, enum_[i])}
/>
{/if}
{#if enumLabels !== undefined}
<input
id="input"
type="text"
bind:value={enumLabels[enum_[i]]}
placeholder="Optional title..."
oninput={(event) => {
if (event?.currentTarget.value === '') {
if (enumLabels === undefined) {
enumLabels = {}
}
if (typeof enum_?.[i] === 'string') {
enum_ && delete enumLabels[enum_[i]]
}
}
}}
/>
{/if}
{#if allowKindChange}
<Button size="sm" on:click={() => enum_ && remove(enum_[i])}>-</Button>
{/if}
</div>
{#if allowKindChange}
<Button
size="sm"
on:click={() => enum_ && typeof enum_[i] === 'string' && remove(enum_[i])}
>-</Button
>
{/if}
</div>
{:else}
<div class="flex flex-row w-full gap-2 pt-2">
{JSON.stringify(enum_[i])} is not a string, remove it
</div>
{/if}
{/each}
{/if}
</div>
frontend/src/lib/components/apps/editor/component/components.ts
+1 -1
View File
@@ -3588,7 +3588,7 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm'
fieldType: 'object',
value: {},
tooltip:
'This enables setting form enum values dynamically using an object: keys are field names, and values are arrays of strings.'
'This enables setting form enum values dynamically using an object: keys are field names, and values are arrays of strings or { "label": "myLabel", "value": "myValue" }.'
},
displayType: {
frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte
+6
View File
@@ -260,6 +260,9 @@
USERS_ADD_GLOBAL: 'users.add_global',
USERS_IMPERSONATE: 'users.impersonate',
USERS_LEAVE_WORKSPACE: 'users.leave_workspace',
USERS_SCIM_CREATE: 'users.scim_create',
USERS_SCIM_DELETE: 'users.scim_delete',
USERS_SCIM_UPDATE: 'users.scim_update',
OAUTH_LOGIN: 'oauth.login',
OAUTH_LOGIN_FAILURE: 'oauth.login_failure',
OAUTH_SIGNUP: 'oauth.signup',
@@ -287,6 +290,9 @@
IGROUP_DELETE: 'igroup.delete',
IGROUP_ADDUSER: 'igroup.adduser',
IGROUP_REMOVEUSER: 'igroup.removeuser',
INSTANCE_GROUPS_SCIM_CREATE: 'instance_groups.scim_create',
INSTANCE_GROUPS_SCIM_DELETE: 'instance_groups.scim_delete',
INSTANCE_GROUPS_SCIM_UPDATE: 'instance_groups.scim_update',
VARIABLES_DECRYPT_SECRET: 'variables.decrypt_secret',
WORKSPACES_EDIT_COMMAND_SCRIPT: 'workspaces.edit_command_script',
WORKSPACES_EDIT_DEPLOY_TO: 'workspaces.edit_deploy_to',
frontend/src/lib/components/auditLogs/AuditLogsTable.svelte
+18 -6
View File
@@ -18,6 +18,7 @@
selectedId?: number | undefined
usernameFilter?: string | undefined
resourceFilter?: string | undefined
showWorkspace?: boolean
onselect?: (id: number) => void
}
@@ -31,6 +32,7 @@
selectedId = undefined,
usernameFilter = $bindable(),
resourceFilter = $bindable(),
showWorkspace = false,
onselect
}: Props = $props()
@@ -128,9 +130,12 @@
class="flex flex-row bg-surface-secondary sticky top-0 w-full p-2 pr-4 text-xs font-semibold"
>
<div class="w-1/12">ID</div>
<div class="w-3/12">Timestamp</div>
<div class="w-3/12">Username</div>
<div class="w-3/12">Operation</div>
<div class={showWorkspace ? "w-2/12" : "w-3/12"}>Timestamp</div>
<div class={showWorkspace ? "w-2/12" : "w-3/12"}>Username</div>
{#if showWorkspace}
<div class="w-2/12">Workspace</div>
{/if}
<div class={showWorkspace ? "w-2/12" : "w-3/12"}>Operation</div>
<div class="w-2/12">Resource</div>
</div>
</div>
@@ -174,10 +179,10 @@
<div class="w-1/12 text-xs truncate">
{logOrDate.log.id}
</div>
<div class="w-3/12 text-xs">
<div class={showWorkspace ? "w-2/12 text-xs" : "w-3/12 text-xs"}>
{displayDate(logOrDate.log.timestamp)}
</div>
<div class="w-3/12 text-xs">
<div class={showWorkspace ? "w-2/12 text-xs" : "w-3/12 text-xs"}>
<div class="flex flex-row gap-2 items-center">
<div class="whitespace-nowrap overflow-x-auto no-scrollbar max-w-60">
{logOrDate.log.username}
@@ -196,7 +201,14 @@
/>
</div>
</div>
<div class="w-3/12 text-xs">
{#if showWorkspace}
<div class="w-2/12 text-xs">
<div class="whitespace-nowrap overflow-x-auto no-scrollbar max-w-60">
{logOrDate.log.workspace_id}
</div>
</div>
{/if}
<div class={showWorkspace ? "w-2/12 text-xs" : "w-3/12 text-xs"}>
<div class="flex flex-row gap-1">
<Badge
on:click={() => {
frontend/src/lib/components/flows/FlowProgressBar.svelte
+30 -20
View File
@@ -2,18 +2,27 @@
import { type Job } from '$lib/gen'
import ProgressBar from '../progressBar/ProgressBar.svelte'
export let job: Job | undefined = undefined
export let currentSubJobProgress: number | undefined = undefined
interface Props {
job?: Job | undefined
currentSubJobProgress?: number | undefined
class?: string
}
let error: number | undefined = undefined
let index = 0
let subIndex: number | undefined = undefined
let subLength: number | undefined = undefined
let length = 1
let nextInProgress = false
let subIndexIsPercent: boolean = false
let {
job = undefined,
currentSubJobProgress = $bindable(undefined),
class: className
}: Props = $props()
$: if (job) updateJobProgress(job)
let error: number | undefined = $state(undefined)
let index = $state(0)
let subIndex: number | undefined = $state(undefined)
let subLength: number | undefined = $state(undefined)
let length = $state(1)
let nextInProgress = $state(false)
let subIndexIsPercent: boolean = $state(false)
let progressBar = $state<ProgressBar | undefined>(undefined)
function updateJobProgress(job: Job) {
const modules = job?.flow_status?.modules
@@ -40,8 +49,8 @@
newError = maxDone
maxDone = maxDone + 1
}
}
subIndexIsPercent = false;
}
subIndexIsPercent = false
// Loop is still iterating
if (module?.iterator) {
@@ -54,12 +63,12 @@
} else if (module?.branchall) {
subStepIndex = module.branchall.branch
subStepLength = module.branchall.len
} else if (module?.progress) {
} else if (module?.progress) {
const clamp = (num, min, max) => Math.min(Math.max(num, min), max)
subStepIndex = clamp(module?.progress, subIndex ?? 0, 99)
// Jitter protection >^^^^^^^^
subStepLength = 100
subIndexIsPercent = true;
subIndexIsPercent = true
currentSubJobProgress = subStepIndex
} else {
currentSubJobProgress = undefined
@@ -73,26 +82,27 @@
nextInProgress = newNextInProgress
}
let resetP: any
export function reset() {
resetP?.()
progressBar?.resetP()
error = undefined
subIndex = undefined
subLength = undefined
length = 1
index = 0
}
$effect(() => {
job && updateJobProgress(job)
})
</script>
<ProgressBar
bind:resetP
bind:this={progressBar}
{length}
{index}
{nextInProgress}
{subLength}
{subIndex}
{error}
bind:subIndexIsPercent
class={$$props.class}
{subIndexIsPercent}
class={className}
/>
frontend/src/lib/components/flows/content/FlowModuleComponent.svelte
+27 -10
View File
@@ -11,7 +11,7 @@
import Toggle from '$lib/components/Toggle.svelte'
import { createScriptFromInlineScript, fork } from '$lib/components/flows/flowStateUtils.svelte'
import type { FlowModule, RawScript, ScriptLang } from '$lib/gen'
import type { FlowModule, FlowModuleValue, RawScript, ScriptLang } from '$lib/gen'
import FlowCard from '../common/FlowCard.svelte'
import FlowModuleHeader from './FlowModuleHeader.svelte'
import { getLatestHashForScript, scriptLangToEditorLang } from '$lib/scripts'
@@ -56,6 +56,7 @@
import AssetsDropdownButton from '$lib/components/assets/AssetsDropdownButton.svelte'
import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent'
import { editor as meditor } from 'monaco-editor'
import { DynamicInput } from '$lib/utils'
const {
selectedId,
@@ -167,7 +168,6 @@
reloadError = undefined
try {
const { input_transforms, schema } = await loadSchemaFromModule(flowModule)
console.log('reload', schema)
validCode = true
if (inputTransformSchemaForm) {
@@ -308,6 +308,30 @@
let modulePreviewResultViewer: ModulePreviewResultViewer | undefined = $state(undefined)
function retrieveDynCodeAndLang(value: FlowModuleValue): DynamicInput.HelperScript | undefined {
let helperScript: DynamicInput.HelperScript | undefined
switch (value.type) {
case 'script':
helperScript = {
source: 'deployed',
path: value.path,
runnable_kind: 'script'
}
break
case 'rawscript':
helperScript = {
source: 'inline',
code: value.content,
lang: value.language
}
break
default:
helperScript = undefined
}
return helperScript
}
function onJobDone() {
modulePreviewResultViewer?.getOutputPickerInner()?.setJobPreview()
}
@@ -570,14 +594,7 @@
}
extraLib={stepPropPicker.extraLib}
{enableAi}
helperScript={(flowModule?.value as RawScript).content &&
(flowModule?.value as RawScript).language
? {
source: 'inline',
code: (flowModule?.value as RawScript).content ?? '',
lang: (flowModule?.value as RawScript).language ?? 'deno'
}
: undefined}
helperScript={retrieveDynCodeAndLang(flowModule.value)}
/>
</PropPickerWrapper>
</div>
frontend/src/lib/components/flows/flowInfers.ts
+8 -1
View File
@@ -77,6 +77,13 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{
type: 'string',
description: 'The system prompt to give as input to the AI agent.'
},
streaming: {
type: 'boolean',
description:
'Whether to stream the output of the AI agent (only used if output_type is text).',
default: false,
showExpr: "fields.output_type === 'text'"
},
user_images: {
type: 'array',
description:
@@ -99,7 +106,7 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{
output_schema: {
type: 'object',
description:
'JSON schema that the AI agent will follow for its response format (only used if output_type is text)',
'JSON schema that the AI agent will follow for its response format (only used if output_type is text).',
format: 'json-schema',
showExpr: "fields.output_type === 'text'"
}
frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte
+30 -15
View File
@@ -4,28 +4,43 @@
import type { Writable } from 'svelte/store'
import { twMerge } from 'tailwind-merge'
export let sourceX: number
export let sourceY: number
export let sourcePosition: Position
export let targetX: number
export let targetY: number
export let targetPosition: Position
export let markerEnd: string | undefined = undefined
export let data: { class?: string } = {}
interface Props {
sourceX: number
sourceY: number
sourcePosition: Position
targetX: number
targetY: number
targetPosition: Position
markerEnd?: string | undefined
data?: { class?: string }
}
const { useDataflow } = getContext<{
useDataflow: Writable<boolean | undefined>
}>('FlowGraphContext')
$: [edgePath] = getBezierPath({
let {
sourceX,
sourceY,
sourcePosition,
targetX,
targetY,
targetPosition,
curvature: 0.25
})
markerEnd = undefined,
data = {}
}: Props = $props()
const { useDataflow } = getContext<{
useDataflow: Writable<boolean | undefined>
}>('FlowGraphContext')
let [edgePath] = $derived(
getBezierPath({
sourceX,
sourceY,
sourcePosition,
targetX,
targetY,
targetPosition,
curvature: 0.25
})
)
</script>
<BaseEdge
frontend/src/lib/components/graph/renderers/nodes/AIToolNode.svelte
+2 -2
View File
@@ -183,8 +183,9 @@
}
}
const sortedNewNodes = clone(nodes)
const sortedNewNodes = nodes
.filter((n) => n.type !== 'asset')
.map((n) => ({ id: n.id, position: $state.snapshot(n.position) }))
.sort((a, b) => a.position.y - b.position.y)
let currentYOffset = 0
let prevYPos = NaN
@@ -234,7 +235,6 @@
import { MessageCircle, Play, Wrench, X } from 'lucide-svelte'
import { twMerge } from 'tailwind-merge'
import { getContext } from 'svelte'
import { clone } from '$lib/utils'
import type { Edge, Node } from '@xyflow/svelte'
import type { Writable } from 'svelte/store'

Some files were not shown because too many files have changed in this diff Show More