diff --git a/CHANGELOG.md b/CHANGELOG.md index 10cabf9519..d0418392da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Changelog +## [1.549.1](https://github.com/windmill-labs/windmill/compare/v1.549.0...v1.549.1) (2025-09-26) + + +### Bug Fixes + +* do not request unecessarily get_scheduled_for ([0269211](https://github.com/windmill-labs/windmill/commit/02692111a1a8eefb2675b14d53f109a66c1b9a78)) +* fix agent_workers completed job back-compatibility deserialization ([db4bc7e](https://github.com/windmill-labs/windmill/commit/db4bc7ee6963955abc7e290bd67ea913b0f5e2ad)) + +## [1.549.0](https://github.com/windmill-labs/windmill/compare/v1.548.3...v1.549.0) (2025-09-26) + + +### Features + +* **backend:** job result stream optimization ([#6673](https://github.com/windmill-labs/windmill/issues/6673)) ([8f4fef9](https://github.com/windmill-labs/windmill/commit/8f4fef98042c49346c89bdf5e0b9b1f2d52e371f)) + + +### Bug Fixes + +* scim group handling when deleting instance user + conversion ([#6677](https://github.com/windmill-labs/windmill/issues/6677)) ([4205e83](https://github.com/windmill-labs/windmill/commit/4205e83cfde453827eab23c31e76a0f0490d31b7)) + +## [1.548.3](https://github.com/windmill-labs/windmill/compare/v1.548.2...v1.548.3) (2025-09-25) + + +### Bug Fixes + +* fix job loader token initialization ([f5d238e](https://github.com/windmill-labs/windmill/commit/f5d238edcfed6b0f066d459cdc718679a7b51187)) +* websocket runnable [#6675](https://github.com/windmill-labs/windmill/issues/6675) ([a308782](https://github.com/windmill-labs/windmill/commit/a308782bcf7ef9913887521d74796b490619d0c8)) + +## [1.548.2](https://github.com/windmill-labs/windmill/compare/v1.548.1...v1.548.2) (2025-09-24) + + +### Bug Fixes + +* **ui:** workers button on navbar require a single click only ([afa8104](https://github.com/windmill-labs/windmill/commit/afa8104cb0c1a8f1a6fe124a6e01c1d32f049afa)) + ## [1.548.1](https://github.com/windmill-labs/windmill/compare/v1.548.0...v1.548.1) (2025-09-24) diff --git a/backend/.gitignore b/backend/.gitignore index f96e7e106d..6645e22618 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -9,4 +9,5 @@ windmill-api/openapi-*.* .duckdb/* *ee.rs generate_mcp_endpoints_tools/venv -bacon.toml \ No newline at end of file +bacon.toml +libwindmill_duckdb_ffi_internal.so \ No newline at end of file diff --git a/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json b/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json new file mode 100644 index 0000000000..6b8589e8e6 --- /dev/null +++ b/backend/.sqlx/query-1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n eig.igroup as group_name,\n ws.auto_add_instance_groups_roles\n FROM email_to_igroup eig\n INNER JOIN workspace_settings ws ON ws.workspace_id = $1\n WHERE eig.email = $2\n AND eig.igroup = ANY(ws.auto_add_instance_groups)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "group_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "auto_add_instance_groups_roles", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + true + ] + }, + "hash": "1ff185d8b8b897a72180cd0002e0f7e9858eef249577ce23969522793c6b5608" +} diff --git a/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json b/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json new file mode 100644 index 0000000000..fb2ab7fcf5 --- /dev/null +++ b/backend/.sqlx/query-25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "25b7c964336321fa10ea988831526b391cc1f02185ee87dbbda3d8a388cc858a" +} diff --git a/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json b/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json new file mode 100644 index 0000000000..b7645b6a44 --- /dev/null +++ b/backend/.sqlx/query-326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "326fd614ebd965b9bb6f3e578f75a54d80812ff144e711100e6ac659785c991d" +} diff --git a/backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json b/backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json deleted file mode 100644 index e7d66a3d75..0000000000 --- a/backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n SUBSTR(rs.stream, $8) AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "running", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "logs", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "new_result_stream", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "mem_peak", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "flow_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "stream_job", - "type_info": "Uuid" - }, - { - "ordinal": 7, - "name": "workflow_as_code_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 8, - "name": "log_offset", - "type_info": "Int4" - }, - { - "ordinal": 9, - "name": "stream_offset", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "progress", - "type_info": "Int4" - }, - { - "ordinal": 12, - "name": "result_stream: Option", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Int4", - "Text", - "Uuid", - "Bool", - "Bool", - "TextArray", - "Bool", - "Int4" - ] - }, - "nullable": [ - null, - null, - null, - null, - null, - null, - null, - null, - null, - null, - false, - null, - false - ] - }, - "hash": "40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773" -} diff --git a/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json b/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json new file mode 100644 index 0000000000..da7115eb81 --- /dev/null +++ b/backend/.sqlx/query-4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "Bool" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "4b056d33215b3a1e9849bb66ce84e96c69d10e3970e38151c97d1fca0cb7388d" +} diff --git a/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json b/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json deleted file mode 100644 index 8c9de521d2..0000000000 --- a/backend/.sqlx/query-4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO job_result_stream (workspace_id, job_id, stream)\n VALUES ($1, $2, $3)\n ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4bc533074c720820cebff8d97a203df52520b7606378ecca267e88383a45b49b" -} diff --git a/backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json b/backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json new file mode 100644 index 0000000000..859f2b23a4 --- /dev/null +++ b/backend/.sqlx/query-5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray" + ] + }, + "nullable": [] + }, + "hash": "5adeb6989648ca42431c96069c4de88d2615e7e6f1267f2fb12ccc325d4e4148" +} diff --git a/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json b/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json new file mode 100644 index 0000000000..59f972f50d --- /dev/null +++ b/backend/.sqlx/query-7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882.json @@ -0,0 +1,101 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $3 AND idx >= $8\n GROUP BY job_id\n )\n SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n rs.stream AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n rs.offset AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "completed", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "running", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "logs", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "new_result_stream", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "mem_peak", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 6, + "name": "stream_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, + "name": "workflow_as_code_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 8, + "name": "log_offset", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 10, + "name": "created_by!", + "type_info": "Varchar" + }, + { + "ordinal": 11, + "name": "progress", + "type_info": "Int4" + }, + { + "ordinal": 12, + "name": "result_stream: Option", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text", + "Uuid", + "Bool", + "Bool", + "TextArray", + "Bool", + "Int4" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + null, + null + ] + }, + "hash": "7b524ee24bb78e494a93c5ea205259bd989a9d769a6a300d37ab116960438882" +} diff --git a/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json b/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json deleted file mode 100644 index 1d56449d18..0000000000 --- a/backend/.sqlx/query-807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "807c920bff25f56b10e88900d879cf5e8484c147e457044d6b075323b163ebaa" -} diff --git a/backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json b/backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json deleted file mode 100644 index 7254b7e84f..0000000000 --- a/backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "stream_offset", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "flow_status: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "stream_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Int4", - "Bool" - ] - }, - "nullable": [ - null, - null, - null, - null, - null - ] - }, - "hash": "80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7" -} diff --git a/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json b/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json deleted file mode 100644 index abc63b0e90..0000000000 --- a/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT \n jc.result as \"result: sqlx::types::Json>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "running: Option", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "stream_offset", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "stream_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Int4", - "Bool" - ] - }, - "nullable": [ - true, - false, - false, - null, - null, - null - ] - }, - "hash": "8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34" -} diff --git a/backend/.sqlx/query-9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde.json b/backend/.sqlx/query-9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde.json new file mode 100644 index 0000000000..de16cf3146 --- /dev/null +++ b/backend/.sqlx/query-9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx)\n VALUES (\n $1, \n $2,\n $3, \n $4\n )\n ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Text", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "9bfc2a821b25641af48b0e3954163078922340294f1c6515400fb2c896666fde" +} diff --git a/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json b/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json new file mode 100644 index 0000000000..5740a1bda5 --- /dev/null +++ b/backend/.sqlx/query-9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $3\n GROUP BY job_id\n )\n SELECT \n jc.result as \"result: sqlx::types::Json>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "running: Option", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "Bool" + ] + }, + "nullable": [ + true, + false, + false, + null, + null, + null + ] + }, + "hash": "9c0bbd44902d8eee393236f7c2372b273d14a093bb29ec12dda8bbfaecd49a35" +} diff --git a/backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json b/backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json similarity index 72% rename from backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json rename to backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json index 9d2440990e..96bc0d74f4 100644 --- a/backend/.sqlx/query-2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1.json +++ b/backend/.sqlx/query-9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o \n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) \n AND workspace_id = $1 and archived = false", + "query": "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o\n WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false)\n AND workspace_id = $1 and archived = false", "describe": { "columns": [ { @@ -42,5 +42,5 @@ true ] }, - "hash": "2d5f58dd2aff3bd49f3891ae76df23e2aa39891931516426f65b229314a0cee1" + "hash": "9c3ddb90295db7d6afcbdb077f017950620e753dde97c8d2d88cd60ff8c3f339" } diff --git a/backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json b/backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json deleted file mode 100644 index 4cca74dd4b..0000000000 --- a/backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "running: Option", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "stream_offset", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "stream_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Bool" - ] - }, - "nullable": [ - null, - false, - null, - null, - null - ] - }, - "hash": "a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05" -} diff --git a/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json b/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json new file mode 100644 index 0000000000..ddf7d06e68 --- /dev/null +++ b/backend/.sqlx/query-a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49.json @@ -0,0 +1,49 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH result_stream AS (\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n job_id, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $1 AND idx >= $3\n GROUP BY job_id\n )\n SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n rs.stream AS \"result_stream: Option\",\n rs.offset AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "running: Option", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Bool" + ] + }, + "nullable": [ + null, + false, + null, + null, + null + ] + }, + "hash": "a76eec5797ca8f97e63ed5542bf03873e7dfd1cf9fe984c769afb5a8bdb48d49" +} diff --git a/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json b/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json new file mode 100644 index 0000000000..d9ca7cfa52 --- /dev/null +++ b/backend/.sqlx/query-bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM job_result_stream_v2 WHERE job_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "bb46e5dcf5490ef3511faa131ad5693dedf34366e51044ddf30695995d194090" +} diff --git a/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json b/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json new file mode 100644 index 0000000000..d30b0bae62 --- /dev/null +++ b/backend/.sqlx/query-dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n string_agg(stream, '' order by idx asc) as stream, \n max(idx) + 1 as offset \n FROM job_result_stream_v2\n WHERE job_id = $2 AND idx >= $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "stream", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "offset", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "dd92bce0ddbfbf06834807aab4d589fc104647cae7abf37ddce3ef7109726261" +} diff --git a/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json b/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json new file mode 100644 index 0000000000..6b2a36ec10 --- /dev/null +++ b/backend/.sqlx/query-eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_admin", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "operator", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "added_via", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "eb110e722ba8ac32d9d69010dc7e5f5763c55a5ac255f939f1fda8d3f9200f8d" +} diff --git a/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json b/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json deleted file mode 100644 index 5afdd9726a..0000000000 --- a/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "new_result_stream", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "stream_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int4", - "Uuid" - ] - }, - "nullable": [ - null, - null - ] - }, - "hash": "f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a" -} diff --git a/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json b/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json new file mode 100644 index 0000000000..70fbb752eb --- /dev/null +++ b/backend/.sqlx/query-f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "f8ac5379ecfbff7b8ae75c821680737b249a64c8d9e8f7dbcc46fce98e874571" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5b8b1a7b5b..e9826712d7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] @@ -620,7 +620,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tokio-util", "tokio-websockets", "tracing", @@ -1245,9 +1245,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.75" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", "cfg-if", @@ -1255,7 +1255,7 @@ dependencies = [ "miniz_oxide 0.8.9", "object", "rustc-demangle", - "windows-targets 0.52.6", + "windows-link 0.2.0", ] [[package]] @@ -1935,9 +1935,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.38" +version = "1.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" +checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f" dependencies = [ "find-msvc-tools", "jobserver", @@ -3625,7 +3625,7 @@ dependencies = [ "serde_json", "thiserror 2.0.16", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tokio-socks", "tokio-util", "tower 0.5.2", @@ -4673,7 +4673,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -5065,7 +5065,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -5893,9 +5893,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "git-version" @@ -6606,7 +6606,7 @@ dependencies = [ "pin-project-lite", "rustls-native-certs 0.7.3", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tower-service", ] @@ -6655,7 +6655,7 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tower-service", "webpki-roots 1.0.2", ] @@ -6764,7 +6764,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.0", + "windows-core 0.62.1", ] [[package]] @@ -7888,7 +7888,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "webpki-roots 0.26.11", ] @@ -8027,9 +8027,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" @@ -8887,9 +8887,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] @@ -9794,9 +9794,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4" dependencies = [ "base64 0.22.1", "byteorder", @@ -9831,7 +9831,7 @@ dependencies = [ "bytes", "chrono", "fallible-iterator 0.2.0", - "postgres-protocol 0.6.8", + "postgres-protocol 0.6.9", "serde", "serde_json", "uuid", @@ -10569,9 +10569,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.2" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -10581,9 +10581,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -10696,7 +10696,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tokio-util", "tower 0.5.2", "tower-http", @@ -11077,7 +11077,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.11.0", - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -11406,7 +11406,7 @@ version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -13296,7 +13296,7 @@ dependencies = [ "getrandom 0.3.3", "once_cell", "rustix 1.1.2", - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -13687,7 +13687,7 @@ dependencies = [ "percent-encoding", "phf 0.11.3", "pin-project-lite", - "postgres-protocol 0.6.8", + "postgres-protocol 0.6.9", "postgres-types 0.2.9", "rand 0.9.0", "socket2 0.5.10", @@ -13729,9 +13729,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls 0.23.29", "tokio", @@ -13823,7 +13823,7 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tokio-util", ] @@ -13917,7 +13917,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.3", + "tokio-rustls 0.26.4", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -15108,7 +15108,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.0", + "windows-sys 0.61.1", ] [[package]] @@ -15119,7 +15119,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15179,7 +15179,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "argon2", @@ -15299,7 +15299,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.548.1" +version = "1.549.1" dependencies = [ "base64 0.22.1", "chrono", @@ -15314,7 +15314,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.548.1" +version = "1.549.1" dependencies = [ "chrono", "serde", @@ -15327,7 +15327,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "axum", @@ -15346,7 +15346,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "async-recursion", @@ -15430,7 +15430,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.548.1" +version = "1.549.1" dependencies = [ "regex", "serde", @@ -15445,7 +15445,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "bytes", @@ -15469,7 +15469,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.548.1" +version = "1.549.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15481,7 +15481,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.548.1" +version = "1.549.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15490,7 +15490,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "lazy_static", @@ -15502,7 +15502,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "serde_json", @@ -15514,7 +15514,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "gosyn", @@ -15526,7 +15526,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "lazy_static", @@ -15538,7 +15538,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "serde_json", @@ -15550,7 +15550,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "nu-parser", @@ -15561,7 +15561,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15572,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15584,7 +15584,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "async-recursion", @@ -15607,7 +15607,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "lazy_static", @@ -15621,7 +15621,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15638,7 +15638,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "lazy_static", @@ -15652,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "lazy_static", @@ -15670,7 +15670,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15695,7 +15695,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "serde_json", @@ -15705,7 +15705,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "async-recursion", @@ -15738,7 +15738,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.548.1" +version = "1.549.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15748,7 +15748,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.548.1" +version = "1.549.1" dependencies = [ "anyhow", "async-once-cell", @@ -15919,8 +15919,8 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", + "windows-implement 0.60.1", + "windows-interface 0.59.2", "windows-link 0.1.3", "windows-result 0.3.4", "windows-strings 0.4.2", @@ -15928,12 +15928,12 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe7168f7de578d2d8a05b07fd61870d2e73b4020e9f49aa00da8471723497c" +checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9" dependencies = [ - "windows-implement 0.60.0", - "windows-interface 0.59.1", + "windows-implement 0.60.1", + "windows-interface 0.59.2", "windows-link 0.2.0", "windows-result 0.4.0", "windows-strings 0.5.0", @@ -15974,9 +15974,9 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "edb307e42a74fb6de9bf3a02d9712678b22399c87e6fa869d6dfcd8c1b7754e0" dependencies = [ "proc-macro2", "quote", @@ -16007,9 +16007,9 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "c0abd1ddbc6964ac14db11c7213d6532ef34bd9aa042c2e5935f59d7908b46a5" dependencies = [ "proc-macro2", "quote", @@ -16127,14 +16127,14 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.3", + "windows-targets 0.53.4", ] [[package]] name = "windows-sys" -version = "0.61.0" +version = "0.61.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f" dependencies = [ "windows-link 0.2.0", ] @@ -16172,11 +16172,11 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" +version = "0.53.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" +checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b" dependencies = [ - "windows-link 0.1.3", + "windows-link 0.2.0", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 173ce528f7..dd64e939bc 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.548.1" +version = "1.549.1" authors.workspace = true edition.workspace = true @@ -34,7 +34,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.548.1" +version = "1.549.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b6e5157568..d00837d201 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -aa4c7936b99d0de2f7af31cde79dfc7b1b482c29 \ No newline at end of file +4011644daea63877e43ebc20cbe6704f381b4b42 diff --git a/backend/migrations/20250924150953_new_job_result_stream_table.down.sql b/backend/migrations/20250924150953_new_job_result_stream_table.down.sql new file mode 100644 index 0000000000..5d3cb6c6fe --- /dev/null +++ b/backend/migrations/20250924150953_new_job_result_stream_table.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TABLE job_result_stream_v2; + +ALTER TABLE job_result_stream ADD CONSTRAINT fk_job_result_stream_job_id FOREIGN KEY (job_id) REFERENCES v2_job_queue(id) ON DELETE CASCADE; \ No newline at end of file diff --git a/backend/migrations/20250924150953_new_job_result_stream_table.up.sql b/backend/migrations/20250924150953_new_job_result_stream_table.up.sql new file mode 100644 index 0000000000..c5230e0645 --- /dev/null +++ b/backend/migrations/20250924150953_new_job_result_stream_table.up.sql @@ -0,0 +1,13 @@ +-- Add up migration script here +CREATE TABLE job_result_stream_v2 ( + job_id UUID NOT NULL, + workspace_id TEXT NOT NULL, + stream TEXT NOT NULL, + idx INT NOT NULL, + PRIMARY KEY (job_id, idx) +); + +GRANT ALL ON TABLE job_result_stream_v2 TO windmill_admin; +GRANT ALL ON TABLE job_result_stream_v2 TO windmill_user; + +ALTER TABLE job_result_stream DROP CONSTRAINT fk_job_result_stream_job_id; \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index a0900da40e..c292c085d1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -950,6 +950,17 @@ pub async fn delete_expired_items(db: &DB) -> () { { tracing::error!("Error deleting job: {:?}", e); } + + // should already be deleted but just in case + if let Err(e) = sqlx::query!( + "DELETE FROM job_result_stream_v2 WHERE job_id = ANY($1)", + &deleted_jobs + ) + .execute(&mut *tx) + .await + { + tracing::error!("Error deleting job result stream: {:?}", e); + } } } Err(e) => { @@ -1654,7 +1665,7 @@ pub async fn monitor_db( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream, job_stats, job_logs, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, concurrency_key, log_file, metrics") .execute(db) .await?; Ok(()) diff --git a/backend/update_sqlx.sh b/backend/update_sqlx.sh index 77b772c970..8ab84c6306 100755 --- a/backend/update_sqlx.sh +++ b/backend/update_sqlx.sh @@ -21,9 +21,17 @@ if [[ "$(uname)" == "Darwin" ]]; then sed -i '' 's/^samael = { version="0.0.14", features = \["xmlsec"\] }/#samael = { version="0.0.14", features = ["xmlsec"] }/' Cargo.toml # Uncomment the git-based samael dependency sed -i '' 's/^# \(samael = { git="https:\/\/github.com\/njaremko\/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = \["xmlsec"\] }\)/\1/' Cargo.toml + + # Run cargo sqlx prepare with deno_core_mac + echo "Running cargo sqlx prepare with deno_core_mac..." + cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private,deno_core_mac +else + # Run cargo sqlx prepare + echo "Running cargo sqlx prepare..." + cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private fi -cargo sqlx prepare --workspace -- --all-targets --features all_sqlx_features,private + # Undo the samael changes on macOS if [[ "$(uname)" == "Darwin" ]]; then diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 263a07785e..6ee5a9651c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.548.1 + version: 1.549.1 title: Windmill API contact: @@ -575,6 +575,27 @@ paths: text/plain: schema: type: string + /w/{workspace}/users/convert_to_group/{username}: + post: + summary: convert manual user to group user (require admin privilege) + operationId: convertUserToGroup + tags: + - user + - admin + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: username + in: path + required: true + schema: + type: string + responses: + "200": + description: convert user to group user + content: + text/plain: + schema: + type: string /github_app/connected_repositories: get: @@ -15353,6 +15374,8 @@ components: AuditLog: type: object properties: + workspace_id: + type: string id: type: integer timestamp: @@ -15458,6 +15481,7 @@ components: span: type: string required: + - workspace_id - id - timestamp - username diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 49a143fb98..d6187db436 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -937,7 +937,7 @@ async fn update_flow( clear_schedule(&mut tx, &schedule.path, &w_id).await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 596e404629..e370896b52 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -175,6 +175,7 @@ pub fn workspaced_service() -> Router { .route( "/run_and_stream/f/*script_path", get(stream_flow_by_path) + .post(stream_flow_by_path) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), @@ -182,6 +183,7 @@ pub fn workspaced_service() -> Router { .route( "/run_and_stream/p/*script_path", get(stream_script_by_path) + .post(stream_script_by_path) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), @@ -189,6 +191,7 @@ pub fn workspaced_service() -> Router { .route( "/run_and_stream/h/:hash", get(stream_script_by_hash) + .post(stream_script_by_hash) .head(|| async { "" }) .layer(cors.clone()) .layer(ce_headers.clone()), @@ -3552,7 +3555,7 @@ pub enum DynamicSelectRunnableRef { #[serde(rename = "deployed")] Deployed { path: String, runnable_kind: RunnableKind }, #[serde(rename = "inline")] - Inline { code: String, language: Option }, + Inline { code: String, lang: Option }, } pub struct QueryOrBody(pub Option); @@ -5220,6 +5223,7 @@ pub async fn stream_flow_by_path( Extension(user_db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, + method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { stream_job( @@ -5230,6 +5234,7 @@ pub async fn stream_flow_by_path( RunnableId::from_flow_path(flow_path.to_path()), args, run_query, + method == http::Method::GET, ) .await } @@ -5240,6 +5245,7 @@ pub async fn stream_script_by_path( Extension(user_db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, + method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { stream_job( @@ -5250,6 +5256,7 @@ pub async fn stream_script_by_path( RunnableId::from_script_path(script_path.to_path()), args, run_query, + method == http::Method::GET, ) .await } @@ -5260,6 +5267,7 @@ pub async fn stream_script_by_hash( Extension(user_db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, + method: hyper::http::Method, args: RawWebhookArgs, ) -> error::Result { stream_job( @@ -5270,6 +5278,7 @@ pub async fn stream_script_by_hash( RunnableId::from_script_hash(script_hash), args, run_query, + method == http::Method::GET, ) .await } @@ -5282,24 +5291,39 @@ pub async fn stream_job( runnable_id: RunnableId, args: RawWebhookArgs, run_query: RunJobQuery, + is_get: bool, ) -> error::Result { - let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { - x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))) - }); + let args = if is_get { + let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| { + Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) + }) + }); - let payload_args = if let Some(payload) = payload_r { - payload? + let payload_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_args); + + let args = args + .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) + .await?; + args } else { - HashMap::new() + args.to_args_from_runnable( + &authed, + &db, + &w_id, + runnable_id.clone(), + run_query.skip_preprocessor, + ) + .await? }; - let mut args = args.process_args(&authed, &db, &w_id, None).await?; - args.body = args::Body::HashMap(payload_args); - - let args = args - .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) - .await?; - let poll_delay_ms = run_query.poll_delay_ms; let uuid = match runnable_id { RunnableId::ScriptId(ScriptId::ScriptPath(script_path)) @@ -6351,7 +6375,7 @@ async fn run_dynamic_select( dynamic_input = dynamic_input_res; } }, - DynamicSelectRunnableRef::Inline { code, language } => { + DynamicSelectRunnableRef::Inline { code, lang: language } => { dynamic_input = DynamicInput { x_windmill_dyn_select_code: code, x_windmill_dyn_select_lang: language.unwrap_or_default(), @@ -6678,7 +6702,7 @@ async fn get_job_update( &job_id, log_offset, stream_offset, - get_progress, + get_progress.unwrap_or(false), running, true, false, @@ -6782,6 +6806,7 @@ fn start_job_update_sse_stream( // Send initial update immediately let mut running = running; let mut mem_peak = 0; + match get_job_update_data( &opt_authed, &opt_tokened, @@ -6790,7 +6815,7 @@ fn start_job_update_sse_stream( &job_id, log_offset, stream_offset, - get_progress, + false, running, true, true, @@ -6847,11 +6872,12 @@ fn start_job_update_sse_stream( } } + let mut get_progress_m: bool = false; // Poll for updates every 1 second let mut i = 0; let start = Instant::now(); let mut last_ping = Instant::now(); - + let mut last_progress_check = Instant::now(); loop { i += 1; @@ -6894,6 +6920,10 @@ fn start_job_update_sse_stream( } tokio::time::sleep(std::time::Duration::from_millis(ms_duration)).await; + // Check progress if the user requested it, and check periodically if the job has progress + // Once it has progress, we always check progress + let check_progress = get_progress.unwrap_or(false) + && (get_progress_m || last_progress_check.elapsed().as_secs() > 5); match get_job_update_data( &opt_authed, &opt_tokened, @@ -6902,7 +6932,7 @@ fn start_job_update_sse_stream( &job_id, log_offset, stream_offset, - get_progress, + check_progress, running, false, true, @@ -6923,6 +6953,13 @@ fn start_job_update_sse_stream( if update.new_logs.as_ref().is_some_and(|x| x.is_empty()) { update.new_logs = None; } + if check_progress { + if update.progress.is_some() { + get_progress_m = true; + } else { + last_progress_check = Instant::now(); + } + } // if !only_result.unwrap_or(false) { // tracing::error!("update {:?}", update); @@ -6990,14 +7027,20 @@ async fn get_flow_stream_delta( ) -> error::Result, Option)>> { if let Some(job_id) = flow_stream_job_id { let record = sqlx::query!( - "SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2", + " + SELECT + string_agg(stream, '' order by idx asc) as stream, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $1 + ", stream_offset.unwrap_or(0), job_id, ) .fetch_optional(db) .await?; if let Some(record) = record { - Ok(Some((record.new_result_stream, record.stream_offset))) + Ok(Some((record.stream, record.offset))) } else { Ok(None) } @@ -7014,7 +7057,7 @@ async fn get_job_update_data( job_id: &Uuid, log_offset: Option, stream_offset: Option, - get_progress: Option, + get_progress: bool, running: Option, log_view: bool, get_full_job_on_completion: bool, @@ -7046,18 +7089,28 @@ async fn get_job_update_data( let (result, running, mut result_stream, mut new_stream_offset, new_flow_stream_job_id) = if let Some(tags) = tags { let r = sqlx::query!( - "SELECT + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) + SELECT jc.result as \"result: sqlx::types::Json>\", v2_job.tag, v2_job_queue.running as \"running: Option\", - SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) AS stream_offset, + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM v2_job LEFT JOIN v2_job_queue USING (id) LEFT JOIN v2_job_completed jc USING (id) LEFT JOIN v2_job_status js USING (id) - LEFT JOIN job_result_stream rs ON rs.job_id = $2 + LEFT JOIN result_stream rs ON rs.job_id = $2 WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", w_id, job_id, @@ -7086,11 +7139,21 @@ async fn get_job_update_data( } else { if running.is_some_and(|x| !x) { let r = sqlx::query!( - "SELECT + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $1 AND idx >= $3 + GROUP BY job_id + ) + SELECT COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\", jq.running as \"running: Option\", - SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( SELECT $1::uuid as job_id, $2::text as workspace_id @@ -7098,7 +7161,7 @@ async fn get_job_update_data( LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id LEFT JOIN v2_job_status js ON js.id = base.job_id - LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $1", job_id, w_id, @@ -7119,10 +7182,20 @@ async fn get_job_update_data( } } else { let q = sqlx::query!( - "SELECT + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) + SELECT COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\", - SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\", CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( @@ -7130,7 +7203,7 @@ async fn get_job_update_data( ) base LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id LEFT JOIN v2_job_status js ON js.id = base.job_id - LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $2", w_id, job_id, @@ -7180,20 +7253,30 @@ async fn get_job_update_data( }) } else { let mut record = sqlx::query!( - "SELECT + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $3 AND idx >= $8 + GROUP BY job_id + ) + SELECT c.id IS NOT NULL AS completed, CASE WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) ELSE false END AS running, CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs, - SUBSTR(rs.stream, $8) AS new_result_stream, + rs.stream AS new_result_stream, COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\", (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job, COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\", CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset, - CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + rs.offset AS stream_offset, created_by AS \"created_by!\", CASE WHEN $4::BOOLEAN THEN ( SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' @@ -7204,14 +7287,14 @@ async fn get_job_update_data( LEFT JOIN v2_job_runtime r USING (id) LEFT JOIN v2_job_status f USING (id) LEFT JOIN v2_job_completed c USING (id) - LEFT JOIN job_result_stream rs ON rs.job_id = $3 + LEFT JOIN result_stream rs ON rs.job_id = $3 LEFT JOIN job_logs ON job_logs.job_id = $3 WHERE j.workspace_id = $2 AND j.id = $3 AND ($6::text[] IS NULL OR j.tag = ANY($6))", log_offset, w_id, job_id, - get_progress.unwrap_or(false), + get_progress, running, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, no_logs.unwrap_or(false), diff --git a/backend/windmill-api/src/schedule.rs b/backend/windmill-api/src/schedule.rs index b5bb69f8ee..9eaa897341 100644 --- a/backend/windmill-api/src/schedule.rs +++ b/backend/windmill-api/src/schedule.rs @@ -277,7 +277,7 @@ async fn create_schedule( .await?; if ns.enabled.unwrap_or(true) { - tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into())).await? + tx = push_scheduled_job(&db, tx, &schedule, Some(&authed.clone().into()), None).await? } tx.commit().await?; @@ -419,7 +419,7 @@ async fn edit_schedule( .await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } tx.commit().await?; @@ -683,7 +683,7 @@ pub async fn set_enabled( .await?; if payload.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } tx.commit().await?; diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index c2a03f6bbb..c321f16b1c 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -900,7 +900,7 @@ async fn create_script_internal<'c>( clear_schedule(&mut tx, &schedule.path, &w_id).await?; if schedule.enabled { - tx = push_scheduled_job(&db, tx, &schedule, None).await?; + tx = push_scheduled_job(&db, tx, &schedule, None, None).await?; } } } else { diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-api/src/triggers/websocket/handler.rs index 827367e195..510f6e97ae 100644 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ b/backend/windmill-api/src/triggers/websocket/handler.rs @@ -12,6 +12,7 @@ use tokio_tungstenite::connect_async; use windmill_common::{ db::UserDB, error::{Error, Result}, + worker::to_raw_value, }; use windmill_git_sync::DeployedObject; @@ -236,7 +237,7 @@ impl TriggerCrud for WebsocketTrigger { url.starts_with("$flow:"), &db, authed.clone(), - config.url_runnable_args.as_ref(), + config.url_runnable_args.as_ref().map(to_raw_value).as_ref(), &workspace_id, ) .await?, diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs index 9a3f83a016..bfecaf479d 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -5,7 +5,7 @@ use crate::triggers::{ trigger_runnable, trigger_runnable_and_wait_for_raw_result, trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, }, - websocket::WebsocketConfig, + websocket::{get_url_from_runnable_value, WebsocketConfig}, Listener, }; use anyhow::Context; @@ -31,43 +31,6 @@ use windmill_common::{ use windmill_queue::PushArgsOwned; impl ListeningTrigger { - async fn get_url_from_runnable(&self, db: &DB) -> Result { - let runnable_kind = if self.is_flow { "Flow" } else { "Script" }; - tracing::info!( - "Running {} {} to get WebSocket URL", - runnable_kind.to_lowercase(), - self.path - ); - - let authed = self.authed(db, "ws").await?; - - let args = raw_value_to_args_hashmap( - self.trigger_config.url_runnable_args.as_ref().map(|r| &r.0), - )?; - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed, - &self.workspace_id, - &self.script_path, - self.is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - Error::BadConfig(format!( - "{} {} did not return a string", - runnable_kind, self.path, - )) - }) - } - async fn send_initial_messages( &self, writer: &mut SplitSink>, Message>, @@ -189,8 +152,12 @@ impl Listener for WebsocketTrigger { )) => { return Ok(None); }, - - url_result = listening_trigger.get_url_from_runnable(&db) => match url_result { + url_result = { + let authed = listening_trigger.authed(db, "ws").await?; + let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0); + let path = url.splitn(2, ':').nth(1).unwrap(); + get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id) + } => match url_result { Ok(url) => Cow::Owned(url), Err(err) => { return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into()); diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs index 86d3bd735b..7a4d98bcc1 100644 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -61,11 +61,11 @@ pub struct TestWebsocketConfig { } pub fn value_to_args_hashmap( - args: Option<&serde_json::Value>, + args: Option<&Box>, ) -> Result>> { let args = if let Some(args) = args { let args_map: Option> = - serde_json::from_value(args.clone()) + serde_json::from_str(args.get()) .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; args_map @@ -89,7 +89,7 @@ pub async fn get_url_from_runnable_value( is_flow: bool, db: &DB, authed: ApiAuthed, - args: Option<&serde_json::Value>, + args: Option<&Box>, workspace_id: &str, ) -> Result { tracing::info!( diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 949d563ac6..722c05ebce 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -71,6 +71,7 @@ pub fn workspaced_service() -> Router { .route("/get/:user", get(get_workspace_user)) .route("/update/:user", post(update_workspace_user)) .route("/delete/:user", delete(delete_workspace_user)) + .route("/convert_to_group/:user", post(convert_user_to_group)) .route("/is_owner/*path", get(is_owner_of_path)) .route("/whois/:username", get(whois)) .route("/whoami", get(whoami)) @@ -1362,6 +1363,138 @@ async fn update_workspace_user( Ok(format!("user {} updated", user_email)) } +async fn convert_user_to_group( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, username_to_convert)): Path<(String, String)>, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + let mut tx = db.begin().await?; + + // Get user email and current status + let user_info = sqlx::query!( + "SELECT email, is_admin, operator, added_via FROM usr WHERE username = $1 AND workspace_id = $2", + username_to_convert, + &w_id + ) + .fetch_optional(&mut *tx) + .await?; + + let user_info = not_found_if_none(user_info, "User", &username_to_convert)?; + + // Check if user is already a group user + if let Some(added_via) = &user_info.added_via { + if added_via.get("source").and_then(|v| v.as_str()) == Some("instance_group") { + return Err(Error::BadRequest("User is already a group user".to_string())); + } + } + + // Find which instance groups this user belongs to that are configured for auto-add in this workspace + let eligible_groups = sqlx::query!( + r#" + SELECT + eig.igroup as group_name, + ws.auto_add_instance_groups_roles + FROM email_to_igroup eig + INNER JOIN workspace_settings ws ON ws.workspace_id = $1 + WHERE eig.email = $2 + AND eig.igroup = ANY(ws.auto_add_instance_groups) + "#, + &w_id, + &user_info.email + ) + .fetch_all(&mut *tx) + .await?; + + if eligible_groups.is_empty() { + return Err(Error::BadRequest( + "User is not a member of any instance groups configured for auto-add in this workspace".to_string() + )); + } + + // Determine the group with highest precedence (same logic as process_instance_group_auto_adds) + let roles: std::collections::HashMap = if let Some(roles_json) = &eligible_groups[0].auto_add_instance_groups_roles { + serde_json::from_value(roles_json.clone()).unwrap_or_default() + } else { + std::collections::HashMap::new() + }; + + let mut best_group = &eligible_groups[0].group_name; + let mut best_precedence = 0u8; + + for group in &eligible_groups { + let default_role = "developer".to_string(); + let role = roles.get(&group.group_name).unwrap_or(&default_role); + + let precedence = match role.as_str() { + "admin" => 3, + "developer" => 2, + "operator" => 1, + _ => 2, + }; + + if precedence > best_precedence { + best_precedence = precedence; + best_group = &group.group_name; + } + } + + let primary_group_name = best_group; + + // Determine role from group configuration using the selected primary group + let default_role = "developer".to_string(); + let role = roles.get(primary_group_name).unwrap_or(&default_role).as_str(); + + let (is_admin, is_operator) = match role { + "admin" => (true, false), + "operator" => (false, true), + _ => (false, false), + }; + + // Update user with instance group information + let instance_group_source = serde_json::json!({ + "source": "instance_group", + "group": primary_group_name + }); + + sqlx::query!( + "UPDATE usr SET added_via = $1, is_admin = $2, operator = $3 WHERE username = $4 AND workspace_id = $5", + instance_group_source, + is_admin, + is_operator, + username_to_convert, + &w_id + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "users.convert_to_group", + ActionKind::Update, + &w_id, + Some(&username_to_convert), + Some([("group", primary_group_name.as_str()), ("role", role)].into()), + ) + .await?; + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::User { email: user_info.email.clone() }, + Some(format!("Converted user '{}' to group user (group: {}, role: {})", &user_info.email, primary_group_name, role)), + true, + ) + .await?; + + Ok(format!("User {} converted to group user (group: {}, role: {})", username_to_convert, primary_group_name, role)) +} + async fn update_user( authed: ApiAuthed, Path(email_to_update): Path, @@ -1459,6 +1592,12 @@ async fn delete_user( .execute(&mut *tx) .await?; } + + // Remove user from all instance groups email_to_igroup + sqlx::query!("DELETE FROM email_to_igroup WHERE email = $1", &email_to_delete) + .execute(&mut *tx) + .await?; + audit_log( &mut *tx, &authed, @@ -2191,8 +2330,8 @@ async fn get_all_runnables( .collect::>(), ); let scripts = sqlx::query!( - "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o - WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) + "SELECT workspace_id as workspace, path, summary, description, schema FROM script as o + WHERE created_at = (select max(created_at) from script where o.path = path and workspace_id = $1 AND archived = false) AND workspace_id = $1 and archived = false", workspace ) .fetch_all(&mut *tx) diff --git a/backend/windmill-common/src/result_stream.rs b/backend/windmill-common/src/result_stream.rs index d44933891d..02bfa75b92 100644 --- a/backend/windmill-common/src/result_stream.rs +++ b/backend/windmill-common/src/result_stream.rs @@ -19,17 +19,24 @@ pub async fn append_result_stream_db( workspace_id: &str, job_id: &Uuid, nstream: &str, + offset: i32, ) -> error::Result<()> { if !nstream.is_empty() { sqlx::query!( r#" - INSERT INTO job_result_stream (workspace_id, job_id, stream) - VALUES ($1, $2, $3) - ON CONFLICT (job_id) DO UPDATE SET stream = job_result_stream.stream || $3 + INSERT INTO job_result_stream_v2 (workspace_id, job_id, stream, idx) + VALUES ( + $1, + $2, + $3, + $4 + ) + ON CONFLICT (job_id, idx) DO UPDATE SET stream = job_result_stream_v2.stream || $3 "#, workspace_id, job_id, nstream, + offset ) .execute(db) .await?; diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index e2c918c3fe..cb5c43a018 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -16,6 +16,17 @@ pub struct Arg { pub json_value: serde_json::Value, } +// Freeing from the caller side crashes the runtime with jemalloc enabled (EXIT CODE 11 SEGFAULT) +#[unsafe(no_mangle)] +pub extern "C" fn free_cstr(string: *mut c_char) -> () { + if string.is_null() { + return; + } + unsafe { + let _ = CString::from_raw(string); + } +} + #[unsafe(no_mangle)] pub extern "C" fn run_duckdb_ffi( query_block_list: *const *const c_char, @@ -289,11 +300,11 @@ fn row_to_value(row: &Row<'_>, column_names: &[String]) -> Result, duckdb::types::Value::UBigInt(u) => serde_json::Value::Number(u.into()), duckdb::types::Value::Float(f) => serde_json::Value::Number( serde_json::Number::from_f64(f as f64) - .ok_or_else(|| ("Could not convert to f64".to_string()))?, + .ok_or_else(|| "Could not convert to f64".to_string())?, ), duckdb::types::Value::Double(f) => serde_json::Value::Number( serde_json::Number::from_f64(f) - .ok_or_else(|| ("Could not convert to f64".to_string()))?, + .ok_or_else(|| "Could not convert to f64".to_string())?, ), duckdb::types::Value::Decimal(d) => serde_json::Value::String(d.to_string()), duckdb::types::Value::Timestamp(_, ts) => serde_json::Value::String(ts.to_string()), @@ -401,7 +412,7 @@ fn json_value_to_duckdb_value( "double" | "float8" => duckdb::types::Value::Double(v), "decimal" | "numeric" => duckdb::types::Value::Decimal( Decimal::from_f64(v) - .ok_or_else(|| ("Could not convert f64 to Decimal".to_string()))?, + .ok_or_else(|| "Could not convert f64 to Decimal".to_string())?, ), _ => duckdb::types::Value::Double(v), // default fallback } @@ -436,7 +447,7 @@ fn string_to_duckdb_timestamp(s: &str) -> Result { fn string_to_duckdb_date(s: &str) -> Result { use chrono::Datelike; let date = chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d") - .map_err(|e| (format!("Invalid date format: {}", e)))?; + .map_err(|e| format!("Invalid date format: {}", e))?; Ok(duckdb::types::Value::Date32(date.num_days_from_ce())) } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 7c6a3b3dcd..40b4fea108 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -147,6 +147,7 @@ pub struct JobCompleted { pub token: String, pub canceled_by: Option, pub duration: Option, + pub has_stream: Option, } pub async fn cancel_single_job<'c>( @@ -737,6 +738,7 @@ pub async fn add_completed_job_error( canceled_by, flow_is_done, duration, + false, ) .await?; Ok(result) @@ -758,6 +760,7 @@ pub async fn add_completed_job( canceled_by: Option, flow_is_done: bool, duration: Option, + has_stream: bool, ) -> Result<(Uuid, i64), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -782,6 +785,7 @@ pub async fn add_completed_job( &canceled_by, flow_is_done, duration, + has_stream, ) }) .retry( @@ -790,7 +794,11 @@ pub async fn add_completed_job( .with_max_times(5) .build(), ) - .when(|err| !matches!(err, Error::QuotaExceeded(_)) && !matches!(err, Error::ResultTooLarge(_))) + .when(|err| { + !matches!(err, Error::QuotaExceeded(_)) + && !matches!(err, Error::ResultTooLarge(_)) + && !matches!(err, Error::AlreadyCompleted(_)) + }) .notify(|err, dur| { tracing::error!("Could not insert completed job, retrying in {dur:#?}, err: {err:#?}"); }) @@ -834,6 +842,7 @@ async fn commit_completed_job( canceled_by: &Option, flow_is_done: bool, duration: Option, + has_stream: bool, ) -> windmill_common::error::Result<(Option, i64, bool)> { // let start = std::time::Instant::now(); @@ -855,7 +864,7 @@ async fn commit_completed_job( return value; } - let _duration = sqlx::query_scalar!( + let duration = sqlx::query_scalar!( "INSERT INTO v2_job_completed AS cj ( workspace_id , id @@ -891,10 +900,33 @@ async fn commit_completed_job( /* $9 */ duration, /* $10 */ result_columns as Option<&Vec>, ) - .fetch_one(&mut *tx) + .fetch_optional(&mut *tx) .await .map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))?; + let duration = if let Some(duration) = duration { + duration + } else { + let already_inserted = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)", + job_id + ) + .fetch_one(&mut *tx) + .await + .map_err(|e| Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")))? + .unwrap_or(false); + + if already_inserted { + return Err(Error::AlreadyCompleted(format!( + "The queued job {job_id} is already completed." + ))); + } else { + return Err(Error::AlreadyCompleted(format!( + "There is no queued job anymore for {job_id} but there is no completed job either." + ))); + } + }; + if let Some(labels) = result.wm_labels() { sqlx::query!( "UPDATE v2_job SET labels = ( @@ -924,7 +956,7 @@ async fn commit_completed_job( ) WHERE id = $3", &queued_job.id.to_string(), - _duration, + duration, parent_job ) .execute(&mut *tx) @@ -1116,6 +1148,12 @@ async fn commit_completed_job( .execute(&mut *tx) .await?; + if !success || has_stream { + sqlx::query!("DELETE FROM job_result_stream_v2 WHERE job_id = $1", job_id) + .execute(&mut *tx) + .await?; + } + tx.commit().await?; tracing::info!( @@ -1124,7 +1162,7 @@ async fn commit_completed_job( path = &queued_job.runnable_path(), job_kind = ?queued_job.kind, started_at = ?queued_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()), - duration = ?_duration, + duration = ?duration, permissioned_as = ?queued_job.permissioned_as, email = ?queued_job.permissioned_as_email, created_by = queued_job.created_by, @@ -1137,7 +1175,7 @@ async fn commit_completed_job( queued_job.id ); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); - Ok((None, _duration, _skip_downstream_error_handlers)) + Ok((None, duration, _skip_downstream_error_handlers)) } async fn check_result_size( @@ -1655,7 +1693,7 @@ pub async fn handle_maybe_scheduled_job<'c>( let push_next_job_future = (|| { tokio::time::timeout(std::time::Duration::from_secs(5), async { let mut tx = db.begin().await?; - tx = push_scheduled_job(db, tx, &schedule, None).await?; + tx = push_scheduled_job(db, tx, &schedule, None, Some(job.scheduled_for)).await?; tx.commit().await?; Ok::<(), Error>(()) }) diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 4fcd22771a..6594837ffa 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -9,6 +9,8 @@ use crate::push; use crate::PushIsolationLevel; use anyhow::Context; +use chrono::DateTime; +use chrono::Utc; use sqlx::{PgExecutor, Postgres, Transaction}; use std::collections::HashMap; use std::str::FromStr; @@ -34,6 +36,7 @@ pub async fn push_scheduled_job<'c>( mut tx: Transaction<'c, Postgres>, schedule: &Schedule, authed: Option<&Authed>, + now_cutoff: Option>, ) -> Result> { if !*LICENSE_KEY_VALID.read().await { return Err(error::Error::BadRequest( @@ -50,6 +53,19 @@ pub async fn push_scheduled_job<'c>( let now = now_from_db(&mut *tx).await?; + let now = match now_cutoff { + Some(now_cutoff) if now_cutoff >= now => { + tracing::error!( + "now_cutoff ({:?}) is after now ({:?}) for schedule {}. Using now_cutoff + 1s. This likely means the pg clock was shifted backwards.", + now_cutoff, + now, + &schedule.path + ); + now_cutoff + chrono::Duration::seconds(1) + } + _ => now, + }; + let starting_from = match schedule.paused_until { Some(paused_until) if paused_until > now => paused_until.with_timezone(&tz), paused_until_o => { diff --git a/backend/windmill-worker/src/ai/mod.rs b/backend/windmill-worker/src/ai/mod.rs index b381e51693..4d11f100a5 100644 --- a/backend/windmill-worker/src/ai/mod.rs +++ b/backend/windmill-worker/src/ai/mod.rs @@ -4,4 +4,5 @@ pub mod image_handler; pub mod providers; pub mod query_builder; +pub mod sse; pub mod types; diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index b5a27a5f99..b94d4f29e5 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -5,7 +5,7 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err use crate::ai::{ image_handler::download_and_encode_s3_image, - query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, types::*, }; @@ -157,18 +157,24 @@ impl QueryBuilder for GoogleAIQueryBuilder { matches!(output_type, OutputType::Text) } + fn supports_streaming(&self) -> bool { + // Google AI supports streaming for text output + true + } + async fn build_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, + stream: bool, ) -> Result { match args.output_type { OutputType::Text => { // For text output, use OpenAI-compatible format let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); openai_builder - .build_request(args, client, workspace_id) + .build_request(args, client, workspace_id, stream) .await } OutputType::Image => self.build_image_request(args, client, workspace_id).await, @@ -238,6 +244,17 @@ impl QueryBuilder for GoogleAIQueryBuilder { } } + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + openai_builder + .parse_streaming_response(response, stream_event_processor) + .await + } + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { match output_type { OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index 421c5761f6..4fc4065766 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -5,7 +5,8 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err use crate::ai::{ image_handler::download_and_encode_s3_image, - query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, + sse::{OpenAISSEParser, SSEParser}, types::*, }; @@ -90,6 +91,7 @@ pub struct OpenAIRequest<'a> { pub max_completion_tokens: Option, #[serde(skip_serializing_if = "Option::is_none")] pub response_format: Option, + pub stream: bool, } pub struct OpenAIQueryBuilder { @@ -162,6 +164,7 @@ impl OpenAIQueryBuilder { args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, + stream: bool, ) -> Result { let prepared_messages = self .prepare_messages_for_api(args.messages, client, workspace_id) @@ -196,6 +199,7 @@ impl OpenAIQueryBuilder { temperature: args.temperature, max_completion_tokens: args.max_tokens, response_format, + stream, }; serde_json::to_string(&request) @@ -255,14 +259,23 @@ impl QueryBuilder for OpenAIQueryBuilder { true } + fn supports_streaming(&self) -> bool { + // OpenAI supports streaming for text output + true + } + async fn build_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, + stream: bool, ) -> Result { match args.output_type { - OutputType::Text => self.build_text_request(args, client, workspace_id).await, + OutputType::Text => { + self.build_text_request(args, client, workspace_id, stream) + .await + } OutputType::Image => self.build_image_request(args, client, workspace_id).await, } } @@ -330,10 +343,48 @@ impl QueryBuilder for OpenAIQueryBuilder { } }), tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + events_str: None, }) } } + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + let mut openai_sse_parser = OpenAISSEParser::new(stream_event_processor); + openai_sse_parser.parse_events(response).await?; + + let OpenAISSEParser { + accumulated_content, + accumulated_tool_calls, + mut events_str, + stream_event_processor, + } = openai_sse_parser; + + // Process streaming events with error handling + + for tool_call in accumulated_tool_calls.values() { + let event = StreamingEvent::ToolCallArguments { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + arguments: tool_call.function.arguments.clone(), + }; + stream_event_processor.send(event, &mut events_str).await?; + } + + Ok(ParsedResponse::Text { + content: if accumulated_content.is_empty() { + None + } else { + Some(accumulated_content) + }, + tool_calls: accumulated_tool_calls.into_values().collect(), + events_str: Some(events_str), + }) + } + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { let path = match output_type { OutputType::Text => "chat/completions", diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs index 3d5dd0314a..9e22a63552 100644 --- a/backend/windmill-worker/src/ai/providers/openrouter.rs +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -5,7 +5,7 @@ use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Err use crate::ai::{ providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, - query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, types::*, }; @@ -70,17 +70,23 @@ impl QueryBuilder for OpenRouterQueryBuilder { true } + fn supports_streaming(&self) -> bool { + // OpenRouter supports streaming for text output + true + } + async fn build_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, + stream: bool, ) -> Result { match args.output_type { OutputType::Text => { // For text, use standard OpenAI format without modalities self.openai_builder - .build_request(args, client, workspace_id) + .build_request(args, client, workspace_id, stream) .await } OutputType::Image => { @@ -184,9 +190,20 @@ impl QueryBuilder for OpenRouterQueryBuilder { .join(" "), }), tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + events_str: None, }) } + async fn parse_streaming_response( + &self, + response: reqwest::Response, + stream_event_processor: StreamEventProcessor, + ) -> Result { + self.openai_builder + .parse_streaming_response(response, stream_event_processor) + .await + } + fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { // OpenRouter uses the same endpoint for both text and image generation format!("{}/chat/completions", base_url) diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index 6b043726bc..e332c21f73 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -1,13 +1,19 @@ use async_trait::async_trait; -use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; +use windmill_common::{ + client::AuthedClient, error::Error, s3_helpers::S3Object, worker::Connection, +}; +use windmill_queue::MiniPulledJob; -use crate::ai::{ - providers::{ - google_ai::GoogleAIQueryBuilder, - openai::{OpenAIQueryBuilder, OpenAIToolCall}, - openrouter::OpenRouterQueryBuilder, +use crate::{ + ai::{ + providers::{ + google_ai::GoogleAIQueryBuilder, + openai::{OpenAIQueryBuilder, OpenAIToolCall}, + openrouter::OpenRouterQueryBuilder, + }, + types::*, }, - types::*, + job_logger::append_result_stream, }; /// Arguments for building an AI request @@ -26,7 +32,7 @@ pub struct BuildRequestArgs<'a> { /// Response from AI provider pub enum ParsedResponse { - Text { content: Option, tool_calls: Vec }, + Text { content: Option, tool_calls: Vec, events_str: Option }, Image { base64_data: String }, } @@ -36,17 +42,32 @@ pub trait QueryBuilder: Send + Sync { /// Check if this provider supports tools with the given output type fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool; + /// Check if this provider supports streaming + fn supports_streaming(&self) -> bool; + /// Build the request body for the provider async fn build_request( &self, args: &BuildRequestArgs<'_>, client: &AuthedClient, workspace_id: &str, + stream: bool, ) -> Result; /// Parse the response from the provider async fn parse_response(&self, response: reqwest::Response) -> Result; + /// Parse streaming response from the provider + async fn parse_streaming_response( + &self, + _response: reqwest::Response, + _stream_event_processor: StreamEventProcessor, + ) -> Result { + return Err(Error::internal_err( + "Missing implementation for parse_streaming_response for this provider".to_string(), + )); + } + /// Get the API endpoint for this provider fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String; @@ -69,3 +90,77 @@ pub fn create_query_builder(provider: &ProviderWithResource) -> Box Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Pass provider kind for Azure handling } } + +pub struct StreamEventProcessor { + tx: tokio::sync::mpsc::Sender, + pub handle: Option>, +} + +impl Clone for StreamEventProcessor { + fn clone(&self) -> Self { + Self { tx: self.tx.clone(), handle: None } + } +} + +impl StreamEventProcessor { + pub fn new(conn: &Connection, job: &MiniPulledJob) -> Self { + let (tx, mut rx) = tokio::sync::mpsc::channel::(100); + let conn = conn.clone(); + let job_id = job.id.clone(); + let workspace_id = job.workspace_id.clone(); + let handle = tokio::spawn(async move { + let mut offset = -1; + while let Some(event) = rx.recv().await { + offset += 1; + match tokio::time::timeout( + std::time::Duration::from_secs(20), + append_result_stream(&conn, &workspace_id, &job_id, &event, offset), + ) + .await + { + Ok(res) => { + if let Err(err) = res { + tracing::error!("Failed to save stream event: {}", err); + } + } + Err(err) => { + tracing::error!("Did not manage to save stream event after 20 seconds, stopping stream event processor: {}", err); + break; + } + } + } + }); + + Self { tx, handle: Some(handle) } + } + + pub async fn send(&self, event: StreamingEvent, events_str: &mut String) -> Result<(), Error> { + match serde_json::to_string(&event) { + Ok(event_json) => { + let event_json = format!("{}\n", event_json); + events_str.push_str(&event_json); + if let Err(err) = self + .tx + .send(event_json.clone()) + .await + .map_err(|e| Error::internal_err(format!("Failed to send event: {}", e))) + { + tracing::error!( + "Failed to send event to stream event processor, skiping event: {}", + err + ); + } + + Ok(()) + } + Err(e) => Err(Error::internal_err(format!( + "Failed to serialize streaming event {:#?}, error is: {}", + event, e + ))), + } + } + + pub fn to_handle(self) -> Option> { + self.handle + } +} diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs new file mode 100644 index 0000000000..fcafc9cd7e --- /dev/null +++ b/backend/windmill-worker/src/ai/sse.rs @@ -0,0 +1,158 @@ +use std::collections::HashMap; + +use reqwest::Response; +use serde::Deserialize; +use serde_json; +use tokio_stream::StreamExt; +use windmill_common::{error::Error, utils::rd_string}; + +use crate::ai::{ + providers::openai::{OpenAIFunction, OpenAIToolCall}, + query_builder::StreamEventProcessor, + types::StreamingEvent, +}; + +#[derive(Deserialize)] +pub struct OpenAIChoiceDeltaToolCallFunction { + pub name: Option, + pub arguments: Option, +} + +#[derive(Deserialize)] +pub struct OpenAIChoiceDeltaToolCall { + pub index: Option, + pub id: Option, + pub function: Option, +} + +#[derive(Deserialize)] +pub struct OpenAIChoiceDelta { + pub content: Option, + pub tool_calls: Option>, +} + +#[derive(Deserialize)] +pub struct OpenAIChoice { + pub delta: Option, +} + +#[derive(Deserialize)] +pub struct OpenAISSEEvent { + pub choices: Option>, +} + +pub trait SSEParser { + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; + + async fn parse_events(&mut self, response: Response) -> Result<(), Error> { + let mut stream = response.bytes_stream(); + let mut buffer = String::new(); + + while let Some(chunk_result) = stream.next().await { + let chunk = chunk_result + .map_err(|e| Error::internal_err(format!("Failed to read chunk: {}", e)))?; + + // Convert chunk to string and add to buffer + let chunk_str = String::from_utf8_lossy(&chunk); + buffer.push_str(&chunk_str); + + // Process complete lines from buffer + while let Some(newline_pos) = buffer.find("\n\n") { + let line = buffer.drain(..newline_pos + 2).collect::(); + let line = line.trim_end_matches('\n'); + + // Skip empty lines and comments + if line.is_empty() || line.starts_with(':') { + continue; + } + + // Parse SSE data field + if let Some(data) = line.strip_prefix("data: ") { + if data == "[DONE]" { + // OpenAI sends [DONE] to indicate end of stream + return Ok(()); + } + + self.parse_event_data(data).await?; + } + } + } + + Ok(()) + } +} + +pub struct OpenAISSEParser { + pub accumulated_content: String, + pub accumulated_tool_calls: HashMap, + pub events_str: String, + pub stream_event_processor: StreamEventProcessor, +} + +impl OpenAISSEParser { + pub fn new(stream_event_processor: StreamEventProcessor) -> Self { + Self { + accumulated_content: String::new(), + accumulated_tool_calls: HashMap::new(), + events_str: String::new(), + stream_event_processor, + } + } +} + +impl SSEParser for OpenAISSEParser { + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { + let event: OpenAISSEEvent = serde_json::from_str(data).map_err(|e| { + Error::internal_err(format!("Failed to parse SSE chunk {}: {}", data, e)) + })?; + + if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { + if let Some(delta) = choices.remove(0).delta { + if let Some(content) = delta.content.filter(|s| !s.is_empty()) { + self.accumulated_content.push_str(&content); + let event = StreamingEvent::TokenDelta { content }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } + + if let Some(tool_calls) = delta.tool_calls { + for (idx, tool_call) in tool_calls.into_iter().enumerate() { + let idx = tool_call.index.unwrap_or_else(|| idx as i64); + + if let Some(function) = tool_call.function { + if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { + if let Some(arguments) = function.arguments { + tool_call.function.arguments += &arguments; + } + } else { + let fun_name = function.name.unwrap_or_default(); + let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); + let event = StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: fun_name.clone(), + }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: fun_name, + arguments: function.arguments.unwrap_or_default(), + }, + r#type: "function".to_string(), + }, + ); + } + } + } + } + } + } + + Ok(()) + } +} diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index cc1417a92d..2ffab9713e 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -116,6 +116,7 @@ pub struct AIAgentArgs { pub output_schema: Option, pub output_type: Option, pub user_images: Option>, + pub streaming: Option, } #[derive(Deserialize, Debug)] @@ -153,6 +154,24 @@ impl ProviderWithResource { pub struct AIAgentResult<'a> { pub output: Box, pub messages: Vec>, + #[serde(skip_serializing_if = "Option::is_none")] + pub wm_stream: Option, +} + +/// Events for streaming AI responses +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum StreamingEvent { + /// Individual token from the AI response + TokenDelta { content: String }, + /// Tool call has started + ToolCall { call_id: String, function_name: String }, + /// Tool call arguments are complete + ToolCallArguments { call_id: String, function_name: String, arguments: String }, + /// Tool execution has started + ToolExecution { call_id: String, function_name: String }, + /// Tool execution result + ToolResult { call_id: String, function_name: String, result: String, success: bool }, } #[derive(Serialize, Deserialize, Clone, Debug)] diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 2eddb657b5..0e14b4f556 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use async_recursion::async_recursion; use regex::Regex; use serde_json::value::RawValue; @@ -5,8 +6,7 @@ use std::{collections::HashMap, sync::Arc}; use ulid; use uuid::Uuid; use windmill_common::{ - ai_providers::AIProvider, - ai_providers::AZURE_API_VERSION, + ai_providers::{AIProvider, AZURE_API_VERSION}, cache, client::AuthedClient, db::DB, @@ -27,10 +27,14 @@ use windmill_queue::{ use crate::{ ai::{ image_handler::upload_image_to_s3, - query_builder::{create_query_builder, BuildRequestArgs, ParsedResponse}, + query_builder::{ + create_query_builder, BuildRequestArgs, ParsedResponse, StreamEventProcessor, + }, types::*, }, - common::{build_args_map, error_to_value, OccupancyMetrics}, + common::{ + build_args_map, error_to_value, resolve_job_timeout, OccupancyMetrics, StreamNotifier, + }, create_job_dir, handle_child::run_future_with_polling_update_job_poller, handle_queued_job, parse_sig_of_lang, @@ -44,7 +48,6 @@ lazy_static::lazy_static! { } const MAX_AGENT_ITERATIONS: usize = 10; -const REQUEST_TIMEOUT_SECONDS: u64 = 120; fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, Error> { let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some @@ -114,6 +117,7 @@ pub async fn handle_ai_agent_job( worker_name: &str, hostname: &str, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + has_stream: &mut bool, ) -> Result, Error> { let args = build_args_map(job, client, conn).await?; @@ -264,6 +268,12 @@ pub async fn handle_ai_agent_job( let mut inner_occupancy_metrics = occupancy_metrics.clone(); + let stream_notifier = StreamNotifier::new(conn, job); + + if let Some(stream_notifier) = stream_notifier { + stream_notifier.update_flow_status_with_stream_job(); + } + let agent_fut = run_agent( db, conn, @@ -279,6 +289,7 @@ pub async fn handle_ai_agent_job( worker_name, hostname, killpill_rx, + has_stream, ); let result = run_future_with_polling_update_job_poller( @@ -412,6 +423,7 @@ pub async fn run_agent( worker_name: &str, hostname: &str, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + has_stream: &mut bool, ) -> error::Result> { let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); let base_url = args.provider.get_base_url(db).await?; @@ -501,6 +513,21 @@ pub async fn run_agent( // For non-Anthropic providers, response_format is handled by the query builder } + // Check if streaming is enabled and supported + let should_stream = args.streaming.unwrap_or(false) + && query_builder.supports_streaming() + && output_type == &OutputType::Text; + + *has_stream = should_stream; + + let mut final_events_str = String::new(); + + let stream_event_processor = if should_stream { + Some(StreamEventProcessor::new(conn, job)) + } else { + None + }; + // Main agent loop for i in 0..MAX_AGENT_ITERATIONS { if used_structured_output_tool { @@ -522,21 +549,25 @@ pub async fn run_agent( }; let request_body = query_builder - .build_request(&build_args, client, &job.workspace_id) + .build_request(&build_args, client, &job.workspace_id, should_stream) .await?; let endpoint = query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); + let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout) + .await + .0; + let mut request = HTTP_CLIENT .post(&endpoint) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECONDS)) + .timeout(timeout) .header("Content-Type", "application/json"); // Apply authentication headers - for (header_name, header_value) in auth_headers { - request = request.header(header_name, header_value); + for (header_name, header_value) in &auth_headers { + request = request.header(*header_name, header_value.clone()); } if args.provider.kind.is_azure_openai(&base_url) { @@ -551,10 +582,21 @@ pub async fn run_agent( match resp.error_for_status_ref() { Ok(_) => { - let parsed = query_builder.parse_response(resp).await?; + let parsed = if let Some(stream_event_processor) = stream_event_processor.clone() { + query_builder + .parse_streaming_response(resp, stream_event_processor) + .await? + } else { + // Handle non-streaming response + query_builder.parse_response(resp).await? + }; match parsed { - ParsedResponse::Text { content: response_content, tool_calls } => { + ParsedResponse::Text { content: response_content, tool_calls, events_str } => { + if let Some(events_str) = events_str { + final_events_str.push_str(&events_str); + } + if let Some(ref response_content) = response_content { actions.push(AgentAction::Message {}); messages.push(OpenAIMessage { @@ -589,6 +631,17 @@ pub async fn run_agent( // Handle tool calls (keeping existing tool execution logic) for tool_call in tool_calls.iter() { + // Stream tool call progress + if let Some(ref stream_event_processor) = stream_event_processor { + let event = StreamingEvent::ToolExecution { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + }; + stream_event_processor + .send(event, &mut final_events_str) + .await?; + } + // Check if this is the structured output tool if structured_output_tool_name .as_ref() @@ -631,10 +684,22 @@ pub async fn run_agent( update_flow_status_module_with_actions(db, parent_job, &actions) .await?; + let raw_tool_call_args = if tool_call.function.arguments.is_empty() + { + "{}".to_string() + } else { + tool_call.function.arguments.clone() + }; let tool_call_args = serde_json::from_str::>>( - &tool_call.function.arguments, - )?; + &raw_tool_call_args, + ) + .with_context(|| { + format!( + "Failed to parse tool call arguments for tool call {}: {}", + tool_call.function.name, tool_call.function.arguments + ) + })?; let job_payload = match tool.module.get_value()? { FlowModuleValue::Script { @@ -806,12 +871,13 @@ pub async fn run_agent( worker_name, ) .await; + let error_message = + format!("Error running tool: {}", err_string); messages.push(OpenAIMessage { role: "tool".to_string(), - content: Some(OpenAIContent::Text(format!( - "Error running tool: {}", - err_string - ))), + content: Some(OpenAIContent::Text( + error_message.clone(), + )), tool_call_id: Some(tool_call.id.clone()), agent_action: Some(AgentAction::ToolCall { job_id, @@ -820,6 +886,21 @@ pub async fn run_agent( }), ..Default::default() }); + // Stream tool result (error case) + if let Some(ref stream_event_processor) = + stream_event_processor + { + let tool_result_event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + result: error_message, + success: false, + }; + stream_event_processor + .send(tool_result_event, &mut final_events_str) + .await?; + } + update_flow_status_module_with_actions_success( db, parent_job, false, ) @@ -857,7 +938,6 @@ pub async fn run_agent( "Tool job completed but no result".to_string(), )); }; - messages.push(OpenAIMessage { role: "tool".to_string(), content: Some(OpenAIContent::Text( @@ -871,6 +951,22 @@ pub async fn run_agent( }), ..Default::default() }); + + // Stream tool result (success case) + if let Some(ref stream_event_processor) = + stream_event_processor + { + let tool_result_event = StreamingEvent::ToolResult { + call_id: tool_call.id.clone(), + function_name: tool_call.function.name.clone(), + result: result.get().to_string(), + success: true, + }; + stream_event_processor + .send(tool_result_event, &mut final_events_str) + .await?; + } + update_flow_status_module_with_actions_success( db, parent_job, success, ) @@ -930,8 +1026,24 @@ pub async fn run_agent( None => to_raw_value(&""), }; + if let Some(stream_event_processor) = stream_event_processor { + if let Some(handle) = stream_event_processor.to_handle() { + if let Err(e) = handle.await { + return Err(Error::internal_err(format!( + "Error waiting for stream event processor: {}", + e + ))); + } + } + } + Ok(to_raw_value(&AIAgentResult { output: output_value, messages: final_messages, + wm_stream: if !final_events_str.is_empty() { + Some(final_events_str) + } else { + None + }, })) } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4016010b63..87f15c3294 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -858,6 +858,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> error::Result> { let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); @@ -1115,7 +1116,7 @@ async function run() {{ let res = await Main.{main_name}(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); }} res = null; }} @@ -1331,6 +1332,7 @@ try {{ false, occupancy_metrics, stream_notifier, + has_stream, ) .await?; tracing::info!( @@ -1493,6 +1495,8 @@ try {{ ) .await?; + *has_stream = handle_result.result_stream.is_some(); + if apply_preprocessor { let args = read_file(&format!("{job_dir}/args.json")) .await diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index 1805b22230..a367c818a3 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -186,14 +186,14 @@ pub async fn handle_dedicated_process( let result = Arc::new(result); append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await; if line.starts_with("wm_res[success]:") { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap() } else { - job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap() + job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap() } }, Err(e) => { tracing::error!("Could not deserialize job result `{line}`: {e:?}"); - job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None }, true).await.unwrap(); + job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None, preprocessed_args: None, has_stream: Some(false) }, true).await.unwrap(); }, }; logs = init_log.clone(); diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 0e15e5ba12..fffe236fe9 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -194,6 +194,7 @@ pub async fn handle_deno_job( envs: HashMap, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + has_stream: &mut bool, ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); @@ -298,7 +299,7 @@ async function run() {{ let res: any = await {main_name}(...argsArr); if (isAsyncIterable(res)) {{ for await (const chunk of res) {{ - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); }} res = null; }} @@ -440,6 +441,9 @@ try {{ stream_notifier, ) .await?; + + *has_stream = handle_result.result_stream.is_some(); + // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); if let Err(e) = tokio::fs::remove_dir_all(format!("{DENO_CACHE_DIR}/gen/file/{job_dir}")).await { diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index aea5f7be15..ceb0ccb96e 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::env; -use std::ffi::{c_char, CString}; +use std::ffi::{c_char, CStr, CString}; use std::ptr::NonNull; use std::sync::{Arc, Mutex}; @@ -193,6 +193,7 @@ struct DuckDbFfiLib { column_order_ptr: *mut *mut c_char, ) -> *mut c_char, >, + free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, } impl DuckDbFfiLib { @@ -232,6 +233,7 @@ impl DuckDbFfiLib { let lib = Box::leak(Box::new(lib)); Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, + free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, }) } } @@ -264,8 +266,9 @@ fn run_duckdb_ffi_safe<'a>( let w_id = CString::new(w_id).map_err(to_anyhow)?; let run_duckdb_ffi = &DuckDbFfiLib::get_singleton()?.run_duckdb_ffi; + let free_cstr = &DuckDbFfiLib::get_singleton()?.free_cstr; let mut column_order: *mut c_char = std::ptr::null_mut(); - let result_cstr = unsafe { + let result_str = unsafe { let ptr = run_duckdb_ffi( query_block_list.as_ptr(), query_block_list_count, @@ -275,27 +278,19 @@ fn run_duckdb_ffi_safe<'a>( w_id.as_ptr(), &mut column_order, ); - CString::from_raw(ptr) // Using from_raw to take ownership and ensure it gets freed + let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); + free_cstr(ptr); + str }; let column_order = if column_order.is_null() { None } else { - Some(unsafe { - serde_json::from_str::>(&CString::from_raw(column_order).to_string_lossy())? - }) + let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() }; + unsafe { free_cstr(column_order) }; + Some(serde_json::from_str::>(&str)?) }; - let result_str = result_cstr - .to_str() - .map_err(|e| { - Error::ExecutionErr(format!( - "Failed to convert result C string to Rust string: {}", - e.to_string() - )) - })? - .to_string(); - if result_str.starts_with("ERROR") { Err(Error::ExecutionErr(result_str[6..].to_string())) } else { diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 6d36b8ca0d..6c8f94d6ac 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -29,7 +29,7 @@ use windmill_queue::{append_logs, CanceledBy}; use std::os::unix::process::ExitStatusExt; use std::process::ExitStatus; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicI32, AtomicU32, Ordering}; use std::sync::Arc; use std::{io, panic, time::Duration}; @@ -388,6 +388,7 @@ pub async fn write_lines( let mut pipe_stdout = pipe_stdout; let is_stream = Arc::new(AtomicBool::new(false)); + let offset = Arc::new(AtomicI32::new(0)); while let Some(line) = output.by_ref().next().await { let do_write_ = do_write.shared(); @@ -486,6 +487,7 @@ pub async fn write_lines( let pg_log_total_size = pg_log_total_size.clone(); let stream_notifier = stream_notifier.clone(); let is_stream = is_stream.clone(); + let offset = offset.clone(); (do_write, write_result) = tokio::spawn(async move { if !nstream.is_empty() { if let Some(stream_notifier) = stream_notifier { @@ -493,9 +495,17 @@ pub async fn write_lines( is_stream.store(true, Ordering::SeqCst); stream_notifier.update_flow_status_with_stream_job(); } - } + }; - if let Err(err) = append_result_stream(&conn, &w_id, &job_id, &nstream).await { + if let Err(err) = append_result_stream( + &conn, + &w_id, + &job_id, + &nstream, + offset.fetch_add(1, Ordering::SeqCst), + ) + .await + { tracing::error!( "Unable to send result stream for job {job_id}. Error was: {:?}", err diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index 263234cee9..11e7785ee5 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -8,6 +8,7 @@ use windmill_common::worker::{Connection, CLOUD_HOSTED}; use windmill_common::{error, DB}; use windmill_queue::append_logs; +use serde::Serialize; use std::sync::atomic::AtomicU32; use std::sync::Arc; @@ -62,22 +63,33 @@ pub async fn append_job_logs( } } +#[derive(Serialize)] +struct ResultStreamBody<'a> { + result_stream: &'a str, + offset: i32, +} + pub async fn append_result_stream( conn: &Connection, workspace_id: &str, job_id: &Uuid, nstream: &str, + offset: i32, ) -> error::Result<()> { match conn { Connection::Sql(db) => { - append_result_stream_db(db, workspace_id, job_id, nstream).await?; + append_result_stream_db(db, workspace_id, job_id, nstream, offset).await?; } Connection::Http(client) => { + let body = ResultStreamBody { result_stream: nstream, offset }; if let Err(e) = client .post::<_, String>( - &format!("/api/w/{}/agent_workers/push_logs/{}", workspace_id, job_id), + &format!( + "/api/w/{}/agent_workers/push_result_stream/{}", + workspace_id, job_id + ), None, - &nstream, + &body, ) .await { diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index cbf19b5946..ec6195a5cc 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -770,6 +770,7 @@ pub async fn eval_fetch_timeout( _load_client: bool, _occupation_metrics: &mut OccupancyMetrics, _stream_notifier: Option, + _has_stream: &mut bool, ) -> anyhow::Result> { use serde_json::value::to_raw_value; Ok(to_raw_value("require deno_core").unwrap()) @@ -792,6 +793,7 @@ pub async fn eval_fetch_timeout( load_client: bool, occupation_metrics: &mut OccupancyMetrics, stream_notifier: Option, + has_stream: &mut bool, ) -> windmill_common::error::Result> { let (sender, mut receiver) = oneshot::channel::(); let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); @@ -808,9 +810,11 @@ pub async fn eval_fetch_timeout( let conn_ = conn.clone(); let w_id_ = w_id.to_string(); tokio::spawn(async move { + let mut offset = -1; while let Some(stream) = result_stream_receiver.recv().await { use crate::job_logger::append_result_stream; - if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream).await { + offset += 1; + if let Err(e) = append_result_stream(&conn_, &w_id_, &job_id, &stream, offset).await { tracing::error!("failed to append result stream: {e}"); } } @@ -978,22 +982,24 @@ pub async fn eval_fetch_timeout( drop(js_runtime); if let Ok(r) = r { match handle.await { - Ok(Some(logs)) => Ok(merge_result_stream(r, Some(logs)).await), - Ok(None) => Ok(r), + Ok(Some(logs)) => { + Ok(merge_result_stream(r, Some(logs)).await.map(|r| (r, true))) + } + Ok(None) => Ok(r.map(|r| (r, false))), Err(e) => Err(Error::ExecutionErr(e.to_string())), } } else { - r + r.map(|r| r.map(|r| (r, false))) } // r }; let r = runtime.block_on(future)?; // tracing::info!("total: {:?}", instant.elapsed()); - r as windmill_common::error::Result> + r as windmill_common::error::Result<(Box, bool)> }); - let res = run_future_with_polling_update_job_poller( + let (res, new_has_stream) = run_future_with_polling_update_job_poller( job_id, job_timeout, conn, @@ -1012,6 +1018,7 @@ pub async fn eval_fetch_timeout( } e })?; + *has_stream = new_has_stream; *mem_peak = (res.get().len() / 1000) as i32; Ok(res) } @@ -1109,7 +1116,7 @@ function processStreamIterative(res) {{ iterator.next().then(function(result) {{ if (!result.done) {{ const chunk = result.value; - console.log("WM_STREAM: " + chunk.replace('\n', '\\n')); + console.log("WM_STREAM: " + chunk.replace(/\n/g, '\\n')); // Continue the loop step(); }} else {{ diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index b31de6e184..5b742feb4b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -544,6 +544,7 @@ pub async fn handle_python_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.runnable_path()); @@ -885,6 +886,8 @@ mount {{ ) .await?; + *has_stream = handle_result.result_stream.is_some(); + if apply_preprocessor { let args = read_file(&format!("{job_dir}/args.json")) .await diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 144c8d9101..5bd3dd1753 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -17,7 +17,14 @@ use windmill_common::otel_oss::FutureExt; use uuid::Uuid; use windmill_common::{ - add_time, error::{self, Error}, flow_status::{FlowJobDuration}, jobs::JobKind, utils::WarnAfterExt, worker::{to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, KillpillSender, DB + add_time, + error::{self, Error}, + flow_status::FlowJobDuration, + jobs::JobKind, + utils::WarnAfterExt, + worker::{to_raw_value, Connection, WORKER_GROUP}, + worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, + KillpillSender, DB, }; #[cfg(feature = "benchmark")] @@ -405,6 +412,7 @@ pub async fn process_result( preprocessed_args: Option>>, conn: &Connection, duration: Option, + has_stream: bool, ) -> error::Result { match result { Ok(result) => { @@ -421,6 +429,7 @@ pub async fn process_result( cached_res_path, token: token.to_string(), duration, + has_stream: Some(has_stream), }, ) .with_context(windmill_common::otel_oss::otel_ctx()) @@ -483,6 +492,7 @@ pub async fn process_result( cached_res_path, token: token.to_string(), duration, + has_stream: Some(has_stream), }, ) .with_context(windmill_common::otel_oss::otel_ctx()) @@ -557,6 +567,7 @@ pub async fn process_completed_job( duration, result_columns, preprocessed_args, + has_stream, .. }: JobCompleted, client: &AuthedClient, @@ -621,6 +632,7 @@ pub async fn process_completed_job( canceled_by, false, duration, + has_stream.unwrap_or(false), ) .await?; drop(job); @@ -679,7 +691,10 @@ pub async fn process_completed_job( &job.workspace_id, false, Arc::new(serde_json::value::to_raw_value(&result).unwrap()), - duration.map(|x| FlowJobDuration { started_at: job.started_at.unwrap(), duration_ms: x }), + duration.map(|x| FlowJobDuration { + started_at: job.started_at.unwrap(), + duration_ms: x, + }), false, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 06a295f820..6c9f278d59 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -765,6 +765,7 @@ pub async fn handle_all_job_kind_error( cached_res_path: None, token: authed_client.token.clone(), duration: None, + has_stream: Some(false), }, false, ) @@ -1687,6 +1688,7 @@ pub async fn run_worker( token: "".to_string(), canceled_by: None, duration: None, + has_stream: Some(false), }, true, ) @@ -2229,6 +2231,7 @@ async fn do_nativets( canceled_by: &mut Option, worker_name: &str, occupancy_metrics: &mut OccupancyMetrics, + has_stream: &mut bool, ) -> windmill_common::error::Result> { let args = build_args_map(job, client, conn).await?.map(Json); let job_args = if args.is_some() { @@ -2255,6 +2258,7 @@ async fn do_nativets( true, occupancy_metrics, stream_notifier, + has_stream, ) .await?) } @@ -2408,6 +2412,7 @@ pub async fn handle_queued_job( cached_res_path: None, token: client.token.clone(), duration: None, + has_stream: Some(false), }, true, ) @@ -2491,6 +2496,7 @@ pub async fn handle_queued_job( let mut column_order: Option> = None; let mut new_args: Option>> = None; + let mut has_stream = false; let result = match job.kind { JobKind::Dependencies => match conn { Connection::Sql(db) => { @@ -2582,6 +2588,7 @@ pub async fn handle_queued_job( worker_name, hostname, killpill_rx, + &mut has_stream, ) .await } @@ -2614,6 +2621,7 @@ pub async fn handle_queued_job( occupancy_metrics, killpill_rx, precomputed_agent_info, + &mut has_stream, ) .await; occupancy_metrics.total_duration_of_running_jobs += @@ -2646,6 +2654,7 @@ pub async fn handle_queued_job( new_args, conn, Some(started.elapsed().as_millis() as i64), + has_stream, ) .await } @@ -2833,6 +2842,7 @@ async fn handle_code_execution_job( occupancy_metrics: &mut OccupancyMetrics, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, precomputed_agent_info: Option, + has_stream: &mut bool, ) -> error::Result> { let script_hash = || { job.runnable_id @@ -3172,6 +3182,7 @@ async fn handle_code_execution_job( canceled_by, worker_name, occupancy_metrics, + has_stream, ) .await?; return Ok(result); @@ -3245,6 +3256,7 @@ mount {{ new_args, occupancy_metrics, precomputed_agent_info, + has_stream, ) .await } @@ -3264,6 +3276,7 @@ mount {{ envs, new_args, occupancy_metrics, + has_stream, ) .await } @@ -3286,6 +3299,7 @@ mount {{ new_args, occupancy_metrics, precomputed_agent_info, + has_stream, ) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index afd2b6e850..8aafb17817 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -1473,6 +1473,7 @@ pub async fn update_flow_status_after_job_completion_internal( None, true, None, + false, ) .await?; duration @@ -1492,6 +1493,7 @@ pub async fn update_flow_status_after_job_completion_internal( None, true, None, + false, ) .await?; duration diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 489c5ab4c0..046c6f672d 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.548.1"; +export const VERSION = "v1.549.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 777ad9f2d0..f6825684d9 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.548.1"; +export const VERSION = "1.549.1"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/windmill-utils-internal/package.json b/cli/windmill-utils-internal/package.json index 72ec2220e8..ff4418f79b 100644 --- a/cli/windmill-utils-internal/package.json +++ b/cli/windmill-utils-internal/package.json @@ -1,6 +1,6 @@ { "name": "windmill-utils-internal", - "version": "1.3.0", + "version": "1.3.1", "description": "Internal utility functions for Windmill", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -21,4 +21,4 @@ "files": [ "dist/**/*" ] -} +} \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/config/index.ts b/cli/windmill-utils-internal/src/config/index.ts index f3ae42b3c8..e23ba6ca86 100644 --- a/cli/windmill-utils-internal/src/config/index.ts +++ b/cli/windmill-utils-internal/src/config/index.ts @@ -1 +1 @@ -export * from "./config.ts"; \ No newline at end of file +export * from "./config"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/index.ts b/cli/windmill-utils-internal/src/index.ts index 635893e2d1..da314a7c5a 100644 --- a/cli/windmill-utils-internal/src/index.ts +++ b/cli/windmill-utils-internal/src/index.ts @@ -8,8 +8,8 @@ * - Cross-platform path constants */ -export * from "./inline-scripts.ts"; -export * from "./path-utils.ts"; -export * from "./parse.ts"; -export * from "./config.ts"; -export { SEP, DELIMITER } from "./constants.ts"; \ No newline at end of file +export * from "./inline-scripts"; +export * from "./path-utils"; +export * from "./parse"; +export * from "./config"; +export { SEP, DELIMITER } from "./constants"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index be2d33c0de..b64e7ca789 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -1,5 +1,5 @@ -import { newPathAssigner } from "../path-utils/path-assigner.ts"; -import { FlowModule } from "../gen/types.gen.ts"; +import { newPathAssigner } from "../path-utils/path-assigner"; +import { FlowModule } from "../gen/types.gen"; /** * Represents an inline script extracted from a flow module diff --git a/cli/windmill-utils-internal/src/inline-scripts/index.ts b/cli/windmill-utils-internal/src/inline-scripts/index.ts index bb3c917dbb..eace8d3e4f 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/index.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/index.ts @@ -1,2 +1,2 @@ -export * from "./replacer.ts"; -export * from "./extractor.ts"; \ No newline at end of file +export * from "./replacer"; +export * from "./extractor"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index 9ec25e9f65..be349785c7 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -1,4 +1,4 @@ -import { FlowModule } from "../gen/types.gen.ts"; +import { FlowModule } from "../gen/types.gen"; /** * Replaces inline script references with actual file content from the filesystem. diff --git a/cli/windmill-utils-internal/src/parse/index.ts b/cli/windmill-utils-internal/src/parse/index.ts index 41d09ed00d..fc26ce611a 100644 --- a/cli/windmill-utils-internal/src/parse/index.ts +++ b/cli/windmill-utils-internal/src/parse/index.ts @@ -1 +1 @@ -export * from "./parse-schema.ts"; \ No newline at end of file +export * from "./parse-schema"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/parse/parse-schema.ts b/cli/windmill-utils-internal/src/parse/parse-schema.ts index a6417a1c01..9cd4a596b4 100644 --- a/cli/windmill-utils-internal/src/parse/parse-schema.ts +++ b/cli/windmill-utils-internal/src/parse/parse-schema.ts @@ -1,7 +1,7 @@ /** * Type alias for enum values - can be an array of strings or undefined */ -export type EnumType = string[] | undefined; +export type EnumType = string[] | { label: string; value: string }[] | undefined; /** * Represents a property in a JSON schema with various validation and display options @@ -17,7 +17,7 @@ export interface SchemaProperty { items?: { type?: "string" | "number" | "bytes" | "object" | "resource"; contentEncoding?: "base64"; - enum?: string[]; + enum?: EnumType; resourceType?: string; properties?: { [name: string]: SchemaProperty }; }; @@ -54,22 +54,22 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { name?: string; props?: { key: string; typ: any }[] }) - | { str: any } - | { object: { name?: string; props?: { key: string; typ: any }[] } } - | null; - } + list: + | (string | { name?: string; props?: { key: string; typ: any }[] }) + | { str: any } + | { object: { name?: string; props?: { key: string; typ: any }[] } } + | null; + } | { dynselect: string } | { dynmultiselect: string } | { str: string[] | null } | { object: { name?: string; props?: { key: string; typ: any }[] } } | { - oneof: { - label: string; - properties: { key: string; typ: any }[]; - }[]; - }, + oneof: { + label: string; + properties: { key: string; typ: any }[]; + }[]; + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: "" }; diff --git a/cli/windmill-utils-internal/src/path-utils/index.ts b/cli/windmill-utils-internal/src/path-utils/index.ts index 6f5c8d68be..ef23185664 100644 --- a/cli/windmill-utils-internal/src/path-utils/index.ts +++ b/cli/windmill-utils-internal/src/path-utils/index.ts @@ -1 +1 @@ -export * from "./path-assigner.ts"; \ No newline at end of file +export * from "./path-assigner"; \ No newline at end of file diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index d1a79a3a32..9020974214 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -1,4 +1,4 @@ -import { RawScript } from "../gen/types.gen.ts"; +import { RawScript } from "../gen/types.gen"; const INLINE_SCRIPT_PREFIX = "inline_script"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e503cf2d65..4659c8bc51 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.548.1", + "version": "1.549.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.548.1", + "version": "1.549.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -86,7 +86,7 @@ "windmill-parser-wasm-ts": "1.538.0", "windmill-parser-wasm-yaml": "1.510.1", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.0", + "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", @@ -13542,9 +13542,9 @@ "integrity": "sha512-uHNL8F72/Tf96xF3hOHnPDjkEyqXw7fNjcPJiUhth9sTQkcwUIoJMOdwm8/cs+j9kKVRJ4tgNYMHEBLylazp6g==" }, "node_modules/windmill-utils-internal": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.0.tgz", - "integrity": "sha512-UH7G+NVODkhm4o3BbjaOrSE2Qu+J6ro7+vpsIs+GvjDZM4ogSN7aJfnQNHW7Ke0VY74BKZVClTKROe/N6I5Reg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/windmill-utils-internal/-/windmill-utils-internal-1.3.1.tgz", + "integrity": "sha512-afRGUDcvaUfGu7FA6DD0xWECQiKnXADs0N4WyQQ+OvaloxZ4oQzdEpLnVab/m3T02hhs29ru4Ilrdxu3ozyT5Q==", "license": "Apache 2.0" }, "node_modules/word-wrap": { diff --git a/frontend/package.json b/frontend/package.json index 111f0195ee..be97c6b950 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.548.1", + "version": "1.549.1", "scripts": { "dev": "vite dev", "build": "vite build", @@ -151,7 +151,7 @@ "windmill-parser-wasm-ts": "1.538.0", "windmill-parser-wasm-yaml": "1.510.1", "windmill-sql-datatype-parser-wasm": "1.512.0", - "windmill-utils-internal": "^1.3.0", + "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", "xterm-readline": "^1.1.2", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index b96af76d03..7ea1de146e 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -15,7 +15,7 @@ export interface PropertyDisplayInfo { propertiesNumber: number } -export type EnumType = string[] | undefined +export type EnumType = string[] | { value: string; label: string }[] | undefined export interface SchemaProperty { type: string | undefined diff --git a/frontend/src/lib/components/AIAgentLogViewer.svelte b/frontend/src/lib/components/AIAgentLogViewer.svelte index 2640ea52eb..c8fd14d6fc 100644 --- a/frontend/src/lib/components/AIAgentLogViewer.svelte +++ b/frontend/src/lib/components/AIAgentLogViewer.svelte @@ -13,14 +13,14 @@ import { onMount } from 'svelte' type AgentActionWithContent = NonNullable[number] & { - content: string + content?: unknown } const resultSchema = z.object({ messages: z.array( z.object({ role: z.string(), - content: z.string().optional(), + content: z.unknown(), agent_action: z .union([ z.object({ diff --git a/frontend/src/lib/components/ArgEnum.svelte b/frontend/src/lib/components/ArgEnum.svelte index 2e9dd26ea6..eff7b834e8 100644 --- a/frontend/src/lib/components/ArgEnum.svelte +++ b/frontend/src/lib/components/ArgEnum.svelte @@ -32,11 +32,19 @@ let customItems: string[] = $state([]) let items = $derived.by(() => { - const l = [...(enum_ ? enum_ : []), ...customItems].map((item) => ({ - value: item, - label: enumLabels?.[item] ?? item - })) - if (create && filterText && l.every((i) => i.value !== filterText)) { + const l = [...(enum_ ? enum_ : []), ...customItems] + .map((item) => { + if (typeof item === 'string') { + return { + value: item, + label: enumLabels?.[item] ?? item + } + } else if (typeof item === 'object') { + return item + } + }) + .filter((i) => i != undefined) + if (create && filterText && l.every((i) => i?.value !== filterText)) { l.push({ value: filterText, label: `Add new: ${filterText}` }) } return l diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 9d7e827cd4..6877795d55 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -255,7 +255,12 @@ if (inputCat === 'string') { nvalue = nullable ? null : '' } else if (inputCat == 'enum' && required) { - nvalue = enum_?.[0] + let firstV = enum_?.[0] + if (typeof firstV === 'string') { + nvalue = firstV + } else if (firstV && typeof firstV === 'object') { + nvalue = firstV.value + } } else if (inputCat == 'boolean') { nvalue = false } else if (inputCat == 'list') { diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index ed2c6c1b32..4a0f3af5dc 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -574,7 +574,7 @@ let themeDarkRaw = $derived($page.url.searchParams.get('activeColorTheme')) let themeDark = $derived(themeDarkRaw == '2' || themeDarkRaw == '4') - $effect(() => { + $effect.pre(() => { setContext<{ token?: string }>('AuthToken', { token }) }) $effect.pre(() => { diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index 1c03241e00..3b962a4eee 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -36,7 +36,7 @@ import Tooltip from './Tooltip.svelte' import HighlightTheme from './HighlightTheme.svelte' import type { DisplayResultUi } from './custom_ui' - import { getContext, hasContext, createEventDispatcher, onDestroy } from 'svelte' + import { getContext, hasContext, createEventDispatcher, onDestroy, untrack } from 'svelte' import { toJsonStr } from '$lib/utils' import { userStore } from '$lib/stores' import ResultStreamDisplay from './ResultStreamDisplay.svelte' @@ -139,24 +139,32 @@ function isTableRowObject(json) { // check array of objects (with possible a first row of headers) + const hasHeaders = + Array.isArray(json[0]) && + json[0].length > 0 && + json[0].length <= 50 && + json[0].every((item) => typeof item === 'string') + return isTableRowObjectInner(json, hasHeaders) + } + + function isTableRowObjectInner(json: any, hasHeaders: boolean) { return ( Array.isArray(json) && - json.length > 0 && - (json.every( - (item) => - item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item) - ) || - (Array.isArray(json[0]) && - json[0].every((item) => typeof item === 'string') && - json - .slice(1) - .every( - (item) => - item && - typeof item === 'object' && - Object.keys(item).length > 0 && - !Array.isArray(item) - ))) + json.length > (hasHeaders ? 1 : 0) && + json.every((item, index) => { + if (hasHeaders && index === 0) { + return true + } + if (item && typeof item === 'object') { + let keys = Object.keys(item) + if (keys.length > 0 && !Array.isArray(item)) { + if (hasHeaders || keys.length <= 50) { + return true + } + } + } + return false + }) ) } @@ -200,6 +208,7 @@ } let size = roughSizeOfObject(result) + console.debug('size of object', size) // Otherwise, check if the result is too large (10kb) for json if (size > TABLE_MAX_SIZE) { @@ -363,28 +372,23 @@ json.length > 0 && Array.isArray(json[0]) && json[0].length > 0 && - json[0].every((item) => typeof item === 'string') && - json - .slice(1) - .every( - (item) => - item && typeof item === 'object' && Object.keys(item).length > 0 && !Array.isArray(item) - ) + json[0].every((item) => typeof item === 'string') ) { const headers = json[0] - const rows = json.slice(1) + const rows: { [key: string]: string }[] = new Array(json.length - 1) - const result = rows.map((row) => { + for (let i = 1; i < json.length; i++) { const obj: { [key: string]: string } = {} + const row = json[i] - for (const header of headers) { - obj[header] = row[header] + for (let j = 0; j < headers.length; j++) { + obj[headers[j]] = row[headers[j]] } - return obj - }) + rows[i - 1] = obj + } - return result + return rows } return json @@ -479,7 +483,9 @@ $effect(() => { ;[result] - resultKind = inferResultKind(result) + untrack(() => { + resultKind = inferResultKind(result) + }) }) $effect(() => { chooseToolbarLocation( diff --git a/frontend/src/lib/components/FlowLoopIterationPreview.svelte b/frontend/src/lib/components/FlowLoopIterationPreview.svelte index 6c296d0804..7c9bc78fa0 100644 --- a/frontend/src/lib/components/FlowLoopIterationPreview.svelte +++ b/frontend/src/lib/components/FlowLoopIterationPreview.svelte @@ -67,7 +67,7 @@ let selectedJobStep: string | undefined = $state(undefined) let isRunning: boolean = $state(false) - let jobProgressReset: (() => void) | undefined = $state(undefined) + let progressBar: FlowProgressBar | undefined = $state(undefined) export function test() { runPreview(previewArgs, undefined) @@ -80,7 +80,7 @@ args: Record, restartedFrom: RestartedFrom | undefined ) { - jobProgressReset?.() + progressBar?.reset() const newFlow = { value: { modules }, summary: '' } jobId = await runFlowPreview(args, newFlow, $pathStore, restartedFrom) isRunning = true @@ -161,7 +161,7 @@
- +
diff --git a/frontend/src/lib/components/FlowMetadata.svelte b/frontend/src/lib/components/FlowMetadata.svelte index ef090c9afb..bcb880dd0e 100644 --- a/frontend/src/lib/components/FlowMetadata.svelte +++ b/frontend/src/lib/components/FlowMetadata.svelte @@ -10,9 +10,13 @@ import { Clock, MemoryStick, Calendar, Bot, User, Code2 } from 'lucide-svelte' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' - export let job: Job const SMALL_ICON_SIZE = 14 - export let scheduleEditor: ScheduleEditor + interface Props { + job: Job + scheduleEditor: ScheduleEditor + } + + let { job, scheduleEditor }: Props = $props()
Schedule: - + + onclick={() => scheduleEditor?.openEdit(job.schedule_path ?? '', job.job_kind == 'flow')} > {truncateRev(job.schedule_path, 40)} diff --git a/frontend/src/lib/components/FlowStatusViewer.svelte b/frontend/src/lib/components/FlowStatusViewer.svelte index 7d5c1c4a12..70c331a80b 100644 --- a/frontend/src/lib/components/FlowStatusViewer.svelte +++ b/frontend/src/lib/components/FlowStatusViewer.svelte @@ -88,7 +88,7 @@ retryStatus.val = {} suspendStatus.val = {} globalRefreshes = {} - flowState.val = {} + flowState = {} localDurationStatuses = {} localModuleStates = {} } diff --git a/frontend/src/lib/components/JobArgs.svelte b/frontend/src/lib/components/JobArgs.svelte index 67435ebc09..00211790bc 100644 --- a/frontend/src/lib/components/JobArgs.svelte +++ b/frontend/src/lib/components/JobArgs.svelte @@ -14,14 +14,18 @@ import { deepEqual } from 'fast-equals' import { isWindmillTooBigObject } from './job_args' - export let id: string | undefined = undefined - export let args: any - export let argLabel: string | undefined = undefined - export let workspace: string | undefined = undefined + interface Props { + id?: string | undefined + args: any + argLabel?: string | undefined + workspace?: string | undefined + } - let jsonViewer: Drawer - let runLocally: Drawer - let jsonStr = '' + let { id = undefined, args, argLabel = undefined, workspace = undefined }: Props = $props() + + let jsonViewer: Drawer | undefined = $state() + let runLocally: Drawer | undefined = $state() + let jsonStr = $state('') function pythonCode() { return ` @@ -53,9 +57,9 @@ ${Object.entries(args) } -{#if args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'PREPROCESSOR_ARGS_ARE_DISCARDED'} +{#if args && typeof args === 'object' && deepEqual( Object.keys(args ?? {}), ['reason'] ) && args['reason'] == 'PREPROCESSOR_ARGS_ARE_DISCARDED'} Preprocessor args are discarded -{:else if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'} +{:else if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args ?? {}), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'} The args are too big in size to be able to fetch alongside job. Please download the JSON file to view them{argLabel ?? 'Arg'} Value - + {#snippet headerAction()} - + {/snippet} - {#if args && typeof args === 'object' && Object.keys(args).length > 0} - {#each Object.entries(args).sort((a, b) => a[0].localeCompare(b[0])) as [arg, value]} + {#if args && typeof args === 'object' && Object.keys(args ?? {}).length > 0} + {#each Object.entries(args ?? {}).sort( (a, b) => a?.[0]?.localeCompare(b?.[0]) ) as [arg, value]} {arg} @@ -124,7 +128,7 @@ ${Object.entries(args) Download - {/if} -
+ {#if allowKindChange} + + {/if} +
+ {:else} +
+ {JSON.stringify(enum_[i])} is not a string, remove it +
+ {/if} {/each} {/if}
diff --git a/frontend/src/lib/components/apps/editor/component/components.ts b/frontend/src/lib/components/apps/editor/component/components.ts index dbbeba0452..6b1041832a 100644 --- a/frontend/src/lib/components/apps/editor/component/components.ts +++ b/frontend/src/lib/components/apps/editor/component/components.ts @@ -3588,7 +3588,7 @@ See date-fns format for more information. By default, it is 'dd.MM.yyyy HH:mm' fieldType: 'object', value: {}, tooltip: - 'This enables setting form enum values dynamically using an object: keys are field names, and values are arrays of strings.' + 'This enables setting form enum values dynamically using an object: keys are field names, and values are arrays of strings or { "label": "myLabel", "value": "myValue" }.' }, displayType: { diff --git a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte index 63f018304a..c5289dc413 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsFilters.svelte @@ -260,6 +260,9 @@ USERS_ADD_GLOBAL: 'users.add_global', USERS_IMPERSONATE: 'users.impersonate', USERS_LEAVE_WORKSPACE: 'users.leave_workspace', + USERS_SCIM_CREATE: 'users.scim_create', + USERS_SCIM_DELETE: 'users.scim_delete', + USERS_SCIM_UPDATE: 'users.scim_update', OAUTH_LOGIN: 'oauth.login', OAUTH_LOGIN_FAILURE: 'oauth.login_failure', OAUTH_SIGNUP: 'oauth.signup', @@ -287,6 +290,9 @@ IGROUP_DELETE: 'igroup.delete', IGROUP_ADDUSER: 'igroup.adduser', IGROUP_REMOVEUSER: 'igroup.removeuser', + INSTANCE_GROUPS_SCIM_CREATE: 'instance_groups.scim_create', + INSTANCE_GROUPS_SCIM_DELETE: 'instance_groups.scim_delete', + INSTANCE_GROUPS_SCIM_UPDATE: 'instance_groups.scim_update', VARIABLES_DECRYPT_SECRET: 'variables.decrypt_secret', WORKSPACES_EDIT_COMMAND_SCRIPT: 'workspaces.edit_command_script', WORKSPACES_EDIT_DEPLOY_TO: 'workspaces.edit_deploy_to', diff --git a/frontend/src/lib/components/auditLogs/AuditLogsTable.svelte b/frontend/src/lib/components/auditLogs/AuditLogsTable.svelte index 3412f1dba2..f94164073b 100644 --- a/frontend/src/lib/components/auditLogs/AuditLogsTable.svelte +++ b/frontend/src/lib/components/auditLogs/AuditLogsTable.svelte @@ -18,6 +18,7 @@ selectedId?: number | undefined usernameFilter?: string | undefined resourceFilter?: string | undefined + showWorkspace?: boolean onselect?: (id: number) => void } @@ -31,6 +32,7 @@ selectedId = undefined, usernameFilter = $bindable(), resourceFilter = $bindable(), + showWorkspace = false, onselect }: Props = $props() @@ -128,9 +130,12 @@ class="flex flex-row bg-surface-secondary sticky top-0 w-full p-2 pr-4 text-xs font-semibold" >
ID
-
Timestamp
-
Username
-
Operation
+
Timestamp
+
Username
+ {#if showWorkspace} +
Workspace
+ {/if} +
Operation
Resource
@@ -174,10 +179,10 @@
{logOrDate.log.id}
-
+
{displayDate(logOrDate.log.timestamp)}
-
+
{logOrDate.log.username} @@ -196,7 +201,14 @@ />
-
+ {#if showWorkspace} +
+
+ {logOrDate.log.workspace_id} +
+
+ {/if} +
{ diff --git a/frontend/src/lib/components/flows/FlowProgressBar.svelte b/frontend/src/lib/components/flows/FlowProgressBar.svelte index 39ccbbf424..4837db5e43 100644 --- a/frontend/src/lib/components/flows/FlowProgressBar.svelte +++ b/frontend/src/lib/components/flows/FlowProgressBar.svelte @@ -2,18 +2,27 @@ import { type Job } from '$lib/gen' import ProgressBar from '../progressBar/ProgressBar.svelte' - export let job: Job | undefined = undefined - export let currentSubJobProgress: number | undefined = undefined + interface Props { + job?: Job | undefined + currentSubJobProgress?: number | undefined + class?: string + } - let error: number | undefined = undefined - let index = 0 - let subIndex: number | undefined = undefined - let subLength: number | undefined = undefined - let length = 1 - let nextInProgress = false - let subIndexIsPercent: boolean = false + let { + job = undefined, + currentSubJobProgress = $bindable(undefined), + class: className + }: Props = $props() - $: if (job) updateJobProgress(job) + let error: number | undefined = $state(undefined) + let index = $state(0) + let subIndex: number | undefined = $state(undefined) + let subLength: number | undefined = $state(undefined) + let length = $state(1) + let nextInProgress = $state(false) + let subIndexIsPercent: boolean = $state(false) + + let progressBar = $state(undefined) function updateJobProgress(job: Job) { const modules = job?.flow_status?.modules @@ -40,8 +49,8 @@ newError = maxDone maxDone = maxDone + 1 } - } - subIndexIsPercent = false; + } + subIndexIsPercent = false // Loop is still iterating if (module?.iterator) { @@ -54,12 +63,12 @@ } else if (module?.branchall) { subStepIndex = module.branchall.branch subStepLength = module.branchall.len - } else if (module?.progress) { + } else if (module?.progress) { const clamp = (num, min, max) => Math.min(Math.max(num, min), max) subStepIndex = clamp(module?.progress, subIndex ?? 0, 99) // Jitter protection >^^^^^^^^ subStepLength = 100 - subIndexIsPercent = true; + subIndexIsPercent = true currentSubJobProgress = subStepIndex } else { currentSubJobProgress = undefined @@ -73,26 +82,27 @@ nextInProgress = newNextInProgress } - let resetP: any - export function reset() { - resetP?.() + progressBar?.resetP() error = undefined subIndex = undefined subLength = undefined length = 1 index = 0 } + $effect(() => { + job && updateJobProgress(job) + }) diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index 3211a9f82e..1eaefab7d2 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -11,7 +11,7 @@ import Toggle from '$lib/components/Toggle.svelte' import { createScriptFromInlineScript, fork } from '$lib/components/flows/flowStateUtils.svelte' - import type { FlowModule, RawScript, ScriptLang } from '$lib/gen' + import type { FlowModule, FlowModuleValue, RawScript, ScriptLang } from '$lib/gen' import FlowCard from '../common/FlowCard.svelte' import FlowModuleHeader from './FlowModuleHeader.svelte' import { getLatestHashForScript, scriptLangToEditorLang } from '$lib/scripts' @@ -56,6 +56,7 @@ import AssetsDropdownButton from '$lib/components/assets/AssetsDropdownButton.svelte' import { useUiIntent } from '$lib/components/copilot/chat/flow/useUiIntent' import { editor as meditor } from 'monaco-editor' + import { DynamicInput } from '$lib/utils' const { selectedId, @@ -167,7 +168,6 @@ reloadError = undefined try { const { input_transforms, schema } = await loadSchemaFromModule(flowModule) - console.log('reload', schema) validCode = true if (inputTransformSchemaForm) { @@ -308,6 +308,30 @@ let modulePreviewResultViewer: ModulePreviewResultViewer | undefined = $state(undefined) + function retrieveDynCodeAndLang(value: FlowModuleValue): DynamicInput.HelperScript | undefined { + let helperScript: DynamicInput.HelperScript | undefined + switch (value.type) { + case 'script': + helperScript = { + source: 'deployed', + path: value.path, + runnable_kind: 'script' + } + break + case 'rawscript': + helperScript = { + source: 'inline', + code: value.content, + lang: value.language + } + break + default: + helperScript = undefined + } + + return helperScript + } + function onJobDone() { modulePreviewResultViewer?.getOutputPickerInner()?.setJobPreview() } @@ -570,14 +594,7 @@ } extraLib={stepPropPicker.extraLib} {enableAi} - helperScript={(flowModule?.value as RawScript).content && - (flowModule?.value as RawScript).language - ? { - source: 'inline', - code: (flowModule?.value as RawScript).content ?? '', - lang: (flowModule?.value as RawScript).language ?? 'deno' - } - : undefined} + helperScript={retrieveDynCodeAndLang(flowModule.value)} />
diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 7c24278cc2..ee25a6338e 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -77,6 +77,13 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{ type: 'string', description: 'The system prompt to give as input to the AI agent.' }, + streaming: { + type: 'boolean', + description: + 'Whether to stream the output of the AI agent (only used if output_type is text).', + default: false, + showExpr: "fields.output_type === 'text'" + }, user_images: { type: 'array', description: @@ -99,7 +106,7 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{ output_schema: { type: 'object', description: - 'JSON schema that the AI agent will follow for its response format (only used if output_type is text)', + 'JSON schema that the AI agent will follow for its response format (only used if output_type is text).', format: 'json-schema', showExpr: "fields.output_type === 'text'" } diff --git a/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte index 6c8c70d9ed..47a099b7c8 100644 --- a/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte @@ -4,28 +4,43 @@ import type { Writable } from 'svelte/store' import { twMerge } from 'tailwind-merge' - export let sourceX: number - export let sourceY: number - export let sourcePosition: Position - export let targetX: number - export let targetY: number - export let targetPosition: Position - export let markerEnd: string | undefined = undefined - export let data: { class?: string } = {} + interface Props { + sourceX: number + sourceY: number + sourcePosition: Position + targetX: number + targetY: number + targetPosition: Position + markerEnd?: string | undefined + data?: { class?: string } + } - const { useDataflow } = getContext<{ - useDataflow: Writable - }>('FlowGraphContext') - - $: [edgePath] = getBezierPath({ + let { sourceX, sourceY, sourcePosition, targetX, targetY, targetPosition, - curvature: 0.25 - }) + markerEnd = undefined, + data = {} + }: Props = $props() + + const { useDataflow } = getContext<{ + useDataflow: Writable + }>('FlowGraphContext') + + let [edgePath] = $derived( + getBezierPath({ + sourceX, + sourceY, + sourcePosition, + targetX, + targetY, + targetPosition, + curvature: 0.25 + }) + ) n.type !== 'asset') + .map((n) => ({ id: n.id, position: $state.snapshot(n.position) })) .sort((a, b) => a.position.y - b.position.y) let currentYOffset = 0 let prevYPos = NaN @@ -234,7 +235,6 @@ import { MessageCircle, Play, Wrench, X } from 'lucide-svelte' import { twMerge } from 'tailwind-merge' import { getContext } from 'svelte' - import { clone } from '$lib/utils' import type { Edge, Node } from '@xyflow/svelte' import type { Writable } from 'svelte/store' diff --git a/frontend/src/lib/components/jobs/JobProgressBar.svelte b/frontend/src/lib/components/jobs/JobProgressBar.svelte index 4b4684e32a..50c8cc4293 100644 --- a/frontend/src/lib/components/jobs/JobProgressBar.svelte +++ b/frontend/src/lib/components/jobs/JobProgressBar.svelte @@ -2,38 +2,47 @@ import { type Job } from '$lib/gen' import ProgressBar from '../progressBar/ProgressBar.svelte' - export let job: Job | undefined = undefined - export let compact: boolean = false; - /// Progress of currently running job - export let scriptProgress: number | undefined = undefined; - // Removes `Step 1` and replaces it with `Running` - export let hideStepTitle: boolean = false - - let error: number | undefined = undefined - let index = 0 - let subIndex: number = 0 - let subLength: number = 100 - let length = 1 - let nextInProgress = false - - $: if (job) updateJobProgress(job); - $: subIndex = scriptProgress ?? 0; - - function updateJobProgress(job: Job) { - if (!job['running'] && !job['success']){ - error = 0; - } else { - error = undefined; - } - // Anything that is success automatically gets 100% progress - if (job['success'] && scriptProgress) - index = 1, subLength = 0, subIndex = 0, scriptProgress = 100; + interface Props { + job?: Job | undefined + compact?: boolean + /// Progress of currently running job + scriptProgress?: number | undefined + // Removes `Step 1` and replaces it with `Running` + hideStepTitle?: boolean + class?: string } - let resetP: any + let { + job = undefined, + compact = false, + scriptProgress = undefined, + hideStepTitle = false, + class: className = '' + }: Props = $props() + + let error: number | undefined = $state(undefined) + let index = $state(0) + let subIndex: number = $state(0) + let subLength: number = $state(100) + let length = $state(1) + let nextInProgress = false + + let progressBar: ProgressBar | undefined = $state(undefined) + let lastJobId = $state() + + function updateJobProgress(job: Job) { + if (!job['running'] && !job['success']) { + error = 0 + } else { + error = undefined + } + // Anything that is success automatically gets 100% progress + if (job['success'] && scriptProgress) + ((index = 1), (subLength = 0), (subIndex = 0), (scriptProgress = 100)) + } export function reset() { - resetP?.() + progressBar?.resetP() error = undefined subIndex = 0 subLength = 100 @@ -42,17 +51,30 @@ scriptProgress = undefined } + $effect(() => { + if (lastJobId && job && job.id !== lastJobId) { + lastJobId = job.id + reset() + } + }) + + $effect(() => { + if (job) updateJobProgress(job) + }) + $effect(() => { + subIndex = scriptProgress ?? 0 + }) diff --git a/frontend/src/lib/components/meltComponents/MenuSingleItem.svelte b/frontend/src/lib/components/meltComponents/MenuSingleItem.svelte index 65ec922bfb..83a78d878e 100644 --- a/frontend/src/lib/components/meltComponents/MenuSingleItem.svelte +++ b/frontend/src/lib/components/meltComponents/MenuSingleItem.svelte @@ -1,24 +1,11 @@
- {@render triggr?.({ trigger })} -
- {@render children?.({ item })} -
+ {@render children?.({})}
diff --git a/frontend/src/lib/components/progressBar/ProgressBar.svelte b/frontend/src/lib/components/progressBar/ProgressBar.svelte index eaa2e8eadd..bb0c6b5328 100644 --- a/frontend/src/lib/components/progressBar/ProgressBar.svelte +++ b/frontend/src/lib/components/progressBar/ProgressBar.svelte @@ -1,77 +1,88 @@ -
+
{#if !compact} -
- - {error != undefined - ? 'Error occured' - : finished - ? 'Done' - : hideStepTitle - ? `Running` - : subIndexIsPercent - ? `Step ${index + 1} (${subIndex !== undefined ? `${subIndex}%)` : ''}` - : `Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`} - - - {$percent.toFixed(0)}% - -
+
+ + {error != undefined + ? 'Error occured' + : finished + ? 'Done' + : hideStepTitle + ? `Running` + : subIndexIsPercent + ? `Step ${index + 1} (${subIndex !== undefined ? `${subIndex}%)` : ''}` + : `Step ${index + 1}${subIndex !== undefined ? `.${subIndex + 1}` : ''}`} + + + {percent.current.toFixed(0)}% + +
{/if} -
+
{#each new Array(length) as _, partIndex (partIndex)}
{#if partIndex == index && nextInProgress} @@ -101,7 +114,7 @@ class="absolute left-0 bottom-0 h-full {error == partIndex ? 'bg-red-400' : 'bg-blue-400'}" - style="width: {getPercent(partIndex, $percent)}%" + style="width: {getPercent(partIndex, percent.current)}%" >
{/if}
diff --git a/frontend/src/lib/components/runs/NoWorkerWithTagWarning.svelte b/frontend/src/lib/components/runs/NoWorkerWithTagWarning.svelte index a44dbc763f..061cd84bf9 100644 --- a/frontend/src/lib/components/runs/NoWorkerWithTagWarning.svelte +++ b/frontend/src/lib/components/runs/NoWorkerWithTagWarning.svelte @@ -4,7 +4,7 @@ import Popover from '../Popover.svelte' import { onDestroy, untrack } from 'svelte' interface Props { - tag: string + tag: string | undefined tagLabel?: string } @@ -17,9 +17,9 @@ let visible = true let customTag = $derived.by(() => { - if (tag.includes('$workspace') || tag.includes('$args')) return + if (tag?.includes('$workspace') || tag?.includes('$args')) return - if (tag.includes('(')) { + if (tag?.includes('(')) { return tag.split('(')[0] } return tag @@ -44,7 +44,7 @@ $effect(() => { customTag - untrack(() => lookForTag()) + untrack(() => timeout && setTimeout(() => lookForTag(), 2500)) }) onDestroy(() => { diff --git a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte index 48dc119171..89b8c3f1af 100644 --- a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte +++ b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte @@ -68,6 +68,25 @@ (filteredUsers || users || []).some((user: User) => user.added_via?.source === 'instance_group' || user.added_via?.source === 'domain') ) + // Function to check if a manual user can be converted to a group user + function canConvertToGroup(user: User): boolean { + // User must be manually added (not via instance group or domain) + if (user.added_via?.source === 'instance_group' || user.added_via?.source === 'domain') { + return false + } + + // Check if user's email is in any configured instance group + const userEmail = user.email + for (const groupName of autoAddInstanceGroups) { + const group = instanceGroups.find(g => g.name === groupName) + if (group && group.emails && group.emails.includes(userEmail)) { + return true + } + } + + return false + } + async function loadSettings(): Promise { const settings = await WorkspaceService.getSettings({ workspace: $workspaceStore! }) @@ -195,6 +214,20 @@ } } + async function convertUserToGroup(username: string): Promise { + try { + await UserService.convertUserToGroup({ + workspace: $workspaceStore ?? '', + username + }) + sendUserToast('User converted to group user') + listUsers() + } catch (e) { + console.error('Failed to convert user:', e) + sendUserToast('Failed to convert user', true) + } + } + let domain = $derived($userStore?.email.split('@')[1]) $effect(() => { @@ -220,6 +253,7 @@ let deleteConfirmedCallback: (() => void) | undefined = $state(undefined) let removeInstanceGroupConfirmedCallback: (() => void) | undefined = $state(undefined) + let convertConfirmedCallback: (() => void) | undefined = $state(undefined) async function removeAllInvitesFromDomain() { await Promise.all( @@ -634,7 +668,8 @@ {#if filteredUsers} - {#each sortedUsers().slice(0, nbDisplayed) as { email, username, is_admin, operator, disabled, added_via }, index (email)} + {#each sortedUsers().slice(0, nbDisplayed) as user, index (user.email)} + {@const { email, username, is_admin, operator, disabled, added_via } = user} {#if hasNonManualUsers && index > 0 && sortedUsers()[index - 1]?.added_via?.source !== 'instance_group' && added_via?.source === 'instance_group'} @@ -774,6 +809,40 @@ Cannot remove users synced from instance groups. Either disable the user or remove them from the SCIM group.
+ {:else if canConvertToGroup(user)} + + {:else}
+ + { + convertConfirmedCallback = undefined + }} + on:confirmed={() => { + if (convertConfirmedCallback) { + convertConfirmedCallback() + } + convertConfirmedCallback = undefined + }} +> +
+ Are you sure you want to convert this user to a group user? + This will: +
    +
  • Change the user's role based on their instance group configuration
    • +
  • Make their role managed through the instance group settings
    • +
  • Prevent manual role changes for this user
    • +
+
+ diff --git a/frontend/src/lib/components/sidebar/MenuLink.svelte b/frontend/src/lib/components/sidebar/MenuLink.svelte index 82188ac9fc..9ecdc65072 100644 --- a/frontend/src/lib/components/sidebar/MenuLink.svelte +++ b/frontend/src/lib/components/sidebar/MenuLink.svelte @@ -19,6 +19,7 @@ lightMode?: boolean item?: MenubarMenuElements['item'] | undefined class?: string + onclick?: (ev: MouseEvent) => any } let { @@ -31,7 +32,8 @@ disabled = false, lightMode = false, item = undefined, - class: classNames = '' + class: classNames = '', + onclick = undefined }: Props = $props() let isSelected = $state(false) @@ -56,6 +58,7 @@ goto(href) } }} + {onclick} class={twMerge( 'group flex items-center px-2 py-2 text-sm font-light rounded-md h-8 gap-3', isSelected diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index f45c698be5..bdbb17ba3f 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -172,7 +172,12 @@ href: `${base}/runs`, icon: Play, aiId: 'sidebar-menu-link-runs', - aiDescription: 'Button to navigate to runs' + aiDescription: 'Button to navigate to runs', + onclick: () => { + setTimeout(() => { + window.dispatchEvent(new Event('popstate')) + }, 100) + } }, { label: 'Variables', @@ -359,7 +364,7 @@ } ] : []), - ...($workspaceStore?.startsWith("wm-fork") + ...($workspaceStore?.startsWith('wm-fork') ? [ { label: 'Delete Forked Workspace', @@ -370,7 +375,7 @@ faIcon: undefined } ] - : []), + : []) ], disabled: $userStore?.operator }, @@ -572,14 +577,9 @@ {/snippet} {:else} - - {#snippet triggr({ trigger })} -
- -
- {/snippet} - {#snippet children({ item })} - + + {#snippet children({})} + {/snippet} {/if} @@ -665,20 +665,20 @@
-{#if $workspaceStore?.startsWith("wm-fork-")} - { - deleteWorkspaceForkModal = false - }} - on:confirmed={() => { - deleteFork() - }} -> -
- Are you sure you want to delete this workspace fork? (deleting {$workspaceStore}) -
- +{#if $workspaceStore?.startsWith('wm-fork-')} + { + deleteWorkspaceForkModal = false + }} + on:confirmed={() => { + deleteFork() + }} + > +
+ Are you sure you want to delete this workspace fork? (deleting {$workspaceStore}) +
+ {/if} diff --git a/frontend/src/lib/components/table/Cell.svelte b/frontend/src/lib/components/table/Cell.svelte index ab43a10c2f..d5fc30f811 100644 --- a/frontend/src/lib/components/table/Cell.svelte +++ b/frontend/src/lib/components/table/Cell.svelte @@ -3,25 +3,42 @@ import { twMerge } from 'tailwind-merge' import type { DatatableContext } from './DataTable.svelte' - export let first: boolean = false - export let last: boolean = false - export let numeric: boolean = false - export let head: boolean = false - export let shouldStopPropagation: boolean = false - export let selected = false - export let sticky: boolean = false - export let wrap: boolean = false + interface Props { + first?: boolean + last?: boolean + numeric?: boolean + head?: boolean + shouldStopPropagation?: boolean + selected?: boolean + sticky?: boolean + wrap?: boolean + children?: import('svelte').Snippet + [key: string]: any + } + + let { + first = false, + last = false, + numeric = false, + head = false, + shouldStopPropagation = false, + selected = false, + sticky = false, + wrap = false, + children, + ...rest + }: Props = $props() let Tag = head ? 'th' : 'td' const { size } = getContext('datatable') - + { + {...rest} + onclick={(e) => { if (shouldStopPropagation) e.stopPropagation() }} class={twMerge( @@ -41,14 +58,14 @@ size === 'xs' ? 'px-1 py-1.5' : '', selected ? 'bg-blue-50 dark:bg-blue-900/50' : '', 'transition-all', - $$restProps.class + rest.class )} > {#if sticky}
- + {@render children?.()}
{:else} - + {@render children?.()} {/if} diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 1f4c65d897..8e84df9a9f 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -172,9 +172,9 @@ export function displayDate( } const dateChoices: Intl.DateTimeFormatOptions = displayDate ? { - day: 'numeric', - month: 'numeric' - } + day: 'numeric', + month: 'numeric' + } : {} return date.toLocaleString(undefined, { ...timeChoices, @@ -1065,25 +1065,24 @@ export async function tryEvery({ try { await tryCode() break - } catch (err) {} + } catch (err) { } i++ } if (i >= times) { timeoutCode() } } - -export function roughSizeOfObject(object: object | string) { - if (typeof object == 'string') { +export function roughSizeOfObject(object: object | string | any) { + if (typeof object === 'string') { return object.length * 2 } - var objectList: any[] = [] - var stack = [object] - var bytes = 0 + const visited = new Set() + const stack = [object] + let bytes = 0 while (stack.length) { - let value: any = stack.pop() + const value = stack.pop() if (typeof value === 'boolean') { bytes += 4 @@ -1091,12 +1090,12 @@ export function roughSizeOfObject(object: object | string) { bytes += value.length * 2 } else if (typeof value === 'number') { bytes += 8 - } else if (typeof value === 'object' && objectList.indexOf(value) === -1) { - objectList.push(value) + } else if (typeof value === 'object' && value !== null && !visited.has(value)) { + visited.add(value) - for (var i in value) { - bytes += 2 * i.length - stack.push(value[i]) + for (const key in value) { + bytes += 2 * key.length + stack.push(value[key]) } } } @@ -1332,7 +1331,7 @@ export function conditionalMelt(node: HTMLElement, meltItem: AnyMeltElement | un if (meltItem) { return meltItem(node) } - return { destroy: () => {} } + return { destroy: () => { } } } export type Item = { @@ -1538,9 +1537,9 @@ export type S3Uri = `s3://${string}/${string}` export type S3Object = | S3Uri | { - s3: string - storage?: string - } + s3: string + storage?: string + } export function parseS3Object(s3Object: S3Object): { s3: string; storage?: string } { if (typeof s3Object === 'object') return s3Object diff --git a/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte b/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte index 30fd36ace4..e56a63a4a3 100644 --- a/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte @@ -170,6 +170,7 @@ bind:usernameFilter={username} bind:resourceFilter={resource} bind:hasMore + showWorkspace={scope === 'instance' || scope === 'all_workspaces'} onselect={(id) => { selectedId = id }} @@ -200,6 +201,7 @@ bind:operation bind:usernameFilter={username} bind:resourceFilter={resource} + showWorkspace={scope === 'instance' || scope === 'all_workspaces'} onselect={(id) => { selectedId = id auditLogDrawer?.openDrawer() diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index e1b69d283d..95848b37c4 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -222,7 +222,11 @@ async function onJobLoaded() { // We want to set up scriptProgress once job is loaded // We need this to show progress bar if job has progress and is finished - if (job && job.type == 'CompletedJob') { + if ( + job && + job.type == 'CompletedJob' && + (job.job_kind == 'script' || isScriptPreview(job.job_kind)) + ) { // If error occured and job is completed // than we fetch progress from server to display on what progress did it fail // Could be displayed after run or as a historical page diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 153d8772ef..c6889d7466 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -66,6 +66,7 @@ let allowWildcards: boolean = $state(page.url.searchParams.get('allow_wildcards') == 'true') let concurrencyKey: string | null = $state(page.url.searchParams.get('concurrency_key')) let tag: string | null = $state(page.url.searchParams.get('tag')) + // Rest of filters handled by RunsFilter let success: 'running' | 'suspended' | 'waiting' | 'success' | 'failure' | undefined = $state( (page.url.searchParams.get('success') ?? undefined) as @@ -350,6 +351,7 @@ } function reset() { + path = page.params.path ?? null minTs = undefined maxTs = undefined jobs = undefined diff --git a/lsp/Pipfile b/lsp/Pipfile index b359fd21e7..aa780ab376 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.548.1" -wmill_pg = ">=1.548.1" +wmill = ">=1.549.1" +wmill_pg = ">=1.549.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index eb9ff3048b..51ab9602f9 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.548.1 + version: 1.549.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 5f7d3fc6c8..58f0eb5910 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.548.1' + ModuleVersion = '1.549.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 1d2d51d46a..65af03be1b 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.548.1" +version = "1.549.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 1a9c263403..f254c759b6 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.548.1" +version = "1.549.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 7643f04807..27acb9a113 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.548.1", + "version": "1.549.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 5c42159de7..51d542b0f4 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.548.1", + "version": "1.549.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index e5158c8391..d9f6023cbf 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.548.1 +1.549.1