From 234b20f8bd55ea19b17b80f08d9ff1e0e00ba739 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Tue, 25 Mar 2025 19:53:48 +0100 Subject: [PATCH 001/133] feat: autocomplete v2 + AI chat (#5323) * feat: autocomplete v2 * wip chat * wip * feat: chat ai review and apply * feat: multiple providers * update cli gen * fixes * improvements * fix build * fix issues * fix build * final nits * nits * nits --- ...172ea9a70835bfba6cef56142556197e9767.json} | 6 +- ...8234ca7d1efeee9661f3901f298da375e73f7.json | 18 +- ...290d810a5e8ea6458d9f9fd484ced549ea82e.json | 29 - ...850bed8f435ea5de16ba796f346fba51d9437.json | 22 + ...bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json | 15 - ...05db96faeec6c18500ad96087cd06c2c85a8b.json | 22 + ...70aafd27ef61c940daba186e7e66f668c31ed.json | 17 - ...dcc40f463cbc52d94ed9315cf9a547d4c89f2.json | 18 +- ...23919cee3cc444725ac6b7906922554bae800.json | 23 + ...c496f1757d950cdec63c93c77fe79d5212cc.json} | 4 +- ...122fd3eae99a382ca3e66606d3183de51843.json} | 40 +- ...085502a1e77e30e59c7310c78976f77a810a3.json | 34 -- ...0c624cd1b00bcab9855edb9284d8afec46ff0.json | 15 + ...fe2635785b53d31d0ceb3d72226a0c316e8a.json} | 36 +- ...5859c924cffa61074562ac6d33106d02d7d6.json} | 4 +- ...fb565f04539fc7eb141ebfaa957456016841.json} | 4 +- ...a9442c35453dff95140d2f252c4fe6a14c6a4.json | 22 - ...50319121050_multiple_ai_providers.down.sql | 24 + ...0250319121050_multiple_ai_providers.up.sql | 38 ++ backend/windmill-api/openapi.yaml | 74 ++- backend/windmill-api/src/ai.rs | 414 +++++++------ backend/windmill-api/src/lib.rs | 1 + backend/windmill-api/src/workspaces.rs | 142 ++--- backend/windmill-api/src/workspaces_export.rs | 37 +- cli/gen/core/OpenAPI.ts | 2 +- cli/gen/services.gen.ts | 2 +- cli/gen/types.gen.ts | 34 +- cli/settings.ts | 18 +- frontend/package-lock.json | 158 ++--- frontend/package.json | 8 +- frontend/src/lib/components/Dev.svelte | 21 +- frontend/src/lib/components/Editor.svelte | 202 ++++--- .../src/lib/components/FlowBuilder.svelte | 11 +- .../src/lib/components/HighlightCode.svelte | 32 +- .../src/lib/components/ScriptEditor.svelte | 196 +++++- frontend/src/lib/components/Toggle.svelte | 2 +- .../editor/settingsPanel/HideButton.svelte | 43 +- .../copilot/CodeCompletionStatus.svelte | 6 +- .../src/lib/components/copilot/CronGen.svelte | 10 +- .../copilot/FlowCopilotStatus.svelte | 2 +- .../lib/components/copilot/IteratorGen.svelte | 10 +- .../lib/components/copilot/MetadataGen.svelte | 11 +- .../components/copilot/PredicateGen.svelte | 6 +- .../lib/components/copilot/RegexGen.svelte | 6 +- .../lib/components/copilot/ScriptFix.svelte | 16 +- .../lib/components/copilot/ScriptGen.svelte | 35 +- .../src/lib/components/copilot/StepGen.svelte | 2 +- .../components/copilot/StepInputGen.svelte | 10 +- .../components/copilot/StepInputsGen.svelte | 8 +- .../copilot/autocomplete/monaco-adapter.ts | 567 ++++++++++++++++++ .../copilot/autocomplete/request.ts | 103 ++++ .../components/copilot/autocomplete/widget.ts | 126 ++++ .../lib/components/copilot/chat/AIChat.svelte | 372 ++++++++++++ .../copilot/chat/AIChatDisplay.svelte | 283 +++++++++ .../copilot/chat/AssistantMessage.svelte | 29 + .../copilot/chat/CodeDisplay.svelte | 206 +++++++ .../copilot/chat/ContextElementBadge.svelte | 79 +++ .../copilot/chat/GlobalReviewButtons.svelte | 19 + .../src/lib/components/copilot/chat/core.ts | 492 +++++++++++++++ .../components/copilot/chat/monaco-adapter.ts | 221 +++++++ .../src/lib/components/copilot/completion.ts | 110 ---- frontend/src/lib/components/copilot/flow.ts | 13 +- frontend/src/lib/components/copilot/lib.ts | 492 ++++----------- frontend/src/lib/components/copilot/shared.ts | 295 +++++++++ frontend/src/lib/components/copilot/utils.ts | 2 +- .../flows/content/FlowInputsQuick.svelte | 2 +- .../flows/map/FlowModuleSchemaMap.svelte | 2 +- .../components/scriptEditor/LogPanel.svelte | 3 + frontend/src/lib/stores.ts | 54 +- .../src/routes/(root)/(logged)/+layout.svelte | 18 +- .../user/(user)/create_workspace/+page.svelte | 22 +- .../(logged)/workspace_settings/+page.svelte | 286 +++++---- typescript-client/build.jsr.sh | 2 +- typescript-client/build.sh | 2 +- 74 files changed, 4185 insertions(+), 1525 deletions(-) rename backend/.sqlx/{query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json => query-0689cdc6c7676f5e1984792a0e0b172ea9a70835bfba6cef56142556197e9767.json} (56%) delete mode 100644 backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json create mode 100644 backend/.sqlx/query-197590e7ab74f64bbf374f23128850bed8f435ea5de16ba796f346fba51d9437.json delete mode 100644 backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json create mode 100644 backend/.sqlx/query-3734117167a1269f78b3949eed005db96faeec6c18500ad96087cd06c2c85a8b.json delete mode 100644 backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json create mode 100644 backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json rename backend/.sqlx/{query-31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5.json => query-9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc.json} (54%) rename backend/.sqlx/{query-4e9c2e0690eaca280ccb5e5160438f207d930d29e3000b9845eef89e87a35ad1.json => query-aff243a11d41085b0b2f52b1b49f122fd3eae99a382ca3e66606d3183de51843.json} (83%) delete mode 100644 backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json create mode 100644 backend/.sqlx/query-ba117e89b5da7ef46a9e702a2f80c624cd1b00bcab9855edb9284d8afec46ff0.json rename backend/.sqlx/{query-dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727.json => query-c12a0b0d423577afbb2772f971bffe2635785b53d31d0ceb3d72226a0c316e8a.json} (56%) rename backend/.sqlx/{query-cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32.json => query-e3c8219420bb859de3f1c322978c5859c924cffa61074562ac6d33106d02d7d6.json} (66%) rename backend/.sqlx/{query-82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8.json => query-e5f1e1e74daeabf410991e2484c1fb565f04539fc7eb141ebfaa957456016841.json} (50%) delete mode 100644 backend/.sqlx/query-f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4.json create mode 100644 backend/migrations/20250319121050_multiple_ai_providers.down.sql create mode 100644 backend/migrations/20250319121050_multiple_ai_providers.up.sql create mode 100644 frontend/src/lib/components/copilot/autocomplete/monaco-adapter.ts create mode 100644 frontend/src/lib/components/copilot/autocomplete/request.ts create mode 100644 frontend/src/lib/components/copilot/autocomplete/widget.ts create mode 100644 frontend/src/lib/components/copilot/chat/AIChat.svelte create mode 100644 frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte create mode 100644 frontend/src/lib/components/copilot/chat/AssistantMessage.svelte create mode 100644 frontend/src/lib/components/copilot/chat/CodeDisplay.svelte create mode 100644 frontend/src/lib/components/copilot/chat/ContextElementBadge.svelte create mode 100644 frontend/src/lib/components/copilot/chat/GlobalReviewButtons.svelte create mode 100644 frontend/src/lib/components/copilot/chat/core.ts create mode 100644 frontend/src/lib/components/copilot/chat/monaco-adapter.ts delete mode 100644 frontend/src/lib/components/copilot/completion.ts create mode 100644 frontend/src/lib/components/copilot/shared.ts diff --git a/backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json b/backend/.sqlx/query-0689cdc6c7676f5e1984792a0e0b172ea9a70835bfba6cef56142556197e9767.json similarity index 56% rename from backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json rename to backend/.sqlx/query-0689cdc6c7676f5e1984792a0e0b172ea9a70835bfba6cef56142556197e9767.json index a309a03762..27c1d1d2d8 100644 --- a/backend/.sqlx/query-ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8.json +++ b/backend/.sqlx/query-0689cdc6c7676f5e1984792a0e0b172ea9a70835bfba6cef56142556197e9767.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", + "query": "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", "describe": { "columns": [ { "ordinal": 0, - "name": "ai_resource", + "name": "ai_config", "type_info": "Jsonb" } ], @@ -18,5 +18,5 @@ true ] }, - "hash": "ceda377fa534656ac12a7f41db77db1f054ec751855c8db7352b0e9c20b63ef8" + "hash": "0689cdc6c7676f5e1984792a0e0b172ea9a70835bfba6cef56142556197e9767" } diff --git a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json index 4bcf3c6ce3..3d07525fa0 100644 --- a/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json +++ b/backend/.sqlx/query-08f288d2781d823e109a9e5b8848234ca7d1efeee9661f3901f298da375e73f7.json @@ -65,7 +65,7 @@ }, { "ordinal": 12, - "name": "ai_resource", + "name": "ai_config", "type_info": "Jsonb" }, { @@ -130,26 +130,16 @@ }, { "ordinal": 25, - "name": "ai_models", - "type_info": "VarcharArray" - }, - { - "ordinal": 26, - "name": "code_completion_model", - "type_info": "Varchar" - }, - { - "ordinal": 27, "name": "teams_command_script", "type_info": "Text" }, { - "ordinal": 28, + "ordinal": 26, "name": "teams_team_id", "type_info": "Text" }, { - "ordinal": 29, + "ordinal": 27, "name": "teams_team_name", "type_info": "Text" } @@ -185,8 +175,6 @@ true, true, true, - false, - true, true, true, true diff --git a/backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json b/backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json deleted file mode 100644 index 15d772ab16..0000000000 --- a/backend/.sqlx/query-103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT value, resource_type FROM resource WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "value", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "resource_type", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "103ef3cf5cf4d25d780e4aefd5b290d810a5e8ea6458d9f9fd484ced549ea82e" -} diff --git a/backend/.sqlx/query-197590e7ab74f64bbf374f23128850bed8f435ea5de16ba796f346fba51d9437.json b/backend/.sqlx/query-197590e7ab74f64bbf374f23128850bed8f435ea5de16ba796f346fba51d9437.json new file mode 100644 index 0000000000..28d629c587 --- /dev/null +++ b/backend/.sqlx/query-197590e7ab74f64bbf374f23128850bed8f435ea5de16ba796f346fba51d9437.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT ai_config as \"ai_config: sqlx::types::Json\" FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "ai_config: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "197590e7ab74f64bbf374f23128850bed8f435ea5de16ba796f346fba51d9437" +} diff --git a/backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json b/backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json deleted file mode 100644 index 4fcd1f0969..0000000000 --- a/backend/.sqlx/query-1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET ai_resource = NULL, code_completion_model = $1, ai_models = '{}' WHERE workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1bbef6baa5b8e2522d685df2979bb1e4b9022f5e841afd9eeb08a81688f6c0c8" -} diff --git a/backend/.sqlx/query-3734117167a1269f78b3949eed005db96faeec6c18500ad96087cd06c2c85a8b.json b/backend/.sqlx/query-3734117167a1269f78b3949eed005db96faeec6c18500ad96087cd06c2c85a8b.json new file mode 100644 index 0000000000..6a3afafb5e --- /dev/null +++ b/backend/.sqlx/query-3734117167a1269f78b3949eed005db96faeec6c18500ad96087cd06c2c85a8b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS (SELECT 1\n FROM workspace_settings\n WHERE workspace_id <> $1\n AND slack_command_script IS NOT NULL\n AND slack_team_id IS NOT NULL\n AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1))\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3734117167a1269f78b3949eed005db96faeec6c18500ad96087cd06c2c85a8b" +} diff --git a/backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json b/backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json deleted file mode 100644 index 4eae8c22fb..0000000000 --- a/backend/.sqlx/query-4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE workspace_settings SET ai_resource = $1, code_completion_model = $2, ai_models = $3 WHERE workspace_id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Varchar", - "VarcharArray", - "Text" - ] - }, - "nullable": [] - }, - "hash": "4edf05cbf35325d444de0e74ee070aafd27ef61c940daba186e7e66f668c31ed" -} diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json index 14685a8bfa..3d06725f17 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json @@ -65,7 +65,7 @@ }, { "ordinal": 12, - "name": "ai_resource", + "name": "ai_config", "type_info": "Jsonb" }, { @@ -130,26 +130,16 @@ }, { "ordinal": 25, - "name": "ai_models", - "type_info": "VarcharArray" - }, - { - "ordinal": 26, - "name": "code_completion_model", - "type_info": "Varchar" - }, - { - "ordinal": 27, "name": "teams_command_script", "type_info": "Text" }, { - "ordinal": 28, + "ordinal": 26, "name": "teams_team_id", "type_info": "Text" }, { - "ordinal": 29, + "ordinal": 27, "name": "teams_team_name", "type_info": "Text" } @@ -185,8 +175,6 @@ true, true, true, - false, - true, true, true, true diff --git a/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json b/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json new file mode 100644 index 0000000000..4abea8d7e5 --- /dev/null +++ b/backend/.sqlx/query-63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "63c16a4277983aaed0aed54972923919cee3cc444725ac6b7906922554bae800" +} diff --git a/backend/.sqlx/query-31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5.json b/backend/.sqlx/query-9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc.json similarity index 54% rename from backend/.sqlx/query-31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5.json rename to backend/.sqlx/query-9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc.json index 0871986a4e..2b30281364 100644 --- a/backend/.sqlx/query-31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5.json +++ b/backend/.sqlx/query-9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ", + "query": "\n SELECT\n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\",\n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\",\n EXISTS(SELECT 1 FROM mqtt_trigger WHERE workspace_id = $1) AS \"mqtt_used!\",\n EXISTS(SELECT 1 FROM sqs_trigger WHERE workspace_id = $1) AS \"sqs_used!\"\n ", "describe": { "columns": [ { @@ -54,5 +54,5 @@ null ] }, - "hash": "31b6fccad46b22bcbba6bbce22209ccb1825116004ed72682854ce3352f454a5" + "hash": "9ae20f2f29406cfa5337736ad710c496f1757d950cdec63c93c77fe79d5212cc" } diff --git a/backend/.sqlx/query-4e9c2e0690eaca280ccb5e5160438f207d930d29e3000b9845eef89e87a35ad1.json b/backend/.sqlx/query-aff243a11d41085b0b2f52b1b49f122fd3eae99a382ca3e66606d3183de51843.json similarity index 83% rename from backend/.sqlx/query-4e9c2e0690eaca280ccb5e5160438f207d930d29e3000b9845eef89e87a35ad1.json rename to backend/.sqlx/query-aff243a11d41085b0b2f52b1b49f122fd3eae99a382ca3e66606d3183de51843.json index 3dd496788f..a38b54e197 100644 --- a/backend/.sqlx/query-4e9c2e0690eaca280ccb5e5160438f207d930d29e3000b9845eef89e87a35ad1.json +++ b/backend/.sqlx/query-aff243a11d41085b0b2f52b1b49f122fd3eae99a382ca3e66606d3183de51843.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_resource, ai_models, code_completion_model, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, automatic_billing, default_scripts, mute_critical_alerts, color, operator_settings FROM workspace_settings WHERE workspace_id = $1", + "query": "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, automatic_billing, default_scripts, mute_critical_alerts, color, operator_settings FROM workspace_settings WHERE workspace_id = $1", "describe": { "columns": [ { @@ -80,76 +80,66 @@ }, { "ordinal": 15, - "name": "ai_resource", + "name": "ai_config", "type_info": "Jsonb" }, { "ordinal": 16, - "name": "ai_models", - "type_info": "VarcharArray" - }, - { - "ordinal": 17, - "name": "code_completion_model", - "type_info": "Varchar" - }, - { - "ordinal": 18, "name": "error_handler", "type_info": "Varchar" }, { - "ordinal": 19, + "ordinal": 17, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 20, + "ordinal": 18, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 21, + "ordinal": 19, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 22, + "ordinal": 20, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 23, + "ordinal": 21, "name": "deploy_ui", "type_info": "Jsonb" }, { - "ordinal": 24, + "ordinal": 22, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 25, + "ordinal": 23, "name": "automatic_billing", "type_info": "Bool" }, { - "ordinal": 26, + "ordinal": 24, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 27, + "ordinal": 25, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 28, + "ordinal": 26, "name": "color", "type_info": "Varchar" }, { - "ordinal": 29, + "ordinal": 27, "name": "operator_settings", "type_info": "Jsonb" } @@ -176,8 +166,6 @@ true, true, true, - false, - true, true, true, false, @@ -192,5 +180,5 @@ true ] }, - "hash": "4e9c2e0690eaca280ccb5e5160438f207d930d29e3000b9845eef89e87a35ad1" + "hash": "aff243a11d41085b0b2f52b1b49f122fd3eae99a382ca3e66606d3183de51843" } diff --git a/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json b/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json deleted file mode 100644 index d856ab0109..0000000000 --- a/backend/.sqlx/query-b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT ai_resource, code_completion_model, ai_models FROM workspace_settings WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "ai_resource", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "code_completion_model", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "ai_models", - "type_info": "VarcharArray" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - true, - true, - false - ] - }, - "hash": "b186efe51e7bb1924efbd7e9b36085502a1e77e30e59c7310c78976f77a810a3" -} diff --git a/backend/.sqlx/query-ba117e89b5da7ef46a9e702a2f80c624cd1b00bcab9855edb9284d8afec46ff0.json b/backend/.sqlx/query-ba117e89b5da7ef46a9e702a2f80c624cd1b00bcab9855edb9284d8afec46ff0.json new file mode 100644 index 0000000000..773dc5d24d --- /dev/null +++ b/backend/.sqlx/query-ba117e89b5da7ef46a9e702a2f80c624cd1b00bcab9855edb9284d8afec46ff0.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "ba117e89b5da7ef46a9e702a2f80c624cd1b00bcab9855edb9284d8afec46ff0" +} diff --git a/backend/.sqlx/query-dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727.json b/backend/.sqlx/query-c12a0b0d423577afbb2772f971bffe2635785b53d31d0ceb3d72226a0c316e8a.json similarity index 56% rename from backend/.sqlx/query-dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727.json rename to backend/.sqlx/query-c12a0b0d423577afbb2772f971bffe2635785b53d31d0ceb3d72226a0c316e8a.json index 3a607fbf31..ca8fd862d2 100644 --- a/backend/.sqlx/query-dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727.json +++ b/backend/.sqlx/query-c12a0b0d423577afbb2772f971bffe2635785b53d31d0ceb3d72226a0c316e8a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n ai_models,\n code_completion_model,\n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name,\n mute_critical_alerts,\n color,\n operator_settings\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", + "query": "SELECT\n -- slack_team_id,\n -- slack_name,\n -- slack_command_script,\n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\",\n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\",\n webhook,\n deploy_to,\n error_handler,\n ai_config,\n error_handler_extra_args,\n error_handler_muted_on_cancel,\n large_file_storage,\n git_sync,\n default_app,\n default_scripts,\n workspace.name,\n mute_critical_alerts,\n color,\n operator_settings\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1", "describe": { "columns": [ { @@ -35,66 +35,56 @@ }, { "ordinal": 6, - "name": "ai_resource", + "name": "ai_config", "type_info": "Jsonb" }, { "ordinal": 7, - "name": "ai_models", - "type_info": "VarcharArray" - }, - { - "ordinal": 8, - "name": "code_completion_model", - "type_info": "Varchar" - }, - { - "ordinal": 9, "name": "error_handler_extra_args", "type_info": "Json" }, { - "ordinal": 10, + "ordinal": 8, "name": "error_handler_muted_on_cancel", "type_info": "Bool" }, { - "ordinal": 11, + "ordinal": 9, "name": "large_file_storage", "type_info": "Jsonb" }, { - "ordinal": 12, + "ordinal": 10, "name": "git_sync", "type_info": "Jsonb" }, { - "ordinal": 13, + "ordinal": 11, "name": "default_app", "type_info": "Varchar" }, { - "ordinal": 14, + "ordinal": 12, "name": "default_scripts", "type_info": "Jsonb" }, { - "ordinal": 15, + "ordinal": 13, "name": "name", "type_info": "Varchar" }, { - "ordinal": 16, + "ordinal": 14, "name": "mute_critical_alerts", "type_info": "Bool" }, { - "ordinal": 17, + "ordinal": 15, "name": "color", "type_info": "Varchar" }, { - "ordinal": 18, + "ordinal": 16, "name": "operator_settings", "type_info": "Jsonb" } @@ -112,8 +102,6 @@ true, true, true, - false, - true, true, false, true, @@ -126,5 +114,5 @@ true ] }, - "hash": "dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727" + "hash": "c12a0b0d423577afbb2772f971bffe2635785b53d31d0ceb3d72226a0c316e8a" } diff --git a/backend/.sqlx/query-cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32.json b/backend/.sqlx/query-e3c8219420bb859de3f1c322978c5859c924cffa61074562ac6d33106d02d7d6.json similarity index 66% rename from backend/.sqlx/query-cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32.json rename to backend/.sqlx/query-e3c8219420bb859de3f1c322978c5859c924cffa61074562ac6d33106d02d7d6.json index 54ae52423d..8ee4e7890e 100644 --- a/backend/.sqlx/query-cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32.json +++ b/backend/.sqlx/query-e3c8219420bb859de3f1c322978c5859c924cffa61074562ac6d33106d02d7d6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members \n FROM usr u\n JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id\n RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_\n WHERE g_.workspace_id = $1 AND g_.name != 'all'\n GROUP BY g_.workspace_id, name, summary, extra_perms", + "query": "SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members\n FROM usr u\n JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id\n RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_\n WHERE g_.workspace_id = $1 AND g_.name != 'all'\n GROUP BY g_.workspace_id, name, summary, extra_perms", "describe": { "columns": [ { @@ -42,5 +42,5 @@ null ] }, - "hash": "cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32" + "hash": "e3c8219420bb859de3f1c322978c5859c924cffa61074562ac6d33106d02d7d6" } diff --git a/backend/.sqlx/query-82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8.json b/backend/.sqlx/query-e5f1e1e74daeabf410991e2484c1fb565f04539fc7eb141ebfaa957456016841.json similarity index 50% rename from backend/.sqlx/query-82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8.json rename to backend/.sqlx/query-e5f1e1e74daeabf410991e2484c1fb565f04539fc7eb141ebfaa957456016841.json index 880e79a957..ddd216acc1 100644 --- a/backend/.sqlx/query-82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8.json +++ b/backend/.sqlx/query-e5f1e1e74daeabf410991e2484c1fb565f04539fc7eb141ebfaa957456016841.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT usage.usage FROM usage \n WHERE is_workspace = true \n AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1", + "query": "\n SELECT usage.usage FROM usage\n WHERE is_workspace = true\n AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)\n AND id = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ false ] }, - "hash": "82b16e771b6e21c4587b5ebf059e312f43b3e5a48f7599133831dbd65886f5d8" + "hash": "e5f1e1e74daeabf410991e2484c1fb565f04539fc7eb141ebfaa957456016841" } diff --git a/backend/.sqlx/query-f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4.json b/backend/.sqlx/query-f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4.json deleted file mode 100644 index 5145efa595..0000000000 --- a/backend/.sqlx/query-f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS (SELECT 1 \n FROM workspace_settings \n WHERE workspace_id <> $1 \n AND slack_command_script IS NOT NULL\n AND slack_team_id IS NOT NULL \n AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1))\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "f632ca2e17a3952fc45bd40a055a9442c35453dff95140d2f252c4fe6a14c6a4" -} diff --git a/backend/migrations/20250319121050_multiple_ai_providers.down.sql b/backend/migrations/20250319121050_multiple_ai_providers.down.sql new file mode 100644 index 0000000000..b6c6ff23bc --- /dev/null +++ b/backend/migrations/20250319121050_multiple_ai_providers.down.sql @@ -0,0 +1,24 @@ +ALTER TABLE workspace_settings RENAME COLUMN ai_config TO ai_resource; + +ALTER TABLE workspace_settings +ADD COLUMN ai_models VARCHAR(255)[] NOT NULL DEFAULT '{}', +ADD COLUMN code_completion_model VARCHAR(255); + +UPDATE workspace_settings +SET ai_resource = CASE + WHEN ai_resource IS NULL THEN NULL + ELSE jsonb_build_object( + 'provider', + COALESCE((SELECT jsonb_object_keys(COALESCE(ai_resource->>'providers', '{}')::jsonb) LIMIT 1), 'openai'), -- Get the first provider key + 'path', + ai_resource->'providers'->(SELECT jsonb_object_keys(COALESCE(ai_resource->>'providers', '{}')::jsonb) LIMIT 1)->>'resource_path' + ) +END, +ai_models = COALESCE(( + SELECT array_agg(model) + FROM jsonb_array_elements_text( + COALESCE(ai_resource->'providers'->(SELECT jsonb_object_keys(COALESCE(ai_resource->>'providers', '{}')::jsonb) LIMIT 1)->>'models', '[]')::jsonb + ) model + WHERE model IS NOT NULL +), '{}'), +code_completion_model = ai_resource->'code_completion_model'->>'model'; diff --git a/backend/migrations/20250319121050_multiple_ai_providers.up.sql b/backend/migrations/20250319121050_multiple_ai_providers.up.sql new file mode 100644 index 0000000000..40353f7501 --- /dev/null +++ b/backend/migrations/20250319121050_multiple_ai_providers.up.sql @@ -0,0 +1,38 @@ +UPDATE workspace_settings +SET ai_resource = CASE + WHEN ai_resource IS NULL OR ai_resource->>'path' IS NULL OR ai_resource->>'provider' IS NULL THEN NULL + ELSE jsonb_build_object( + 'providers', jsonb_build_object( + ai_resource->>'provider', + jsonb_build_object( + 'resource_path', ai_resource->>'path', + 'models', to_jsonb(ai_models) + ) + ), + 'default_model', + CASE + WHEN array_length(ai_models, 1) > 0 THEN jsonb_build_object( + 'model', ai_models[1], + 'provider', ai_resource->>'provider' + ) + ELSE NULL + END, + 'code_completion_model', + CASE + WHEN code_completion_model IS NULL THEN NULL + ELSE jsonb_build_object( + 'model', code_completion_model, + 'provider', ai_resource->>'provider' + ) + END + ) +END; + +ALTER TABLE workspace_settings +DROP COLUMN code_completion_model, +DROP COLUMN ai_models; + +ALTER TABLE workspace_settings RENAME COLUMN ai_resource TO ai_config; + + +-- { providers: { [provider]: { resource_path: resource_path, models: ai_models}, default_model: ai_models[0], code_completion_model: code_completion_model} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 3d73899486..03724d8b20 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1755,14 +1755,8 @@ paths: type: string deploy_to: type: string - ai_resource: - $ref: "#/components/schemas/AIResource" - code_completion_model: - type: string - ai_models: - type: array - items: - type: string + ai_config: + $ref: "#/components/schemas/AIConfig" error_handler: type: string error_handler_extra_args: @@ -1786,7 +1780,6 @@ paths: operator_settings: $ref: "#/components/schemas/OperatorSettings" required: - - ai_models - automatic_billing - error_handler_muted_on_cancel @@ -2235,18 +2228,7 @@ paths: content: application/json: schema: - type: object - required: - - ai_models - properties: - ai_resource: - $ref: "#/components/schemas/AIResource" - code_completion_model: - type: string - ai_models: - type: array - items: - type: string + $ref: "#/components/schemas/AIConfig" responses: "200": description: status @@ -2268,23 +2250,10 @@ paths: "200": description: status content: - text/plain: + application/json: schema: - type: object - properties: - ai_provider: - $ref: "#/components/schemas/AIProvider" - exists_ai_resource: - type: boolean - code_completion_model: - type: string - ai_models: - type: array - items: - type: string - required: - - exists_ai_resource - - ai_models + $ref: "#/components/schemas/AIConfig" + /w/{workspace}/workspaces/edit_error_handler: post: @@ -12415,17 +12384,42 @@ components: type: string enum: [openai, anthropic, mistral, deepseek, googleai, groq, openrouter, customai] - AIResource: + AIProviderModel: type: object properties: - path: + model: type: string provider: $ref: "#/components/schemas/AIProvider" required: - - path + - model - provider + AIProviderConfig: + type: object + properties: + resource_path: + type: string + models: + type: array + items: + type: string + required: + - resource_path + - models + + AIConfig: + type: object + properties: + providers: + type: object + additionalProperties: + $ref: "#/components/schemas/AIProviderConfig" + default_model: + $ref: "#/components/schemas/AIProviderModel" + code_completion_model: + $ref: "#/components/schemas/AIProviderModel" + Script: type: object properties: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index d9469900b1..4ff0f77d57 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,12 @@ use crate::{ variables::get_variable_or_self, }; -use anthropic::AnthropicCache; use anyhow::Context; use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; use http::HeaderMap; use lazy_static::lazy_static; +use openai::OpenaiCache; +use openai_api_compatible::OpenaiApiCompatibleCache; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -16,10 +17,6 @@ use std::collections::HashMap; use windmill_audit::{audit_ee::audit_log, ActionKind}; use windmill_common::error::{to_anyhow, Error, Result}; -use mistral::MistralCache; -use openai::OpenaiCache; -use openai_api_compatible::OpenaiApiCompatibleCache; - lazy_static::lazy_static! { static ref HTTP_CLIENT: Client = reqwest::ClientBuilder::new() .timeout(std::time::Duration::from_secs(60 * 5)) @@ -64,7 +61,12 @@ mod openai_api_compatible { Value::Object(mut obj) => obj .remove("api_key") .map(|v| serde_json::from_value::(v.clone()).ok()) - .flatten(), + .flatten() + .or_else(|| { + obj.remove("apiKey") + .map(|v| serde_json::from_value::(v.clone()).ok()) + .flatten() + }), _ => None, }; OpenaiApiCompatibleCache { base_url, api_key } @@ -134,7 +136,7 @@ mod openai { } } - const BASE_URL: &str = "https://api.openai.com/v1"; + pub const BASE_URL: &str = "https://api.openai.com/v1"; impl OpenaiCache { pub fn prepare_request(self, openai_path: &str, mut body: Bytes) -> Result { let OpenaiCache { api_key, azure_base_path, organization_id, user } = self; @@ -274,92 +276,21 @@ mod openai { } } -mod anthropic { - - use super::*; - - #[derive(Clone, Deserialize, Debug)] - pub struct AnthropicCache { - #[serde(rename = "apiKey")] - pub api_key: String, - } - - const API_VERSION: &str = "2023-06-01"; - - const BASE_URL: &str = "https://api.anthropic.com"; - impl AnthropicCache { - pub fn prepare_request(self, anthropic_path: &str, body: Bytes) -> Result { - let AnthropicCache { api_key } = self; - let url = format!("{}/{}", BASE_URL, anthropic_path); - let request = HTTP_CLIENT - .post(url) - .header("x-api-key", api_key) - .header("anthropic-version", API_VERSION) - .header("content-type", "application/json") - .body(body); - Ok(request) - } - } - - pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { - let mut resource: AnthropicCache = serde_json::from_value(resource) - .map_err(|e| Error::internal_err(format!("validating anthropic resource {e:#}")))?; - resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; - Ok(KeyCache::Anthropic(resource)) - } -} - -mod mistral { - use super::*; - #[derive(Deserialize, Clone, Debug)] - pub struct MistralCache { - #[serde(rename = "apiKey")] - pub api_key: String, - } - - const BASE_URL: &str = "https://api.mistral.ai"; - impl MistralCache { - pub fn prepare_request(self, mistral_path: &str, body: Bytes) -> Result { - let MistralCache { api_key } = self; - - let url = format!("{}/{}", BASE_URL, mistral_path); - let request = HTTP_CLIENT - .post(url) - .header("content-type", "application/json") - .header("Accept", "application/json") - .header("authorization", format!("Bearer {}", api_key)) - .body(body); - Ok(request) - } - } - - pub async fn get_cached_value(db: &DB, w_id: &str, resource: Value) -> Result { - let mut resource: MistralCache = serde_json::from_value(resource) - .map_err(|e| Error::internal_err(format!("validating mistral resource {e:#}")))?; - resource.api_key = get_variable_or_self(resource.api_key, db, w_id).await?; - Ok(KeyCache::Mistral(resource)) - } -} - #[derive(Clone, Debug)] pub enum KeyCache { Openai(OpenaiCache), - Anthropic(AnthropicCache), - Mistral(MistralCache), OpenaiApiCompatible(OpenaiApiCompatibleCache), } #[derive(Clone, Debug)] pub struct AICache { - pub path: String, pub cached_key: KeyCache, pub expires_at: std::time::Instant, } impl AICache { - pub fn new(path: String, cached_key: KeyCache) -> Self { + pub fn new(cached_key: KeyCache) -> Self { Self { - path, cached_key, expires_at: std::time::Instant::now() + std::time::Duration::from_secs(60), } @@ -370,10 +301,10 @@ impl AICache { } lazy_static! { - pub static ref AI_KEY_CACHE: Cache = Cache::new(500); + pub static ref AI_KEY_CACHE: Cache<(String, AIProvider), AICache> = Cache::new(500); } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "lowercase")] pub enum AIProvider { OpenAI, @@ -387,7 +318,7 @@ pub enum AIProvider { } impl AIProvider { - pub fn get_openai_compatible_base_url(&self) -> Result> { + pub fn get_base_url(&self) -> Result> { match self { AIProvider::DeepSeek => Ok(Some("https://api.deepseek.com/v1".to_string())), AIProvider::GoogleAI => Ok(Some( @@ -395,10 +326,10 @@ impl AIProvider { )), AIProvider::Groq => Ok(Some("https://api.groq.com/openai/v1".to_string())), AIProvider::OpenRouter => Ok(Some("https://openrouter.ai/api/v1".to_string())), + AIProvider::Anthropic => Ok(Some("https://api.anthropic.com/v1".to_string())), + AIProvider::Mistral => Ok(Some("https://api.mistral.ai/v1".to_string())), AIProvider::CustomAI => Ok(None), - _ => Err(Error::BadRequest( - "Please use the specific provider instead of the OpenAI compatible one".to_string(), - )), + AIProvider::OpenAI => Ok(Some(openai::BASE_URL.to_string())), } } } @@ -420,141 +351,232 @@ impl TryFrom<&str> for AIProvider { } } -#[derive(Deserialize, Debug)] -pub struct AIResource { - pub path: Option, +#[derive(Serialize, Deserialize, Debug)] +pub struct ProviderConfig { + pub resource_path: String, + pub models: Vec, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct ProviderModel { + pub model: String, pub provider: AIProvider, } -pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/*ai", post(proxy)); - - router +#[derive(Serialize, Deserialize, Debug)] +pub struct AIConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub providers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub default_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code_completion_model: Option, } -async fn proxy( +pub fn global_service() -> Router { + Router::new().route("/proxy/*ai", post(global_proxy)) +} + +pub fn workspaced_service() -> Router { + Router::new().route("/proxy/*ai", post(proxy)) +} + +async fn global_proxy( authed: ApiAuthed, Extension(db): Extension, - Path((w_id, ai_path)): Path<(String, String)>, + Path(ai_path): Path, headers: HeaderMap, body: Bytes, ) -> impl IntoResponse { - let workspace_cache = AI_KEY_CACHE.get(&w_id); - let forced_resource_path = headers - .get("X-Resource-Path") + let provider = headers + .get("X-Provider") + .map(|v| v.to_str().unwrap_or("").to_string()); + let api_key = headers + .get("X-API-Key") .map(|v| v.to_str().unwrap_or("").to_string()); - let ai_cache = match workspace_cache { - Some(cache) if !cache.is_expired() && forced_resource_path.is_none() => cache.cached_key, - _ => { - let (resource, resource_path, ai_provider) = if let Some(resource_path) = - forced_resource_path - { - // guess the provider from the resource type - let record = sqlx::query!( - "SELECT value, resource_type FROM resource WHERE path = $1 AND workspace_id = $2", - &resource_path, - &w_id - ) - .fetch_optional(&db) - .await? - .ok_or_else(|| { - Error::NotFound(format!( - "Could not find the resource {}, update the resource path in the workspace settings", resource_path - )) - })?; - ( - record.value, - resource_path, - AIProvider::try_from(record.resource_type.as_str())?, - ) - } else { - let ai_resource = sqlx::query_scalar!( - "SELECT ai_resource FROM workspace_settings WHERE workspace_id = $1", - &w_id - ) - .fetch_one(&db) - .await?; - - if ai_resource.is_none() { - return Err(Error::internal_err( - "AI resource not configured".to_string(), - )); - } - - let ai_resource = serde_json::from_value::(ai_resource.unwrap()) - .map_err(|e| Error::BadRequest(e.to_string()))?; - - let path = ai_resource.path.unwrap_or("".to_string()); - if path.is_empty() { - return Err(Error::BadRequest("Resource path is empty".to_string())); - } - let resource = sqlx::query_scalar!( - "SELECT value - FROM resource - WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_optional(&db) - .await? - .ok_or_else(|| { - Error::NotFound(format!( - "Could not find the {:?} resource at path {}, update the resource path in the workspace settings", ai_resource.provider, path - )) - })?; - - (resource, path, ai_resource.provider) - }; - - if resource.is_none() { - return Err(Error::internal_err(format!( - "{:?} resource missing value", - ai_provider - ))); - } - - let resource = resource.unwrap(); - - let ai_cache = match ai_provider { - AIProvider::OpenAI => openai::get_cached_value(&db, &w_id, resource).await, - AIProvider::Anthropic => anthropic::get_cached_value(&db, &w_id, resource).await, - AIProvider::Mistral => mistral::get_cached_value(&db, &w_id, resource).await, - _ => { - openai_api_compatible::get_cached_value( - &db, - &w_id, - resource, - ai_provider.get_openai_compatible_base_url()?, - ) - .await - } - }; - let ai_cache = ai_cache?; - AI_KEY_CACHE.insert(w_id.clone(), AICache::new(resource_path, ai_cache.clone())); - ai_cache - } + let provider = match provider { + Some(provider) => AIProvider::try_from(provider.as_str())?, + None => return Err(Error::BadRequest("Provider is required".to_string())), }; - let request = match ai_cache { - KeyCache::Openai(cached) => cached.prepare_request(&ai_path, body), - KeyCache::Anthropic(cached) => cached.prepare_request(&ai_path, body), - KeyCache::Mistral(cached) => cached.prepare_request(&ai_path, body), - KeyCache::OpenaiApiCompatible(cached) => cached.prepare_request(&ai_path, body), + let Some(api_key) = api_key else { + return Err(Error::BadRequest("API key is required".to_string())); }; - let response = request?.send().await.map_err(to_anyhow)?; + let base_url = provider.get_base_url()?; + + let Some(base_url) = base_url else { + return Err(Error::BadRequest("Provider is not supported".to_string())); + }; + + let url = format!("{}/{}", base_url, ai_path); + + let request = HTTP_CLIENT + .post(url) + .header("content-type", "application/json") + .header("Authorization", format!("Bearer {}", api_key)) + .body(body); + + let response = request.send().await.map_err(to_anyhow)?; let mut tx = db.begin().await?; audit_log( &mut *tx, &authed, - "ai.request", + "ai.global_request", ActionKind::Execute, - &w_id, + "global", Some(&authed.email), - Some([("ai_resource_path", &format!("{:?}", ai_path)[..])].into()), + None, + ) + .await?; + tx.commit().await?; + + if response.error_for_status_ref().is_err() { + let err_msg = response.text().await.unwrap_or("".to_string()); + return Err(Error::AiError(err_msg)); + } + + let status_code = response.status(); + let headers = response.headers().clone(); + let stream = response.bytes_stream(); + Ok((status_code, headers, axum::body::Body::from_stream(stream))) +} + +async fn proxy( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, ai_path)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, +) -> impl IntoResponse { + let provider = headers + .get("X-Provider") + .map(|v| v.to_str().unwrap_or("").to_string()); + + let provider = match provider { + Some(provider) => AIProvider::try_from(provider.as_str())?, + None => return Err(Error::BadRequest("Provider is required".to_string())), + }; + + let workspace_cache = AI_KEY_CACHE.get(&(w_id.clone(), provider.clone())); + + let forced_resource_path = headers + .get("X-Resource-Path") + .map(|v| v.to_str().unwrap_or("").to_string()); + let ai_cache = match workspace_cache { + Some(cache) if !cache.is_expired() && forced_resource_path.is_none() => cache.cached_key, + _ => { + let (resource, ai_provider, save_to_cache) = if let Some(resource_path) = + forced_resource_path + { + // forced resource path, get the resource directly + let resource = sqlx::query_scalar!( + "SELECT value FROM resource WHERE path = $1 AND workspace_id = $2", + &resource_path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Could not find the resource {}, update the resource path in the workspace settings", resource_path + )) + })?; + + (resource, provider, false) + } else { + let ai_config = sqlx::query_scalar!( + "SELECT ai_config FROM workspace_settings WHERE workspace_id = $1", + &w_id + ) + .fetch_one(&db) + .await?; + + if ai_config.is_none() { + return Err(Error::internal_err( + "AI resource not configured".to_string(), + )); + } + + let ai_config = serde_json::from_value::(ai_config.unwrap()) + .map_err(|e| Error::BadRequest(e.to_string()))?; + + let provider_config = ai_config + .providers + .as_ref() + .map(|providers| providers.get(&provider)) + .flatten() + .ok_or_else(|| { + Error::BadRequest(format!("Provider {:?} not configured", provider)) + })?; + + if provider_config.resource_path.is_empty() { + return Err(Error::BadRequest("Resource path is empty".to_string())); + } + let resource = sqlx::query_scalar!( + "SELECT value + FROM resource + WHERE path = $1 AND workspace_id = $2", + &provider_config.resource_path, + &w_id + ) + .fetch_optional(&db) + .await? + .ok_or_else(|| { + Error::NotFound(format!( + "Could not find the {:?} resource at path {}, update the resource path in the workspace settings", provider, provider_config.resource_path + )) + })?; + + (resource, provider, true) + }; + + let Some(resource) = resource else { + return Err(Error::internal_err(format!( + "{:?} resource missing value", + ai_provider + ))); + }; + + let ai_cache = match ai_provider { + AIProvider::OpenAI => openai::get_cached_value(&db, &w_id, resource).await?, + _ => { + openai_api_compatible::get_cached_value( + &db, + &w_id, + resource, + ai_provider.get_base_url()?, + ) + .await? + } + }; + if save_to_cache { + AI_KEY_CACHE.insert((w_id.clone(), ai_provider), AICache::new(ai_cache.clone())); + } + ai_cache + } + }; + + let request = match ai_cache { + KeyCache::Openai(cached) => cached.prepare_request(&ai_path, body), + KeyCache::OpenaiApiCompatible(cached) => cached.prepare_request(&ai_path, body), + }; + + let response = request?.send().await.map_err(to_anyhow)?; + + let mut tx = db.begin().await?; + + audit_log( + &mut *tx, + &authed, + "ai.request", + ActionKind::Execute, + &w_id, + Some(&authed.email), + Some([("ai_config_path", &format!("{:?}", ai_path)[..])].into()), ) .await?; tx.commit().await?; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 86a42aa8ba..442d88ea65 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -487,6 +487,7 @@ pub async fn run_server( .nest("/apps", apps::global_service().layer(cors.clone())) .nest("/schedules", schedule::global_service()) .nest("/embeddings", embeddings::global_service()) + .nest("/ai", ai::global_service()) .route_layer(from_extractor::()) .route_layer(from_extractor::()) .nest("/jobs", jobs::global_root_service()) diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 03ba214bba..c94a7704cf 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -8,7 +8,7 @@ use std::collections::HashMap; -use crate::ai::{AIProvider, AIResource, AI_KEY_CACHE}; +use crate::ai::{AIConfig, AI_KEY_CACHE}; use crate::db::ApiAuthed; use crate::users_ee::send_email_if_possible; use crate::utils::get_instance_username_or_create_pending; @@ -52,6 +52,9 @@ use windmill_git_sync::handle_deployment_metadata; #[cfg(feature = "enterprise")] use windmill_common::utils::require_admin_or_devops; +#[cfg(not(feature = "enterprise"))] +use crate::ai::AIProvider; + use hyper::StatusCode; use serde::{Deserialize, Serialize}; use sqlx::{FromRow, Postgres, Transaction}; @@ -211,10 +214,7 @@ pub struct WorkspaceSettings { #[serde(skip_serializing_if = "Option::is_none")] pub deploy_to: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub ai_resource: Option, - pub ai_models: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub code_completion_model: Option, + pub ai_config: Option, #[serde(skip_serializing_if = "Option::is_none")] pub error_handler: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -291,13 +291,6 @@ struct EditWebhook { webhook: Option, } -#[derive(Deserialize)] -struct EditCopilotConfig { - ai_resource: Option, - code_completion_model: Option, - ai_models: Vec, -} - #[derive(Deserialize, Serialize, Debug)] struct LargeFileStorageWithSecondary { #[serde(flatten)] @@ -449,7 +442,7 @@ async fn get_settings( let mut tx = user_db.begin(&authed).await?; let settings = sqlx::query_as!( WorkspaceSettings, - "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_resource, ai_models, code_completion_model, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, automatic_billing, default_scripts, mute_critical_alerts, color, operator_settings FROM workspace_settings WHERE workspace_id = $1", + "SELECT workspace_id, slack_team_id, teams_team_id, teams_team_name, slack_name, slack_command_script, teams_command_script, slack_email, auto_invite_domain, auto_invite_operator, auto_add, customer_id, plan, webhook, deploy_to, ai_config, error_handler, error_handler_extra_args, error_handler_muted_on_cancel, large_file_storage, git_sync, deploy_ui, default_app, automatic_billing, default_scripts, mute_critical_alerts, color, operator_settings FROM workspace_settings WHERE workspace_id = $1", &w_id ) .fetch_one(&mut *tx) @@ -495,11 +488,11 @@ async fn edit_slack_command( if es.slack_command_script.is_some() { let exists_slack_command_with_team_id = sqlx::query_scalar!( r#" - SELECT EXISTS (SELECT 1 - FROM workspace_settings - WHERE workspace_id <> $1 + SELECT EXISTS (SELECT 1 + FROM workspace_settings + WHERE workspace_id <> $1 AND slack_command_script IS NOT NULL - AND slack_team_id IS NOT NULL + AND slack_team_id IS NOT NULL AND slack_team_id = (SELECT slack_team_id FROM workspace_settings WHERE workspace_id = $1)) "#, &w_id @@ -697,51 +690,35 @@ async fn edit_copilot_config( Extension(db): Extension, Path(w_id): Path, ApiAuthed { is_admin, username, .. }: ApiAuthed, - Json(eo): Json, + Json(ai_config): Json, ) -> Result { require_admin(is_admin, &username)?; let mut tx = db.begin().await?; - if let Some(ai_resource) = &eo.ai_resource { - let parsed_ai_resource = serde_json::from_value::(ai_resource.clone()) - .map_err(|e| Error::BadRequest(e.to_string()))?; + sqlx::query!( + "UPDATE workspace_settings SET ai_config = $1 WHERE workspace_id = $2", + sqlx::types::Json(&ai_config) as sqlx::types::Json<&AIConfig>, + &w_id + ) + .execute(&mut *tx) + .await?; - #[cfg(not(feature = "enterprise"))] - { - if matches!(parsed_ai_resource.provider, AIProvider::CustomAI) { - return Err(Error::BadRequest( - "Custom AI is only available on EE".to_string(), - )); - } - } - - sqlx::query!( - "UPDATE workspace_settings SET ai_resource = $1, code_completion_model = $2, ai_models = $3 WHERE workspace_id = $4", - ai_resource, - eo.code_completion_model, - eo.ai_models.as_slice(), - &w_id - ) - .execute(&mut *tx) - .await?; - - if let Some(cached) = AI_KEY_CACHE.get(&w_id) { - if parsed_ai_resource.path.is_none() || parsed_ai_resource.path.unwrap() != cached.path + if let Some(ref providers) = ai_config.providers { + for provider in providers.keys() { + #[cfg(not(feature = "enterprise"))] { - AI_KEY_CACHE.remove(&w_id); + if matches!(provider, &AIProvider::CustomAI) { + return Err(Error::BadRequest( + "Custom AI is only available on EE".to_string(), + )); + } } + + AI_KEY_CACHE.remove(&(w_id.clone(), provider.clone())); } - } else { - sqlx::query!( - "UPDATE workspace_settings SET ai_resource = NULL, code_completion_model = $1, ai_models = '{}' WHERE workspace_id = $2", - eo.code_completion_model, - &w_id, - ) - .execute(&mut *tx) - .await?; - AI_KEY_CACHE.remove(&w_id); } + audit_log( &mut *tx, &authed, @@ -749,16 +726,7 @@ async fn edit_copilot_config( ActionKind::Update, &w_id, Some(&authed.email), - Some( - [ - ("ai_resource", &format!("{:?}", eo.ai_resource)[..]), - ( - "code_completion_model", - &format!("{:?}", eo.code_completion_model)[..], - ), - ] - .into(), - ), + Some([("ai_config", &format!("{:?}", ai_config)[..])].into()), ) .await?; tx.commit().await?; @@ -766,42 +734,33 @@ async fn edit_copilot_config( Ok(format!("Edit copilot config for workspace {}", &w_id)) } -#[derive(Serialize)] -struct CopilotInfo { - #[serde(skip_serializing_if = "Option::is_none")] - pub ai_provider: Option, - pub exists_ai_resource: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub code_completion_model: Option, - pub ai_models: Vec, -} async fn get_copilot_info( Extension(db): Extension, Path(w_id): Path, -) -> JsonResult { +) -> JsonResult { let mut tx = db.begin().await?; - let record = sqlx::query!( - "SELECT ai_resource, code_completion_model, ai_models FROM workspace_settings WHERE workspace_id = $1", + let copilot_info = sqlx::query_scalar!( + "SELECT ai_config as \"ai_config: sqlx::types::Json\" FROM workspace_settings WHERE workspace_id = $1", &w_id ) .fetch_one(&mut *tx) .await - .map_err(|e| Error::internal_err(format!("getting ai_resource and code_completion_model: {e:#}")))?; + .map_err(|e| { + Error::internal_err(format!( + "getting ai config: {e:#}" + )) + })?; tx.commit().await?; - let (ai_provider, exists_ai_resource) = if let Some(ai_resource) = record.ai_resource { - let ai_resource = serde_json::from_value::(ai_resource)?; - (Some(ai_resource.provider), ai_resource.path.is_some()) + if let Some(sqlx::types::Json(copilot_info)) = copilot_info { + Ok(Json(copilot_info)) } else { - (None, false) - }; - - Ok(Json(CopilotInfo { - ai_provider, - exists_ai_resource, - code_completion_model: record.code_completion_model, - ai_models: record.ai_models, - })) + Ok(Json(AIConfig { + providers: None, + default_model: None, + code_completion_model: None, + })) + } } async fn edit_large_file_storage_config( @@ -1370,9 +1329,8 @@ async fn get_used_triggers( let websocket_used = sqlx::query_as!( UsedTriggers, r#" - SELECT - - EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS "websocket_used!", + SELECT + EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS "websocket_used!", EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS "http_routes_used!", EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as "kafka_used!", EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!", @@ -2100,8 +2058,8 @@ async fn change_workspace_color( async fn get_usage(Extension(db): Extension, Path(w_id): Path) -> Result { let usage = sqlx::query_scalar!( " - SELECT usage.usage FROM usage - WHERE is_workspace = true + SELECT usage.usage FROM usage + WHERE is_workspace = true AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date) AND id = $1", w_id diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 003a01c002..4560d9b415 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -250,10 +250,7 @@ struct SimplifiedSettings { error_handler_extra_args: Option, error_handler_muted_on_cancel: bool, #[serde(skip_serializing_if = "Option::is_none")] - ai_resource: Option, - ai_models: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - code_completion_model: Option, + ai_config: Option, #[serde(skip_serializing_if = "Option::is_none")] large_file_storage: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -500,8 +497,8 @@ pub(crate) async fn tarball_workspace( { let apps = sqlx::query_as::<_, AppWithLastVersion>( "SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path, - app.extra_perms, app_version.value, - app_version.created_at, app_version.created_by from app, app_version + app.extra_perms, app_version.value, + app_version.created_at, app_version.created_by from app, app_version WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]", ) .bind(&w_id) @@ -711,7 +708,7 @@ pub(crate) async fn tarball_workspace( if include_groups.unwrap_or(false) { let groups = sqlx::query!( - r#"SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members + r#"SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members FROM usr u JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_ @@ -773,22 +770,20 @@ pub(crate) async fn tarball_workspace( let settings = sqlx::query_as!( SimplifiedSettings, r#"SELECT - -- slack_team_id, - -- slack_name, - -- slack_command_script, + -- slack_team_id, + -- slack_name, + -- slack_command_script, -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email, auto_invite_domain IS NOT NULL AS "auto_invite_enabled!", - CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!", - CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!", - webhook, - deploy_to, - error_handler, - ai_resource, - ai_models, - code_completion_model, - error_handler_extra_args, - error_handler_muted_on_cancel, - large_file_storage, + CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!", + CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!", + webhook, + deploy_to, + error_handler, + ai_config, + error_handler_extra_args, + error_handler_muted_on_cancel, + large_file_storage, git_sync, default_app, default_scripts, diff --git a/cli/gen/core/OpenAPI.ts b/cli/gen/core/OpenAPI.ts index 3aa0d92808..9965c2ef9a 100644 --- a/cli/gen/core/OpenAPI.ts +++ b/cli/gen/core/OpenAPI.ts @@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = { PASSWORD: undefined, TOKEN: getEnv("WM_TOKEN"), USERNAME: undefined, - VERSION: '1.477.1', + VERSION: '1.478.1', WITH_CREDENTIALS: true, interceptors: { request: new Interceptors(), diff --git a/cli/gen/services.gen.ts b/cli/gen/services.gen.ts index b3fa22bf66..bee9069a5b 100644 --- a/cli/gen/services.gen.ts +++ b/cli/gen/services.gen.ts @@ -1374,7 +1374,7 @@ export const editCopilotConfig = (data: EditCopilotConfigData): CancelablePromis * get copilot info * @param data The data for the request. * @param data.workspace - * @returns unknown status + * @returns AIConfig status * @throws ApiError */ export const getCopilotInfo = (data: GetCopilotInfoData): CancelablePromise => { return __request(OpenAPI, { diff --git a/cli/gen/types.gen.ts b/cli/gen/types.gen.ts index 13d30a5967..0d73b24876 100644 --- a/cli/gen/types.gen.ts +++ b/cli/gen/types.gen.ts @@ -2,11 +2,24 @@ export type AIProvider = 'openai' | 'anthropic' | 'mistral' | 'deepseek' | 'googleai' | 'groq' | 'openrouter' | 'customai'; -export type AIResource = { - path: string; +export type AIProviderModel = { + model: string; provider: AIProvider; }; +export type AIProviderConfig = { + resource_path: string; + models: Array<(string)>; +}; + +export type AIConfig = { + providers?: { + [key: string]: AIProviderConfig; + }; + default_model?: AIProviderModel; + code_completion_model?: AIProviderModel; +}; + export type Script = { workspace_id?: string; hash: string; @@ -2538,9 +2551,7 @@ export type GetSettingsResponse = ({ customer_id?: string; webhook?: string; deploy_to?: string; - ai_resource?: AIResource; - code_completion_model?: string; - ai_models: Array<(string)>; + ai_config?: AIConfig; error_handler?: string; error_handler_extra_args?: ScriptArgs; error_handler_muted_on_cancel: boolean; @@ -2742,11 +2753,7 @@ export type EditCopilotConfigData = { /** * WorkspaceCopilotConfig */ - requestBody: { - ai_resource?: AIResource; - code_completion_model?: string; - ai_models: Array<(string)>; - }; + requestBody: AIConfig; workspace: string; }; @@ -2756,12 +2763,7 @@ export type GetCopilotInfoData = { workspace: string; }; -export type GetCopilotInfoResponse = ({ - ai_provider?: AIProvider; - exists_ai_resource: boolean; - code_completion_model?: string; - ai_models: Array<(string)>; -}); +export type GetCopilotInfoResponse = (AIConfig); export type EditErrorHandlerData = { /** diff --git a/cli/settings.ts b/cli/settings.ts index 054763a16b..9877e085c2 100644 --- a/cli/settings.ts +++ b/cli/settings.ts @@ -1,7 +1,7 @@ import process from "node:process"; import { colors, Confirm, log, yamlParseFile, yamlStringify } from "./deps.ts"; import * as wmill from "./gen/services.gen.ts"; -import { AIResource, Config, GlobalSetting } from "./gen/types.gen.ts"; +import { AIConfig, Config, GlobalSetting } from "./gen/types.gen.ts"; import { compareInstanceObjects, InstanceSyncOptions } from "./instance.ts"; import { isSuperset } from "./types.ts"; import { deepEqual } from "./utils.ts"; @@ -20,9 +20,7 @@ export interface SimplifiedSettings { error_handler?: string; error_handler_extra_args?: any; error_handler_muted_on_cancel?: boolean; - ai_resource?: AIResource; - code_completion_model?: string; - ai_models: string[]; + ai_config?: AIConfig; large_file_storage?: any; git_sync?: any; default_app?: string; @@ -156,19 +154,11 @@ export async function pushWorkspaceSettings( }); } } - if ( - localSettings.ai_resource != settings.ai_resource || - localSettings.code_completion_model != settings.code_completion_model || - !deepEqual(localSettings.ai_models, settings.ai_models) - ) { + if (!deepEqual(localSettings.ai_config, settings.ai_config)) { log.debug(`Updating copilot settings...`); await wmill.editCopilotConfig({ workspace, - requestBody: { - ai_resource: localSettings.ai_resource, - code_completion_model: localSettings.code_completion_model, - ai_models: localSettings.ai_models, - }, + requestBody: localSettings.ai_config ?? {}, }); } if ( diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 819d7a117e..b31aeec507 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,7 +9,6 @@ "version": "1.478.1", "license": "AGPL-3.0", "dependencies": { - "@anthropic-ai/sdk": "^0.37.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~11.1.2", "@codingame/monaco-vscode-standalone-css-language-features": "~11.1.2", @@ -18,7 +17,6 @@ "@codingame/monaco-vscode-standalone-typescript-language-features": "~11.1.2", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", - "@mistralai/mistralai": "^1.3.0", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", @@ -34,13 +32,14 @@ "chartjs-plugin-zoom": "^2.0.0", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", - "diff": "^5.1.0", + "diff": "^7.0.0", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", "graphql": "^16.7.1", "hash-sum": "^2.0.0", "highlight.js": "^11.8.0", + "idb": "^8.0.2", "lucide-svelte": "^0.399.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~11.1.2", @@ -49,7 +48,7 @@ "monaco-languageclient": "9.1.1", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.57.2", + "openai": "^4.87.1", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", @@ -97,6 +96,7 @@ "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", "@types/d3-zoom": "^3.0.3", + "@types/diff": "^7.0.1", "@types/lodash": "^4.14.195", "@types/node": "^20.3.3", "@types/vscode": "^1.83.5", @@ -185,30 +185,6 @@ "node": ">=6.0.0" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.37.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.37.0.tgz", - "integrity": "sha512-tHjX2YbkUBwEgg0JZU3EFSSAQPoK4qQR/NFYa8Vtzd5UAyXzZksCw2In69Rml4R/TyHPBfRYaLK35XiOe33pjw==", - "license": "MIT", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - } - }, - "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { - "version": "18.19.64", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.64.tgz", - "integrity": "sha512-955mDqvO2vFf/oL7V3WiUtiz+BugyX8uVbaT2H8oj3+8dRyH2FLiNdowe7eNqRM7IOIZvzDH76EoAT+gwm6aIQ==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "11.6.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-11.6.1.tgz", @@ -1758,14 +1734,6 @@ "node": "^18 || >=20" } }, - "node_modules/@mistralai/mistralai": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@mistralai/mistralai/-/mistralai-1.3.0.tgz", - "integrity": "sha512-G5DPCSC8sEhG3LUEZDYLD7qEZWDuZXgaX3IcoC3a/ydm9jFuh2pRZtknsgMx2sU8d7kxRuxblY3fPH5C38wnhQ==", - "peerDependencies": { - "zod": ">= 3" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2746,6 +2714,13 @@ "@types/ms": "*" } }, + "node_modules/@types/diff": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-7.0.1.tgz", + "integrity": "sha512-R/BHQFripuhW6XPXy05hIvXJQdQ4540KnTvEFHSLjXfHYM41liOLKgIJEyYYiQe796xpaMHfe4Uj/p7Uvng2vA==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.5.tgz", @@ -2829,11 +2804,6 @@ "integrity": "sha512-Yg4LkgFYvn1faISbDNWmcAC1XoDT8IoMUFspp5mnagKk+UvD2N0IWt5A7GRdMubsNWqgCLmrkf8rXkzNqb4szA==", "dev": true }, - "node_modules/@types/qs": { - "version": "6.9.15", - "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.15.tgz", - "integrity": "sha512-uXHQKES6DQKKCLh441Xv/dwxOq1TVS3JPUMlEqoEglvlhR6Mxnlew/Xq/LRVHpLyk7iK3zODe1qYHIMltO7XGg==" - }, "node_modules/@types/semver": { "version": "7.5.5", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.5.tgz", @@ -3334,6 +3304,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==", + "license": "MIT", "optional": true }, "node_modules/asynckit": { @@ -4573,9 +4544,10 @@ "dev": true }, "node_modules/diff": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.1.0.tgz", - "integrity": "sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "license": "BSD-3-Clause", "engines": { "node": ">=0.3.1" } @@ -6130,6 +6102,12 @@ "ms": "^2.0.0" } }, + "node_modules/idb": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.2.tgz", + "integrity": "sha512-CX70rYhx7GDDQzwwQMDwF6kDRQi5vVs6khHUumDrMecBylKkwvZ8HWvKV08AGb7VbpoGCWUQ4aHzNDgoUiOIUg==", + "license": "ISC" + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -8913,17 +8891,6 @@ "node": ">= 6" } }, - "node_modules/object-inspect": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", - "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/object-is": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", @@ -9004,27 +8971,30 @@ } }, "node_modules/openai": { - "version": "4.57.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.57.2.tgz", - "integrity": "sha512-IgIxNjo9tfgnfx6gmwNMg3tdF9giK/2lbwG5DY7zs4TP9Gz+h6h2hBOMoalLPFUVOO5HLOgMI/PFV5VDAUvvMg==", + "version": "4.87.1", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.87.1.tgz", + "integrity": "sha512-mwZ4N4KKGUE5JSTR4IdQZzuxFyStJFqSR62JuWob7ka286tbFacYkOf+Ypk/2IPAUCmhzft5C3UqzcFYaliVMA==", + "license": "Apache-2.0", "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", - "@types/qs": "^6.9.7", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7", - "qs": "^6.10.3" + "node-fetch": "^2.6.7" }, "bin": { "openai": "bin/cli" }, "peerDependencies": { + "ws": "^8.18.0", "zod": "^3.23.8" }, "peerDependenciesMeta": { + "ws": { + "optional": true + }, "zod": { "optional": true } @@ -10166,20 +10136,6 @@ "node": ">=6" } }, - "node_modules/qs": { - "version": "6.13.0", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", - "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", - "dependencies": { - "side-channel": "^1.0.6" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/quadprog": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/quadprog/-/quadprog-1.6.1.tgz", @@ -10801,23 +10757,6 @@ "node": ">=8" } }, - "node_modules/side-channel": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz", - "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==", - "dependencies": { - "call-bind": "^1.0.7", - "es-errors": "^1.3.0", - "get-intrinsic": "^1.2.4", - "object-inspect": "^1.13.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/simple-concat": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", @@ -12671,12 +12610,26 @@ } }, "node_modules/ws": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.2.tgz", - "integrity": "sha512-zmhltoSR8u1cnDsD43TX59mzoMZsLKqUweyYBAIvTngR3shc0W6aOZylZmq/7hqyVxPdi+5Ud2QInblgyE72fw==", + "version": "8.18.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.1.tgz", + "integrity": "sha512-RKW2aJZMXeMxVpnZ6bck+RswznaxmzdULiBr6KY7XkTnW8uvt0iT9H5DkHUChXrc+uurzwa0rVI16n/Xzjdz1w==", + "license": "MIT", "optional": true, - "dependencies": { - "async-limiter": "~1.0.0" + "peer": true, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } } }, "node_modules/xml-utils": { @@ -12775,6 +12728,16 @@ "yjs": "^13.5.6" } }, + "node_modules/y-websocket/node_modules/ws": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.3.tgz", + "integrity": "sha512-jmTjYU0j60B+vHey6TfR3Z7RD61z/hmxBS3VMSGIrroOWXQEneK1zNuotOUrGyBHQj0yrpsLHPWtigEFd13ndA==", + "license": "MIT", + "optional": true, + "dependencies": { + "async-limiter": "~1.0.0" + } + }, "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", @@ -12838,6 +12801,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", "license": "MIT", + "optional": true, "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/frontend/package.json b/frontend/package.json index 6356634a93..489dba9869 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -31,6 +31,7 @@ "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", "@types/d3-zoom": "^3.0.3", + "@types/diff": "^7.0.1", "@types/lodash": "^4.14.195", "@types/node": "^20.3.3", "@types/vscode": "^1.83.5", @@ -84,7 +85,6 @@ }, "type": "module", "dependencies": { - "@anthropic-ai/sdk": "^0.37.0", "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~11.1.2", "@codingame/monaco-vscode-standalone-css-language-features": "~11.1.2", @@ -93,7 +93,6 @@ "@codingame/monaco-vscode-standalone-typescript-language-features": "~11.1.2", "@json2csv/plainjs": "^7.0.6", "@leeoniya/ufuzzy": "^1.0.8", - "@mistralai/mistralai": "^1.3.0", "@popperjs/core": "^2.11.6", "@redocly/json-to-json-schema": "^0.0.1", "@tanstack/svelte-table": "^8.9.9", @@ -109,13 +108,14 @@ "chartjs-plugin-zoom": "^2.0.0", "d3-zoom": "^3.0.0", "date-fns": "^2.30.0", - "diff": "^5.1.0", + "diff": "^7.0.0", "driver.js": "^1.3.0", "esm-env": "^1.0.0", "fast-equals": "^5.0.1", "graphql": "^16.7.1", "hash-sum": "^2.0.0", "highlight.js": "^11.8.0", + "idb": "^8.0.2", "lucide-svelte": "^0.399.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~11.1.2", @@ -124,7 +124,7 @@ "monaco-languageclient": "9.1.1", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^4.57.2", + "openai": "^4.87.1", "p-limit": "^6.1.0", "panzoom": "^9.4.3", "pdfjs-dist": "4.8.69", diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 8e96e68e7c..fb8ce2c8f9 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -19,7 +19,7 @@ type TriggersCount } from '$lib/gen' import { inferArgs } from '$lib/infer' - import { copilotInfo, userStore, workspaceStore } from '$lib/stores' + import { setCopilotInfo, userStore, workspaceStore } from '$lib/stores' import { emptySchema, sendUserToast } from '$lib/utils' import { Pane, Splitpanes } from 'svelte-splitpanes' import { onDestroy, onMount, setContext } from 'svelte' @@ -108,30 +108,21 @@ setContext('FlowCopilotContext', flowCopilotContext) - async function setCopilotInfo() { + async function setupCopilotInfo() { if (workspace) { workspaceAIClients.init(workspace) try { const info = await WorkspaceService.getCopilotInfo({ workspace }) - copilotInfo.set({ - ...info, - ai_provider: info.ai_provider ?? 'openai' - }) + setCopilotInfo(info) } catch (err) { - copilotInfo.set({ - ai_provider: 'openai', - exists_ai_resource: false, - code_completion_model: undefined, - ai_models: [] - }) - - console.error('Could not get copilot info') + console.error('Could not get copilot info', err) + setCopilotInfo({}) } } } $: if (workspace) { $workspaceStore = workspace - setCopilotInfo() + setupCopilotInfo() } $: if (workspace && token) { diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index db74de17b4..807e16e69b 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -140,13 +140,11 @@ import type { Disposable } from 'vscode' import type { DocumentUri, MessageTransports } from 'vscode-languageclient' import { workspaceStore } from '$lib/stores' - import { type Preview, UserService } from '$lib/gen' + import { type Preview, ResourceService, UserService } from '$lib/gen' import type { Text } from 'yjs' import { initializeVscode } from '$lib/components/vscode' import { initializeMode } from 'monaco-graphql/esm/initializeMode.js' - import { sleep } from '$lib/utils' - import { editorCodeCompletion } from '$lib/components/copilot/completion' import { editor as meditor, languages, @@ -172,7 +170,11 @@ import { initVim } from './monaco_keybindings' import { buildWorkerDefinition } from '$lib/monaco_workers/build_workers' import { parseTypescriptDeps } from '$lib/relative_imports' - + import { Autocompletor } from './copilot/autocomplete/monaco-adapter' + import { AIChatEditorHandler } from './copilot/chat/monaco-adapter' + import GlobalReviewButtons from './copilot/chat/GlobalReviewButtons.svelte' + import { writable } from 'svelte/store' + import { formatResourceTypes } from './copilot/chat/core' // import EditorTheme from './EditorTheme.svelte' let divEl: HTMLDivElement | null = null @@ -202,7 +204,7 @@ ruff: false, deno: false, go: false, - shellcheck: false, + shellcheck: false } export let shouldBindKey: boolean = true export let fixedOverflowWidgets = true @@ -346,7 +348,7 @@ } } - export function append(code): void { + export function append(code: string): void { if (editor) { const lineCount = editor.getModel()?.getLineCount() || 0 const lastLineLength = editor.getModel()?.getLineLength(lineCount) || 0 @@ -601,79 +603,86 @@ } } - let copilotCompletor: Disposable | undefined = undefined - let copilotTs = Date.now() - let abortController: AbortController | undefined = undefined - - function addCopilotSuggestions() { - if (copilotCompletor) { - copilotCompletor.dispose() - } - copilotCompletor = vscode.languages.registerInlineCompletionItemProvider( - { pattern: '**' }, - { - async provideInlineCompletionItems(model, position, context, token) { - abortController?.abort() - const textUntilPosition = model.getText( - new vscode.Range(0, 0, position.line, position.character) - ) - let items: vscode.InlineCompletionItem[] = [] - - const lastChar = textUntilPosition[textUntilPosition.length - 1] - if (textUntilPosition.trim().length > 5 && lastChar.match(/[\(\{\s:=]/)) { - const textAfterPosition = model.getText( - new vscode.Range(position.line, position.character, model.lineCount + 1, 1) - ) - - const thisTs = Date.now() - copilotTs = thisTs - await sleep(200) - if (copilotTs === thisTs) { - abortController?.abort() - abortController = new AbortController() - token.onCancellationRequested(() => { - abortController?.abort() - }) - const aiProvider = $copilotInfo.ai_provider - const insertText = await editorCodeCompletion( - textUntilPosition, - textAfterPosition, - lang, - abortController, - aiProvider - ) - if (insertText) { - items = [ - { - insertText, - range: new vscode.Range( - position.line, - position.character, - position.line, - position.character - ) - } - ] - } - } - } - - return { - items, - commands: [] - } - } - } - ) + let reviewingChanges = writable(false) + let aiChatEditorHandler: AIChatEditorHandler | undefined = undefined + export function reviewAndApplyCode(code: string) { + aiChatEditorHandler?.reviewAndApply(code) } - $: $copilotInfo.exists_ai_resource && - $copilotInfo.code_completion_model && + function addChatHandler(editor: meditor.IStandaloneCodeEditor) { + aiChatEditorHandler = new AIChatEditorHandler(editor) + reviewingChanges = aiChatEditorHandler.reviewingChanges + } + + $: $reviewingChanges && autocompletor?.reject() + + let completorDisposable: Disposable | undefined = undefined + let autocompletor: Autocompletor | undefined = undefined + function addSuperCompletor(editor: meditor.IStandaloneCodeEditor) { + if (completorDisposable) { + completorDisposable.dispose() + } + autocompletor = new Autocompletor(editor, lang) + + // last user events (currently disabled): + // let lastTs = Date.now() + // editor.onDidChangeModelContent((e) => { + // const thisTs = Date.now() + // lastTs = thisTs + // setTimeout(() => { + // if (thisTs === lastTs) { + // autocompletor?.savePatch() + // } + // }, 150) + // }) + + completorDisposable = editor.onDidChangeCursorPosition((e) => { + autocompletor?.reject() + if ($reviewingChanges) { + return + } + const position = editor.getPosition() + if (!position) { + return + } + const upToText = editor.getModel()?.getValueInRange({ + startLineNumber: position.lineNumber, + startColumn: 0, + endLineNumber: position.lineNumber, + endColumn: position.column + }) + const lastChar = upToText ? upToText[upToText.length - 1] : '' + if (lastChar && lastChar.match(/[\(\{\s:="',]/)) { + autocompletor?.predict() + } + }) + + editor.addCommand(KeyCode.Tab, () => { + if (autocompletor?.hasChanges()) { + autocompletor?.accept() + autocompletor?.predict() + } else { + editor.trigger('keyboard', 'tab', {}) + } + }) + + editor.onKeyDown((e) => { + if (e.keyCode === KeyCode.Escape) { + autocompletor?.reject() + } + }) + } + + $: $copilotInfo.enabled && + $copilotInfo.codeCompletionModel && $codeCompletionSessionEnabled && initialized && - addCopilotSuggestions() + editor && + addSuperCompletor(editor) - $: !$codeCompletionSessionEnabled && copilotCompletor && copilotCompletor.dispose() + $: $copilotInfo.enabled && initialized && editor && addChatHandler(editor) + + $: !$codeCompletionSessionEnabled && completorDisposable && completorDisposable.dispose() const outputChannel = { name: 'Language Server Client', @@ -1121,7 +1130,7 @@ initialized = true try { - model = meditor.createModel(code, (lang == 'nu') ? 'python' : lang, mUri.parse(uri)) + model = meditor.createModel(code, lang == 'nu' ? 'python' : lang, mUri.parse(uri)) } catch (err) { console.log('model already existed', err) const nmodel = meditor.getModel(mUri.parse(uri)) @@ -1182,6 +1191,14 @@ editor?.trigger('keyboard', 'editor.action.commentLine', {}) }) + editor?.addCommand(KeyMod.CtrlCmd | KeyCode.KeyL, function () { + dispatch('toggleAiPanel') + }) + + editor?.addCommand(KeyMod.CtrlCmd | KeyCode.KeyU, function () { + dispatch('toggleTestPanel') + }) + if ( !websocketAlive.deno && !websocketAlive.pyright && @@ -1199,6 +1216,7 @@ reloadWebsocket() setTypescriptExtraLibs() + setTypescriptRTNamespace() return () => { console.log('disposing editor') ata = undefined @@ -1214,6 +1232,27 @@ } } + async function setTypescriptRTNamespace() { + if ( + scriptLang && + (scriptLang === 'bun' || + scriptLang === 'deno' || + scriptLang === 'bunnative' || + scriptLang === 'nativets') + ) { + const resourceTypes = await ResourceService.listResourceType({ + workspace: $workspaceStore ?? '' + }) + + const namespace = formatResourceTypes( + resourceTypes, + scriptLang === 'bunnative' ? 'bun' : scriptLang + ) + + languages.typescript.typescriptDefaults.addExtraLib(namespace, 'rt.d.ts') + } + } + async function setTypescriptExtraLibs() { if (lang === 'typescript' && scriptLang != 'deno') { const hostname = getHostname() @@ -1314,7 +1353,7 @@ disposeMethod && disposeMethod() websocketInterval && clearInterval(websocketInterval) sqlSchemaCompletor && sqlSchemaCompletor.dispose() - copilotCompletor && copilotCompletor.dispose() + completorDisposable && completorDisposable.dispose() sqlTypeCompletor && sqlTypeCompletor.dispose() timeoutModel && clearTimeout(timeoutModel) }) @@ -1341,6 +1380,17 @@
{/if} +{#if $reviewingChanges} + { + aiChatEditorHandler?.acceptAll() + }} + on:rejectAll={() => { + aiChatEditorHandler?.rejectAll() + }} + /> +{/if} + diff --git a/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte new file mode 100644 index 0000000000..2d4da2bbe4 --- /dev/null +++ b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte @@ -0,0 +1,646 @@ + + + + +
+
+
+
+ {#if !disableHistory} + + + + {/if} +
+ {/if} + + {#if !disableMock && !isLoading} + + + Pin data + + {/if} + + {#if jsonView} +
+ +
+ {#if selectedJob && 'result' in selectedJob && displayResultJob && toolbarLocationJob === 'external'} + { + if (displayResultJob && typeof displayResultJob.openDrawer === 'function') { + displayResultJob.openDrawer() + } + }} + /> + {:else if mock?.enabled && displayResultMock && toolbarLocationMock === 'external'} + { + if (displayResultMock && typeof displayResultMock.openDrawer === 'function') { + displayResultMock.openDrawer() + } + }} + /> + {/if} +
+
+
+ + + +
+
{ + if ( + !event.target || + !(event.target instanceof HTMLElement) || + event.target.closest('[data-interactive]') || + event.target.closest('button') || + event.target.closest('input') || + event.target.closest('textarea') || + event.target.closest('select') || + event.target.closest('option') || + event.target.closest('label') || + event.target.closest('a') || + event.target.closest('svg') + ) { + dblClickDisabled = true + } else { + dblClickDisabled = false + } + }} + on:dblclick={() => { + if (canEditWithDblClick) { + stepHistoryPopover?.close() + jsonView = true + tmpMock = undefined + } + }} + on:mouseenter={() => { + hoveringResult = true + }} + on:mouseleave={() => { + hoveringResult = false + }} + > + {#if isLoading} +
+ +
+ {:else if connectingData || simpleViewer} + + {:else if jsonView} + { + if (mock?.enabled) { + const newMock = { + enabled: true, + return_value: structuredClone(detail) + } + tmpMock = newMock + } + }} + code={JSON.stringify( + mock?.enabled && mock.return_value ? mock.return_value : '', + null, + 2 + )} + class="h-full" + /> + {:else if (mock?.enabled || preview == 'mock') && preview != 'job'} + {#if fullResult} +
+ { + toolbarLocationMock = detail + }} + /> +
+ {:else} + + {/if} + {:else if selectedJob != undefined && 'result' in selectedJob} + {#if fullResult} +
+ {#key selectedJob} + { + toolbarLocationJob = detail + }} + > + + + + + {/key} +
+ {:else} + + {/if} + {:else if !lastJob} +
+

+ Test this step to see results{#if !disableMock} + or + {:else}.{/if} +

+
+ {/if} +
+
+
+ + diff --git a/frontend/src/lib/components/flows/propPicker/StepHistory.svelte b/frontend/src/lib/components/flows/propPicker/StepHistory.svelte new file mode 100644 index 0000000000..1fb5448801 --- /dev/null +++ b/frontend/src/lib/components/flows/propPicker/StepHistory.svelte @@ -0,0 +1,132 @@ + + + + + {#if mockValue} + +
+ + + + {mockEnabled ? 'Pin' : 'Last pin'} + +
+ + {/if} + + + + + + + + + + + +
+ {noHistory === 'isLoop' + ? 'History is not available with loops.' + : noHistory === 'isInsideLoop' + ? 'History is not available inside loops.' + : 'No run in history for this step'} +
+ + diff --git a/frontend/src/lib/components/flows/utils.ts b/frontend/src/lib/components/flows/utils.ts index 11146c2a17..b3270331ea 100644 --- a/frontend/src/lib/components/flows/utils.ts +++ b/frontend/src/lib/components/flows/utils.ts @@ -12,7 +12,7 @@ import { workspaceStore } from '$lib/stores' import { cleanExpr, emptySchema } from '$lib/utils' import { get } from 'svelte/store' import type { FlowModuleState } from './flowState' -import type { PickableProperties } from './previousResults' +import { type PickableProperties, dfs } from './previousResults' import { NEVER_TESTED_THIS_FAR } from './models' import { sendUserToast } from '$lib/toast' import type { Schema } from '$lib/common' @@ -316,3 +316,17 @@ export async function initFlowStepWarnings( return messages } + +export function checkIfParentLoop( + flowStore: ExtendedOpenFlow, + modId: string +): { id: string; type: 'forloopflow' | 'whileloopflow' } | undefined { + const flow: ExtendedOpenFlow = JSON.parse(JSON.stringify(flowStore)) + const parents = dfs(modId, flow, true) + for (const parent of parents.slice(1)) { + if (parent.value.type === 'forloopflow' || parent.value.type === 'whileloopflow') { + return { id: parent.id, type: parent.value.type } + } + } + return undefined +} diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 245426a041..6fa84591f2 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -71,6 +71,7 @@ ) export let triggerNode = false export let workspace: string = $workspaceStore ?? 'NO_WORKSPACE' + export let editMode = false let useDataflow: Writable = writable(false) @@ -218,6 +219,9 @@ minimizeSubflow: (id: string) => { delete expandedSubflows[id] expandedSubflows = expandedSubflows + }, + updateMock: () => { + dispatch('updateMock') } } @@ -242,7 +246,8 @@ path, newFlow, cache, - earlyStop + earlyStop, + editMode }, failureModule, preprocessorModule, diff --git a/frontend/src/lib/components/graph/graphBuilder.ts b/frontend/src/lib/components/graph/graphBuilder.ts index 4e37d6e863..0011106ade 100644 --- a/frontend/src/lib/components/graph/graphBuilder.ts +++ b/frontend/src/lib/components/graph/graphBuilder.ts @@ -17,6 +17,7 @@ export type GraphEventHandlers = { simplifyFlow: (detail: boolean) => void expandSubflow: (id: string, path: string) => void minimizeSubflow: (id: string) => void + updateMock: () => void } export type SimplifiableFlow = { simplifiedFlow: boolean } diff --git a/frontend/src/lib/components/graph/renderers/nodes/ForLoopStartNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/ForLoopStartNode.svelte index 596a32892f..278018360e 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/ForLoopStartNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/ForLoopStartNode.svelte @@ -12,9 +12,12 @@ offset: number id: string modules: FlowModule[] + module: FlowModule flowModuleStates: Record | undefined eventHandlers: GraphEventHandlers simplifiedTriggerView: boolean + selectedId: string + editMode: boolean } const propPickerContext = getContext('PropPickerContext') @@ -23,8 +26,16 @@ $: filteredInput = filterIterFromInput($pickablePropertiesFiltered?.flow_input) function filterIterFromInput(inputJson: Record | undefined): Record { - if (!inputJson || typeof inputJson !== 'object' || !inputJson.iter) return {} - return { iter: inputJson.iter } + if (!inputJson || typeof inputJson !== 'object' || (!inputJson.iter && !inputJson.iter_parent)) + return {} + const selectedIdIsDescendant = isSelectedDescendant(data.module, data.selectedId) + if (selectedIdIsDescendant === 'child') { + return { iter: inputJson.iter } + } + if (selectedIdIsDescendant === 'grandchild') { + return { iter_parent: inputJson.iter_parent } + } + return {} } function computeStatus(state: GraphModuleState | undefined): FlowStatusModule["type"] | undefined { @@ -34,10 +45,30 @@ return r ? 'Success' : 'InProgress' } } + + function isSelectedDescendant( + module: FlowModule, + selectedId: string + ): 'child' | 'grandchild' | 'none' { + if (!selectedId) return 'none' + // Check direct children + if (module.value.type === 'forloopflow' || module.value.type === 'whileloopflow') { + if (module.value.modules.some((m) => m.id === selectedId)) { + return 'child' + } + // Check grandchildren + return module.value.modules.some( + (m) => + (m.value.type === 'forloopflow' || m.value.type === 'whileloopflow') && + m.value.modules.some((gm) => gm.id === selectedId) + ) + ? 'grandchild' + : 'none' + } + return 'none' + } - - diff --git a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte index bdfcdd30cb..247870d0a4 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/InputNode.svelte @@ -7,7 +7,9 @@ import { getContext } from 'svelte' import type { Writable } from 'svelte/store' import InsertModuleButton from '$lib/components/flows/map/InsertModuleButton.svelte' - import type { PropPickerContext } from '$lib/components/prop_picker' + import { schemaToObject } from '$lib/schema' + import type { Schema } from '$lib/common' + import type { FlowEditorContext } from '$lib/components/flows/types' export let data: { hasPreprocessor: boolean @@ -20,25 +22,20 @@ disableMoveIds: string[] cache: boolean earlyStop: boolean + editMode: boolean } const { selectedId } = getContext<{ selectedId: Writable }>('FlowGraphContext') - const propPickerContext = getContext('PropPickerContext') - const pickablePropertiesFiltered = propPickerContext?.pickablePropertiesFiltered + const { previewArgs, flowStore } = + getContext('FlowEditorContext') || {} - function filterIterFromInput(inputJson: Record | undefined): Record { - if (!inputJson || typeof inputJson !== 'object') return {} - - const newJson = { ...inputJson } - delete newJson.iter - - return newJson - } - - $: filteredInput = filterIterFromInput($pickablePropertiesFiltered?.flow_input) + $: topFlowInput = + flowStore && previewArgs && $flowStore?.schema + ? schemaToObject($flowStore.schema as Schema, $previewArgs || {}) + : undefined @@ -82,10 +79,11 @@ on:select={(e) => { data.eventHandlers?.select(e.detail) }} - inputJson={filteredInput} + inputJson={topFlowInput} prefix="flow_input" alwaysPluggable cache={data.cache} earlyStop={data.earlyStop} + editMode={data.editMode} /> diff --git a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte index 05867a844e..66139fb034 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/ModuleNode.svelte @@ -26,6 +26,7 @@ eventHandlers: GraphEventHandlers flowModuleStates: Record | undefined selected: boolean + editMode: boolean } $: type = data.flowModuleStates?.[data.module.id]?.type @@ -42,15 +43,12 @@ flowJobsSuccess: state?.flow_jobs_success } : (undefined as any) - - - {#if data.module.value.type == 'flow'} + + { + if (componentInput?.value?.s3) { + s3FileUploadRawMode = true + } + }} + bind:selectedFileKey={componentInput.value} + /> + {:else if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} { diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte index 4758d1f357..85d975278b 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/UploadInputEditor.svelte @@ -15,6 +15,7 @@ export let s3: boolean | undefined = false export let prefix: string | undefined = undefined export let workspace: string | undefined = undefined + export let s3FileUploadRawMode: boolean = false let fileUploads: Writable = writable([]) @@ -38,11 +39,13 @@ return `${cleanPrefix}${file.name}` }} on:addition={({ detail }) => { + // @ts-ignore componentInput = { ...componentInput, - type: 'static', - value: `s3://${detail.path}` + type: 'uploadS3', + value: { s3: detail.path } } + s3FileUploadRawMode = true }} /> {:else} diff --git a/frontend/src/lib/components/apps/inputType.ts b/frontend/src/lib/components/apps/inputType.ts index 4bc2fc8532..e348300628 100644 --- a/frontend/src/lib/components/apps/inputType.ts +++ b/frontend/src/lib/components/apps/inputType.ts @@ -78,7 +78,7 @@ export type UploadInput = { export type UploadS3Input = { type: 'uploadS3' - value: string + value: any } export type FileUploadData = { diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index 519acbb1d9..ec94243179 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -15,9 +15,9 @@ focus-within:border-blue-500 hover:bg-blue-50 dark:hover:bg-frost-900 focus-with duration-200 rounded-lg p-1 gap-2" href={`${base}/api/w/${workspaceId ?? $workspaceStore}${ appPath ? `/apps_u/download_s3_file/${appPath}` : '/job_helpers/download_s3_file' - }?file_key=${encodeURIComponent(s3object?.s3 ?? '')}${ + }?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(s3object?.s3 ?? '')}${ s3object?.storage ? `&storage=${s3object.storage}` : '' - }`} + }${appPath && s3object?.presigned ? `&${s3object.presigned}` : ''}`} download={s3object?.s3.split('/').pop() ?? 'unnamed_download.file'} > diff --git a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte index 1d2555f577..9e5180b76c 100644 --- a/frontend/src/lib/components/common/fileUpload/FileUpload.svelte +++ b/frontend/src/lib/components/common/fileUpload/FileUpload.svelte @@ -19,8 +19,8 @@ export let containerText: string = folderOnly ? 'Drag and drop a folder here or click to browse' : allowMultiple - ? 'Drag and drop files here or click to browse' - : 'Drag and drop a file here or click to browse' + ? 'Drag and drop files here or click to browse' + : 'Drag and drop a file here or click to browse' export let customResourcePath: string | undefined = undefined export let customResourceType: 's3' | 'azure_blob' | undefined = undefined // when customResourcePath is provided, this should be provided as well. Will default to S3 if not export let customClass: string = '' @@ -70,7 +70,10 @@ | undefined) | undefined = undefined - const dispatch = createEventDispatcher() + const dispatch = createEventDispatcher<{ + addition: { path?: string; filename?: string } + deletion: { path: string } + }>() type FileUploadData = { name: string @@ -146,9 +149,9 @@ } else { path = typeof pathTransformer == 'function' - ? (await pathTransformer?.({ + ? ((await pathTransformer?.({ file: fileToUpload - })) ?? fileToUploadKey + })) ?? fileToUploadKey) : fileToUploadKey } const uploadData: FileUploadData = { @@ -269,10 +272,10 @@ appPath ? `/api/w/${ workspace ?? $workspaceStore - }/apps_u/upload_s3_file/${appPath}?${params.toString()}` + }/apps_u/upload_s3_file/${appPath}?${params.toString()}` : `/api/w/${ workspace ?? $workspaceStore - }/job_helpers/upload_s3_file?${params.toString()}`, + }/job_helpers/upload_s3_file?${params.toString()}`, true ) xhr?.setRequestHeader('Content-Type', 'application/octet-stream') @@ -469,10 +472,10 @@ color={fileUpload.errorMessage ? '#ef4444' : fileUpload.cancelled - ? '#eab308' - : fileUpload.progress === 100 - ? '#22c55e' - : '#3b82f6'} + ? '#eab308' + : fileUpload.progress === 100 + ? '#22c55e' + : '#3b82f6'} ended={fileUpload.cancelled || fileUpload.errorMessage !== undefined} > {#if fileUpload.errorMessage} diff --git a/frontend/src/lib/components/multiselect/MultiSelect.svelte b/frontend/src/lib/components/multiselect/MultiSelect.svelte index 5bf1a96ed3..1e4f00c66d 100644 --- a/frontend/src/lib/components/multiselect/MultiSelect.svelte +++ b/frontend/src/lib/components/multiselect/MultiSelect.svelte @@ -572,7 +572,7 @@ {/if} {/if} - {#if (searchText && noMatchingOptionsMsg) || options?.length > 0} + {#if allowUserOptions || (searchText && noMatchingOptionsMsg) || options?.length > 0}
0) { + $: if (portalRef && outerDiv && (allowUserOptions || items?.length > 0)) { tick().then(() => { moveOptionsToPortal() }) @@ -53,6 +53,7 @@ {#if !value || Array.isArray(value)}
{/if} @@ -385,20 +385,6 @@ code={JSON.stringify(static_asset_config ?? { s3: '' }, null, 2)} /> {/if} - {#if can_write} - - {/if} {:else} {#key is_static_website} {/key} {/if} + {#if can_write} + + {/if}
{:else} @@ -612,10 +612,10 @@ href={itemKind === 'flow' ? `/flows/add?${SECRET_KEY_PATH}=${encodeURIComponent(variable_path)}&hub=${ HubFlow.SIGNATURE_TEMPLATE - }` + }` : `/scripts/add?${SECRET_KEY_PATH}=${encodeURIComponent( variable_path - )}&hub=hub%2F${HUB_SCRIPT_ID}`} + )}&hub=hub%2F${HUB_SCRIPT_ID}`} target="_blank">Create from template diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 614d74faaf..d4ed2d7dd8 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -2,7 +2,6 @@ import { WorkspaceService, type AIConfig, type AIProvider } from '$lib/gen' import { setCopilotInfo, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import MultiSelect from 'svelte-multiselect' import { AI_DEFAULT_MODELS } from '../copilot/lib' import TestAiKey from '../copilot/TestAIKey.svelte' import Description from '../Description.svelte' @@ -11,6 +10,7 @@ import Toggle from '../Toggle.svelte' import ArgEnum from '../ArgEnum.svelte' import Button from '../common/button/Button.svelte' + import MultiSelectWrapper from '../multiselect/MultiSelectWrapper.svelte' const aiProviderLabels: [AIProvider, string][] = [ ['openai', 'OpenAI'], @@ -122,6 +122,22 @@ aiProviders = Object.fromEntries( Object.entries(aiProviders).filter(([key]) => key !== provider) ) + if (defaultModel) { + const currentDefaultModel = Object.values(aiProviders).find( + (p) => defaultModel && p.models.includes(defaultModel) + ) + if (!currentDefaultModel) { + defaultModel = undefined + } + } + if (codeCompletionModel) { + const currentCodeCompletionModel = Object.values(aiProviders).find( + (p) => codeCompletionModel && p.models.includes(codeCompletionModel) + ) + if (!currentCodeCompletionModel) { + codeCompletionModel = undefined + } + } } }} /> @@ -139,8 +155,8 @@ bind:value={aiProviders[provider].resource_path} on:change={() => { if ( - aiProviders[provider].resource_path && - aiProviders[provider].models.length === 0 && + aiProviders[provider]?.resource_path && + aiProviders[provider]?.models.length === 0 && AI_DEFAULT_MODELS[provider].length > 0 ) { aiProviders[provider].models = AI_DEFAULT_MODELS[provider].slice(0, 1) @@ -158,11 +174,11 @@ @@ -177,16 +193,18 @@

Settings

diff --git a/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte new file mode 100644 index 0000000000..51caaa1a42 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/StorageSettings.svelte @@ -0,0 +1,252 @@ + + + + + + +
+
+
Workspace Object Storage (S3/Azure Blob)
+ + Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users + to read and write from S3 without having to have access to the credentials. + +
+
+{#if !$enterpriseLicense} + + Windmill S3 bucket browser will not work for buckets containing more than 20 files and uploads + are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature with large + buckets. + +{:else} + + This setting is only for storage of large files allowing to upload files directly to object + storage using S3Object and use the wmill sdk to read and write large files backed by an object + storage. Large-scale log management and distributed dependency caching is under Instance object storage, set by the superadmins in the instance settings UI. + +{/if} +{#if s3ResourceSettings} +
+
+ + + + S3 + Azure Blob + AWS OIDC + Azure Workload Identity + +
+
+ + + + +
+
+ {#if s3ResourceSettings.resourceType == 's3'} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + + S3 resource public access is ON, which means that the entire content of the S3 bucket will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. Similarly, certain Windmill SDK endpoints can be used in scripts to + access the resource details, including public and private keys. + + {/if} +
+ {:else} +
+ + + + {#if s3ResourceSettings.publicResource === true} +
+ + object public access is ON, which means that the entire content of the object store will + be accessible to all the users of this workspace regardless of whether they have access + the resource or not. + + {/if} +
+ {/if} +
+
+ {#each s3ResourceSettings.secondaryStorage ?? [] as _, idx} +
+ s3ResourceSettings.secondaryStorage?.[idx]?.[0] || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][0] = v + } + } + } + placeholder="Storage name" + /> + + + + s3ResourceSettings.secondaryStorage?.[idx]?.[1].resourcePath || '', + (v) => { + if (s3ResourceSettings.secondaryStorage?.[idx]) { + s3ResourceSettings.secondaryStorage[idx][1].resourcePath = v + } + } + } + /> + + +
+ {/each} +
+ + + Secondary storage is a feature that allows you to read and write from storage that isn't + your main storage by specifying it in the s3 object as "secondary_storage" with the name + of it + +
+
+
+
+ +
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 2809592688..a672288ff2 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -10,7 +10,6 @@ import PageHeader from '$lib/components/PageHeader.svelte' import ResourcePicker from '$lib/components/ResourcePicker.svelte' import ScriptPicker from '$lib/components/ScriptPicker.svelte' - import S3FilePicker from '$lib/components/S3FilePicker.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import WorkspaceUserSettings from '$lib/components/settings/WorkspaceUserSettings.svelte' @@ -46,7 +45,6 @@ import PremiumInfo from '$lib/components/settings/PremiumInfo.svelte' import Toggle from '$lib/components/Toggle.svelte' - import Portal from '$lib/components/Portal.svelte' import { fade } from 'svelte/transition' import ChangeWorkspaceName from '$lib/components/settings/ChangeWorkspaceName.svelte' @@ -54,14 +52,14 @@ import ChangeWorkspaceColor from '$lib/components/settings/ChangeWorkspaceColor.svelte' import { convertBackendSettingsToFrontendSettings, - convertFrontendToBackendSetting, type S3ResourceSettings } from '$lib/workspace_settings' import { base } from '$lib/base' import { hubPaths } from '$lib/hub' import Description from '$lib/components/Description.svelte' import ConnectionSection from '$lib/components/ConnectionSection.svelte' - import AiSettings from '$lib/components/workspaceSettings/AISettings.svelte' + import AISettings from '$lib/components/workspaceSettings/AISettings.svelte' + import StorageSettings from '$lib/components/workspaceSettings/StorageSettings.svelte' type GitSyncTypeMap = { scripts: boolean @@ -89,8 +87,6 @@ | 'user' | 'group' - let s3FileViewer: S3FilePicker - let slackInitialPath: string let slackScriptPath: string let teamsInitialPath: string @@ -212,18 +208,6 @@ } } - async function editWindmillLFSSettings(): Promise { - const large_file_storage = convertFrontendToBackendSetting(s3ResourceSettings) - await WorkspaceService.editLargeFileStorageConfig({ - workspace: $workspaceStore!, - requestBody: { - large_file_storage: large_file_storage - } - }) - console.log('Large file storage settings changed', large_file_storage) - sendUserToast(`Large file storage settings changed`) - } - async function editWindmillGitSyncSettings(): Promise { let alreadySeenResource: string[] = [] let repositories = gitSyncSettings.repositories.map((elmt) => { @@ -629,10 +613,6 @@ $: updateFromSearchTab($page.url.searchParams.get('tab')) - - - - {#if $userStore?.is_admin || $superadmin}
{:else if tab == 'ai'} - {:else if tab == 'windmill_lfs'} -
-
-
Workspace Object Storage (S3/Azure Blob)
- - Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable - users to read and write from S3 without having to have access to the credentials. - -
-
- {#if !$enterpriseLicense} - - Windmill S3 bucket browser will not work for buckets containing more than 20 files and - uploads are limited to files {'<'} 50MB. Consider upgrading to Windmill EE to use this feature - with large buckets. - - {:else} - - This setting is only for storage of large files allowing to upload files directly to - object storage using S3Object and use the wmill sdk to read and write large files backed - by an object storage. Large-scale log management and distributed dependency caching is - under Instance object storage, set by the superadmins in the instance settings UI. - - {/if} - {#if s3ResourceSettings} -
-
- - S3 - Azure Blob - AWS OIDC - Azure Workload Identity - -
-
- - -
-
- {#if s3ResourceSettings.resourceType == 's3'} -
- - {#if s3ResourceSettings.publicResource === true} -
- - - S3 resource public access is ON, which means that the entire content of the S3 - bucket will be accessible to all the users of this workspace regardless of whether - they have access the resource or not. Similarly, certain Windmill SDK endpoints can - be used in scripts to access the resource details, including public and private - keys. - - {/if} -
- {:else} -
- - {#if s3ResourceSettings.publicResource === true} -
- - object public access is ON, which means that the entire content of the object store - will be accessible to all the users of this workspace regardless of whether they - have access the resource or not. - - {/if} -
- {/if} -
-
- {#each s3ResourceSettings.secondaryStorage ?? [] as secondaryStorage, idx} -
- - - - - -
- {/each} -
- - - Secondary storage is a feature that allows you to read and write from storage that - isn't your main storage by specifying it in the s3 object as "secondary_storage" - with the name of it - -
-
-
-
- -
- {/if} + {:else if tab == 'git_sync'}
diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index e1f16765ff..bca007b438 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -586,6 +586,12 @@ class Windmill: raise Exception("Could not write file to S3") from e return S3Object(s3=response["file_key"]) + def sign_s3_objects(self, s3_objects: list[S3Object]) -> list[S3Object]: + return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": s3_objects}).json() + + def sign_s3_object(self, s3_object: S3Object) -> S3Object: + return self.post(f"/w/{self.workspace}/apps/sign_s3_objects", json={"s3_objects": [s3_object]}).json()[0] + def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings: endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://" return Boto3ConnectionSettings( @@ -974,6 +980,24 @@ def write_s3_file( return _client.write_s3_file(s3object, file_content, s3_resource_path if s3_resource_path != "" else None, content_type, content_disposition) +@init_global_client +def sign_s3_objects(s3_objects: list[S3Object]) -> list[S3Object]: + """ + Sign S3 objects to be used by anonymous users in public apps + Returns a list of signed s3 tokens + """ + return _client.sign_s3_objects(s3_objects) + + +@init_global_client +def sign_s3_object(s3_object: S3Object) -> S3Object: + """ + Sign S3 object to be used by anonymous users in public apps + Returns a signed s3 object + """ + return _client.sign_s3_object(s3_object) + + @init_global_client def whoami() -> dict: """ diff --git a/python-client/wmill/wmill/s3_types.py b/python-client/wmill/wmill/s3_types.py index f633532591..b1736db9e9 100644 --- a/python-client/wmill/wmill/s3_types.py +++ b/python-client/wmill/wmill/s3_types.py @@ -1,6 +1,7 @@ class S3Object(dict): s3: str storage: str | None + presigned: str | None def __getattr__(self, attr): return self[attr] diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index 4d2859f204..06dae6605b 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -14,5 +14,5 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 73e2486d6b..5f9cdbc4ec 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -39,4 +39,4 @@ cp "${script_dirpath}/s3Types.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { S3Object, DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, loadS3FileStream, loadS3File, writeS3File, signS3Objects, signS3Object, task, runScript, runScriptAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, Sql } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 37fd4c9df6..258ea76215 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -3,10 +3,11 @@ import { VariableService, JobService, HelpersService, + AppService, MetricsService, OidcService, UserService, - TeamsService + TeamsService, } from "./index"; import { OpenAPI } from "./index"; // import type { DenoS3LightClientSettings } from "./index"; @@ -770,6 +771,33 @@ export async function writeS3File( }; } +/** + * Sign S3 objects to be used by anonymous users in public apps + * @param s3objects s3 objects to sign + * @returns signed s3 objects + */ +export async function signS3Objects( + s3objects: S3Object[] +): Promise { + const signedKeys = await AppService.signS3Objects({ + workspace: getWorkspace(), + requestBody: { + s3_objects: s3objects, + }, + }); + return signedKeys; +} + +/** + * Sign S3 object to be used by anonymous users in public apps + * @param s3object s3 object to sign + * @returns signed s3 object + */ +export async function signS3Object(s3object: S3Object): Promise { + const [signedObject] = await signS3Objects([s3object]); + return signedObject; +} + /** * Get URLs needed for resuming a flow after this step * @param approver approver name diff --git a/typescript-client/s3Types.ts b/typescript-client/s3Types.ts index 16641477d4..a46248d778 100644 --- a/typescript-client/s3Types.ts +++ b/typescript-client/s3Types.ts @@ -1,6 +1,7 @@ export type S3Object = { s3: string; storage?: string; + presigned?: string; }; export type DenoS3LightClientSettings = { From 39ebe18607c21fe1806618c5063e691e2da3e124 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Fri, 11 Apr 2025 17:20:24 +0100 Subject: [PATCH 112/133] allow 0ms duration for flows (#5608) --- frontend/src/lib/components/FlowTimeline.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/FlowTimeline.svelte b/frontend/src/lib/components/FlowTimeline.svelte index a5a444ac54..968b1bc612 100644 --- a/frontend/src/lib/components/FlowTimeline.svelte +++ b/frontend/src/lib/components/FlowTimeline.svelte @@ -72,7 +72,7 @@ } if (!isStillRunning) { - if (v.started_at && v.duration_ms) { + if (v.started_at && v.duration_ms != undefined) { let lmax = v.started_at + v.duration_ms if (!nmax) { nmax = lmax From 177e16bb18eed0d1c454b967aaa59547f61e8d26 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 11 Apr 2025 13:42:22 -0400 Subject: [PATCH 113/133] feat(frontend): app editor code input component (monaco) (#5566) * feat(frontend): app editor code input component (monaco) * only import when needed + svelte 5 * simple editor -> svelte5 * removing unneccessary rename * fix vimMode * nit fixes * fix height * rm global * add html support --------- Co-authored-by: Ruben Fiszel --- frontend/package-lock.json | 10 + frontend/package.json | 1 + .../src/lib/components/SimpleEditor.svelte | 186 ++++++++++++++---- .../inputs/AppCodeInputComponent.svelte | 104 ++++++++++ .../editor/component/ComponentInner.svelte | 3 + .../apps/editor/component/components.ts | 64 +++++- .../components/apps/editor/component/sets.ts | 1 + .../componentsPanel/componentControlUtils.ts | 1 + .../apps/editor/componentsPanel/cssUtils.ts | 13 ++ .../componentsPanel/quickStyleProperties.ts | 3 + 10 files changed, 344 insertions(+), 42 deletions(-) create mode 100644 frontend/src/lib/components/apps/components/inputs/AppCodeInputComponent.svelte diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c7127b4173..4ca3ed9a53 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -12,6 +12,7 @@ "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~11.1.2", "@codingame/monaco-vscode-standalone-css-language-features": "~11.1.2", + "@codingame/monaco-vscode-standalone-html-language-features": "^11.1.2", "@codingame/monaco-vscode-standalone-json-language-features": "~11.1.2", "@codingame/monaco-vscode-standalone-languages": "~11.1.2", "@codingame/monaco-vscode-standalone-typescript-language-features": "~11.1.2", @@ -567,6 +568,15 @@ "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@11.1.2" } }, + "node_modules/@codingame/monaco-vscode-standalone-html-language-features": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/@codingame/monaco-vscode-standalone-html-language-features/-/monaco-vscode-standalone-html-language-features-11.1.2.tgz", + "integrity": "sha512-PHRiZRH9ENI7hBJ7b+VSk78TCfQNqGVaS/fbr3jHcS98ohHdlfTRry/Kl1CT7aRIZj7yV9iNWX57hN+lzPp2vg==", + "license": "MIT", + "dependencies": { + "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@11.1.2" + } + }, "node_modules/@codingame/monaco-vscode-standalone-json-language-features": { "version": "11.1.2", "license": "MIT", diff --git a/frontend/package.json b/frontend/package.json index d2cb878ce5..f8ceffa287 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -85,6 +85,7 @@ "@aws-crypto/sha256-js": "^4.0.0", "@codingame/monaco-vscode-configuration-service-override": "~11.1.2", "@codingame/monaco-vscode-standalone-css-language-features": "~11.1.2", + "@codingame/monaco-vscode-standalone-html-language-features": "^11.1.2", "@codingame/monaco-vscode-standalone-json-language-features": "~11.1.2", "@codingame/monaco-vscode-standalone-languages": "~11.1.2", "@codingame/monaco-vscode-standalone-typescript-language-features": "~11.1.2", diff --git a/frontend/src/lib/components/SimpleEditor.svelte b/frontend/src/lib/components/SimpleEditor.svelte index cc3f120092..a7f390170c 100644 --- a/frontend/src/lib/components/SimpleEditor.svelte +++ b/frontend/src/lib/components/SimpleEditor.svelte @@ -1,12 +1,12 @@ - + + + + + +{#each Object.keys(components['codeinputcomponent'].initialData.configuration) as key (key)} + +{/each} + +{#if render} +
{ + e.stopPropagation() + if (!$connectingInput.opened) { + $selectedComponent = [id] + } + }} + > + {#await import('$lib/components/SimpleEditor.svelte')} +
+
Loading editor...
+
+ {:then Module} + + {/await} +
+{/if} + + diff --git a/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte b/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte index 71e3b3cf43..acbe2d5b83 100644 --- a/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte +++ b/frontend/src/lib/components/apps/editor/component/ComponentInner.svelte @@ -44,6 +44,7 @@ import AppTable from '../../components/display/table/AppTable.svelte' import AppAggridTable from '../../components/display/table/AppAggridTable.svelte' import AppText from '../../components/display/AppText.svelte' + import AppCodeInputComponent from '../../components/inputs/AppCodeInputComponent.svelte' import AppButton from '../../components/buttons/AppButton.svelte' import AppForm from '../../components/buttons/AppForm.svelte' import AppFormButton from '../../components/buttons/AppFormButton.svelte' @@ -295,6 +296,8 @@ componentInput={component.componentInput} {render} /> +{:else if component.type === 'codeinputcomponent'} + {:else if component.type === 'buttoncomponent'} export type TextInputComponent = BaseComponent<'textinputcomponent'> export type QuillComponent = BaseComponent<'quillcomponent'> +export type CodeInputComponent = BaseComponent<'codeinputcomponent'> export type TextareaInputComponent = BaseComponent<'textareainputcomponent'> export type PasswordInputComponent = BaseComponent<'passwordinputcomponent'> export type EmailInputComponent = BaseComponent<'emailinputcomponent'> @@ -304,6 +306,7 @@ export type TypedComponent = | JobIdFlowStatusComponent | TextInputComponent | QuillComponent + | CodeInputComponent | TextareaInputComponent | PasswordInputComponent | EmailInputComponent @@ -1204,6 +1207,65 @@ export const components = { } } }, + codeinputcomponent: { + name: 'Code Input', + icon: Code, + dims: '2:1-4:4' as AppComponentDimensions, + documentationLink: `${documentationBaseUrl}/code`, + customCss: { + text: { class: '', style: '' }, + container: { class: '', style: '' } + }, + initialData: { + componentInput: undefined, + configuration: { + placeholder: { + type: 'static', + value: 'Type...', + fieldType: 'text' + }, + defaultValue: { + type: 'static', + value: undefined, + fieldType: 'text' + }, + lang: { + type: 'static', + fieldType: 'select', + value: 'javascript', + selectOptions: [ + 'javascript', + 'typescript', + 'python', + 'sql', + 'json', + 'html', + 'css', + 'markdown', + 'yaml' + ] + }, + disableSuggestions: { + type: 'static', + fieldType: 'boolean', + value: false, + tooltip: 'Disable code completion suggestions' + }, + disableLinting: { + type: 'static', + fieldType: 'boolean', + value: false, + tooltip: 'Disable code validation/linting (keeps only syntax highlighting)' + }, + hideLineNumbers: { + type: 'static', + fieldType: 'boolean', + value: false, + tooltip: 'Hide line numbers in the editor' + } + } + } + }, buttoncomponent: { name: 'Button', icon: Inspect, diff --git a/frontend/src/lib/components/apps/editor/component/sets.ts b/frontend/src/lib/components/apps/editor/component/sets.ts index 999db0b63a..370e2703b2 100644 --- a/frontend/src/lib/components/apps/editor/component/sets.ts +++ b/frontend/src/lib/components/apps/editor/component/sets.ts @@ -37,6 +37,7 @@ const inputs: ComponentSet = { components: [ 'schemaformcomponent', 'textinputcomponent', + 'codeinputcomponent', 'textareainputcomponent', 'quillcomponent', 'passwordinputcomponent', diff --git a/frontend/src/lib/components/apps/editor/componentsPanel/componentControlUtils.ts b/frontend/src/lib/components/apps/editor/componentsPanel/componentControlUtils.ts index c8a6e06dbd..0df79e01f4 100644 --- a/frontend/src/lib/components/apps/editor/componentsPanel/componentControlUtils.ts +++ b/frontend/src/lib/components/apps/editor/componentsPanel/componentControlUtils.ts @@ -121,6 +121,7 @@ export function getComponentControl(type: keyof typeof components): Array Date: Fri, 11 Apr 2025 20:00:57 +0200 Subject: [PATCH 114/133] improve public app performances --- .../flows/propPicker/OutputPickerInner.svelte | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte index 468101ea64..4b820e08f9 100644 --- a/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte +++ b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte @@ -18,7 +18,6 @@ import Button from '$lib/components/common/button/Button.svelte' import { Pin, History, Pen, Check, X, Loader2 } from 'lucide-svelte' import ObjectViewer from '$lib/components/propertyPicker/ObjectViewer.svelte' - import JsonEditor from '$lib/components/JsonEditor.svelte' import StepHistory from './StepHistory.svelte' import { Popover } from '$lib/components/meltComponents' import { createEventDispatcher } from 'svelte' @@ -535,9 +534,12 @@ {allowCopy} /> {:else if jsonView} - + {:then Module} + { if (mock?.enabled) { const newMock = { @@ -552,8 +554,9 @@ null, 2 )} - class="h-full" - /> + class="h-full" + /> + {/await} {:else if (mock?.enabled || preview == 'mock') && preview != 'job'} {#if fullResult}
From cdb0e42979c527142f413dcabc43b5fc67d2b8a5 Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 11 Apr 2025 16:35:27 -0400 Subject: [PATCH 115/133] github app linking: do the redirect via browser (#5611) --- frontend/src/routes/gh_success/+page.svelte | 105 ++++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 frontend/src/routes/gh_success/+page.svelte diff --git a/frontend/src/routes/gh_success/+page.svelte b/frontend/src/routes/gh_success/+page.svelte new file mode 100644 index 0000000000..01234945d5 --- /dev/null +++ b/frontend/src/routes/gh_success/+page.svelte @@ -0,0 +1,105 @@ + + +
+
+ {#if isLoading} +
+
+

Processing GitHub app installation...

+
+ {:else if isSuccess} +
+

+ + Windmill GitHub app installation completed successfully +

+

+ The GitHub app has been successfully installed. You can now close this window and return to Windmill to start using the GitHub integration. +

+ +
+ {:else} +
+

+ + Failed to install Windmill GitHub app +

+

There was an error during the installation process:

+
+ {errorMessage} +
+

+ Please try again or contact your administrator for assistance. +

+ +
+ {/if} +
+
From d5186da27129f77f302d3fed54ecdc333cdceeab Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Apr 2025 23:19:29 +0200 Subject: [PATCH 116/133] allow multiple workers on agent mode (#5607) --- ...05d394a7cbcf0038c72a78add5c7b02ef5927.json | 2 +- ...1fe2dce615ff76a53f72cb5386dc17e4e07aa.json | 21 +++++ backend/Cargo.lock | 11 +++ backend/Cargo.toml | 2 + backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 84 +++++++++-------- backend/windmill-api/src/jobs.rs | 15 +++ backend/windmill-api/src/lib.rs | 6 +- backend/windmill-worker/Cargo.toml | 1 + .../windmill-worker/src/result_processor.rs | 31 ++++-- backend/windmill-worker/src/worker.rs | 94 +++++++++++-------- 11 files changed, 176 insertions(+), 93 deletions(-) create mode 100644 backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json diff --git a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json index 5bfff47576..c2dfed73a2 100644 --- a/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json +++ b/backend/.sqlx/query-ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "ddf2eccb78a310ed00c7d8b9c3f05d394a7cbcf0038c72a78add5c7b02ef5927" diff --git a/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json b/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json new file mode 100644 index 0000000000..1ca60067d0 --- /dev/null +++ b/backend/.sqlx/query-fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "UuidArray", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "fe7221651a982861dede4116bc71fe2dce615ff76a53f72cb5386dc17e4e07aa" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 46ef0b5b2d..4737d318f6 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4804,6 +4804,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", + "nanorand", "spin 0.9.8", ] @@ -7519,6 +7520,15 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "napi_sym" version = "0.120.0" @@ -14480,6 +14490,7 @@ dependencies = [ "deno_webidl", "dotenv", "dyn-iter", + "flume", "futures", "gcp_auth", "git-version", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 4ca7a5fa49..ef4d7c5712 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -365,6 +365,8 @@ tantivy = "0.22.0" backon = "1.3.0" +flume = { version = "0.11.1", features = ["async"] } + # Macro-related proc-macro2 = "1.0" pulldown-cmark = "0.9" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b9a3d2881d..9c55ceeb81 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -85c37983ffb8f622458425c182613206625c6cee +44c3e23922097d4386183e56b6b3dc540c70d71b \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index a076b4c63f..7f4bcda628 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -329,26 +329,16 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; - let mut first_worker_suffix = None; - let mut worker_names = vec![]; - - for _ in 0..num_workers { + let (conn, first_suffix) = if mode == Mode::Agent { + tracing::info!( + "Creating http client for cluster using base internal url {}", + std::env::var("BASE_INTERNAL_URL").unwrap_or_default() + ); let suffix = windmill_common::utils::worker_suffix(&hostname, &rd_string(5)); - worker_names.push(windmill_common::utils::worker_name_with_suffix( - mode == Mode::Agent, - WORKER_GROUP.as_str(), - &suffix, - )); - if first_worker_suffix.is_none() { - first_worker_suffix = Some(suffix); - } - } - - let conn = if mode == Mode::Agent { - let worker_suffix = first_worker_suffix.unwrap_or_else(|| { - panic!("there must be at least one worker in agent mode"); - }); - Connection::Http(build_agent_http_client(&worker_suffix)) + ( + Connection::Http(build_agent_http_client(&suffix)), + Some(suffix), + ) } else { println!("Connecting to database..."); @@ -366,7 +356,7 @@ async fn windmill_main() -> anyhow::Result<()> { load_otel(&db).await; tracing::info!("Database connected"); - Connection::Sql(db) + (Connection::Sql(db), None) }; let environment = load_base_url(&conn) @@ -409,6 +399,7 @@ async fn windmill_main() -> anyhow::Result<()> { let conn = if mode == Mode::Agent { conn } else { + // This time we use a pool of connections let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; Connection::Sql(db) }; @@ -439,16 +430,6 @@ Windmill Community Edition {GIT_VERSION} display_config(&ENV_SETTINGS); - if let Err(e) = reload_base_url_setting(&conn).await { - tracing::error!("Error loading base url: {:?}", e) - } - - if let Some(db) = conn.as_sql() { - if let Err(e) = reload_critical_error_channels_setting(&db).await { - tracing::error!("Could loading critical error emails setting: {:?}", e); - } - } - #[cfg(feature = "enterprise")] { // load the license key and check if it's valid @@ -686,14 +667,34 @@ Windmill Community Edition {GIT_VERSION} if !killpill_rx.try_recv().is_ok() { let base_internal_url = base_internal_rx.await?; if worker_mode { + let mut workers = vec![]; + for i in 0..num_workers { + let suffix: String = if i == 0 && first_suffix.as_ref().is_some() { + first_suffix.as_ref().unwrap().clone() + } else { + windmill_common::utils::worker_suffix(&hostname, &rd_string(5)) + }; + let worker_conn = WorkerConn { + conn: if i == 0 || mode != Mode::Agent { + conn.clone() + } else { + Connection::Http(build_agent_http_client(&suffix)) + }, + worker_name: windmill_common::utils::worker_name_with_suffix( + mode == Mode::Agent, + WORKER_GROUP.as_str(), + &suffix, + ), + }; + workers.push(worker_conn); + } + run_workers( - conn.clone(), rx, killpill_tx.clone(), - num_workers, base_internal_url.clone(), hostname.clone(), - &worker_names, + &workers, ) .await?; tracing::info!("All workers exited."); @@ -1134,16 +1135,20 @@ fn display_config(envs: &[&str]) { ) } +pub struct WorkerConn { + conn: Connection, + worker_name: String, +} + pub async fn run_workers( - db: Connection, mut rx: tokio::sync::broadcast::Receiver<()>, tx: KillpillSender, - num_workers: i32, base_internal_url: String, hostname: String, - worker_names: &[String], + workers: &[WorkerConn], ) -> anyhow::Result<()> { let mut killpill_rxs = vec![]; + let num_workers = workers.len(); for _ in 0..num_workers { killpill_rxs.push(rx.resubscribe()); } @@ -1202,8 +1207,9 @@ pub async fn run_workers( *windmill_worker::SLEEP_QUEUE ); for i in 1..(num_workers + 1) { - let db1 = db.clone(); - let worker_name = worker_names[i as usize - 1].clone(); + let wk_conf = &workers[i as usize - 1]; + let conn1 = wk_conf.conn.clone(); + let worker_name = wk_conf.worker_name.clone(); let ip = ip.clone(); let rx = killpill_rxs.pop().unwrap(); let tx = tx.clone(); @@ -1216,7 +1222,7 @@ pub async fn run_workers( } let f = windmill_worker::run_worker( - &db1, + &conn1, &hostname, worker_name, i as u64, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 71ec9c6c8f..4111f28b6b 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -5095,6 +5095,21 @@ async fn add_batch_jobs( .execute(&mut *tx) .await?; + sqlx::query!( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + &uuids, + authed.email, + authed.username, + authed.is_admin, + authed.is_operator, + &[], + &[], + w_id, + ) + .execute(&mut *tx) + .await?; + if let Some(flow_status) = flow_status { sqlx::query!( "INSERT INTO v2_job_status (id, flow_status) diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 922cb57ceb..ece80c64b1 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -728,10 +728,10 @@ pub async fn run_server( server.await?; #[cfg(feature = "agent_worker_server")] - if let Some(bg_processor) = agent_workers_bg_processor { - tracing::info!("server off. shutting down agent workers bg processor"); + for (i, bg_processor) in agent_workers_bg_processor.into_iter().enumerate() { + tracing::info!("server off. shutting down agent worker bg processor {i}"); bg_processor.await?; - tracing::info!("agent workers bg processor shut down"); + tracing::info!("agent worker bg processor {i} shut down"); } Ok(()) } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index cad3ccc84a..e4de400d84 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -52,6 +52,7 @@ windmill-parser-sql.workspace = true windmill-parser-graphql.workspace = true windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true +flume.workspace = true sqlx.workspace = true uuid.workspace = true tracing.workspace = true diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 141c886fe0..389bd067a8 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -34,7 +34,7 @@ use windmill_queue::{ use serde_json::{json, value::RawValue}; -use tokio::{sync::mpsc::Receiver, task::JoinHandle}; +use tokio::{sync::broadcast, task::JoinHandle}; use windmill_queue::{add_completed_job, add_completed_job_error}; @@ -118,7 +118,7 @@ async fn process_jc( } pub fn start_background_processor( - mut job_completed_rx: Receiver, + job_completed_rx: flume::Receiver, job_completed_sender: JobCompletedSender, same_worker_queue_size: Arc, job_completed_processor_is_done: Arc, @@ -127,6 +127,7 @@ pub fn start_background_processor( worker_dir: String, same_worker_tx: SameWorkerSender, worker_name: String, + mut killpill_rx: broadcast::Receiver<()>, killpill_tx: KillpillSender, is_dedicated_worker: bool, ) -> JoinHandle<()> { @@ -136,19 +137,33 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] let mut infos = BenchmarkInfo::new(); + enum JobCompletedRx { + JobCompleted(SendResult), + Killpill, + } //if we have been killed, we want to drain the queue of jobs while let Some(sr) = { if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 { - job_completed_rx.try_recv().ok() + job_completed_rx + .try_recv() + .ok() + .map(JobCompletedRx::JobCompleted) } else { - job_completed_rx.recv().await + tokio::select! { + result = job_completed_rx.recv_async() => { + result.ok().map(JobCompletedRx::JobCompleted) + } + _ = killpill_rx.recv() => { + Some(JobCompletedRx::Killpill) + } + } } } { #[cfg(feature = "benchmark")] let mut bench = BenchmarkIter::new(); match sr { - SendResult::JobCompleted(jc) => { + JobCompletedRx::JobCompleted(SendResult::JobCompleted(jc)) => { let is_init_script_and_failure = !jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG; let is_dependency_job = matches!( @@ -192,7 +207,7 @@ pub fn start_background_processor( infos.add_iter(bench, true); } } - SendResult::UpdateFlow { + JobCompletedRx::JobCompleted(SendResult::UpdateFlow { flow, w_id, success, @@ -200,7 +215,7 @@ pub fn start_background_processor( worker_dir, stop_early_override, token, - } => { + }) => { // let r; tracing::info!(parent_flow = %flow, "updating flow status"); if let Err(e) = update_flow_status_after_job_completion( @@ -230,7 +245,7 @@ pub fn start_background_processor( tracing::error!("Error updating flow status after job completion for {flow} on {worker_name}: {e:#}"); } } - SendResult::Kill => { + JobCompletedRx::Killpill => { has_been_killed = true; } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 46420fb73e..2081b90f5f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -85,6 +85,7 @@ use tokio::fs::symlink_file as symlink; use tokio::{ sync::{ + broadcast, mpsc::{self, Receiver, Sender}, RwLock, }, @@ -210,8 +211,8 @@ pub const DEFAULT_NATIVE_JOBS: usize = 1; const VACUUM_PERIOD: u32 = 50000; -#[cfg(any(target_os = "linux"))] -const DROP_CACHE_PERIOD: u32 = 1000; +// #[cfg(any(target_os = "linux"))] +// const DROP_CACHE_PERIOD: u32 = 1000; pub const MAX_BUFFERED_DEDICATED_JOBS: usize = 3; @@ -518,20 +519,33 @@ impl AuthedClient { } } +#[derive(Clone)] +pub struct SameWorkerSender(pub Sender, pub Arc); + #[allow(dead_code)] #[derive(Clone)] pub enum JobCompletedSender { - Sql(Sender), + Sql(flume::Sender, broadcast::Sender<()>), Http(HttpClient), NeverUsed, } impl JobCompletedSender { - pub fn new(conn: &Connection, buffer_size: usize) -> (Self, Option>) { + pub fn new( + conn: &Connection, + buffer_size: usize, + ) -> ( + Self, + Option<(flume::Receiver, broadcast::Receiver<()>)>, + ) { match conn { Connection::Sql(_) => { - let (sender, receiver) = mpsc::channel::(buffer_size); - (Self::Sql(sender), Some(receiver)) + let (sender, receiver) = flume::bounded::(buffer_size); + let (killpill_tx, killpill_rx) = broadcast::channel::<()>(buffer_size); + ( + Self::Sql(sender, killpill_tx), + Some((receiver, killpill_rx)), + ) } Connection::Http(client) => (Self::Http(client.clone()), None), } @@ -539,16 +553,11 @@ impl JobCompletedSender { pub fn new_never_used() -> (Self, Option>) { (Self::NeverUsed, None) } -} -#[derive(Clone)] -pub struct SameWorkerSender(pub Sender, pub Arc); - -impl JobCompletedSender { pub async fn send_job(&self, jc: JobCompleted) -> anyhow::Result<()> { match self { - Self::Sql(sender) => sender - .send(SendResult::JobCompleted(jc)) + Self::Sql(sender, _) => sender + .send_async(SendResult::JobCompleted(jc)) .await .map_err(|_e| { anyhow::anyhow!("Failed to send job completed to background processor") @@ -566,12 +575,9 @@ impl JobCompletedSender { } } - pub async fn send( - &self, - send_result: SendResult, - ) -> Result<(), tokio::sync::mpsc::error::SendError> { + pub async fn send(&self, send_result: SendResult) -> Result<(), flume::SendError> { match self { - Self::Sql(sender) => sender.send(send_result).await, + Self::Sql(sender, _) => sender.send_async(send_result).await, Self::Http(_) => { tracing::error!("Sending job completed to http client, this should not happen"); Ok(()) @@ -585,9 +591,13 @@ impl JobCompletedSender { } } - pub async fn kill(&self) -> Result<(), tokio::sync::mpsc::error::SendError> { + pub async fn kill(&self) -> Result<(), broadcast::error::SendError<()>> { match self { - Self::Sql(sender) => sender.send(SendResult::Kill).await, + Self::Sql(_, killpill_tx) => { + tracing::info!("Sending killpill to bg processors"); + killpill_tx.send(())?; + Ok(()) + } Self::Http(_) => { tracing::error!("Sending kill to http client, this should not happen"); Ok(()) @@ -627,11 +637,11 @@ pub async fn drop_cache() { Ok(mut file) => { // Write '3' to the file to drop caches if let Err(e) = tokio::io::AsyncWriteExt::write_all(&mut file, b"3").await { - tracing::warn!("Failed to write to /proc/sys/vm/drop_caches (expected to not work in not in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer): {}", e); + tracing::warn!("Failed to write to /proc/sys/vm/drop_caches (expected to work only in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer): {}", e); } } Err(e) => { - tracing::warn!("Failed to open /proc/sys/vm/drop_caches (expected to not work in not in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer):: {}", e); + tracing::warn!("Failed to open /proc/sys/vm/drop_caches (expected to work only in privileged mode, only required to forcefully drop the cache to avoid spurrious oom killer):: {}", e); } } } @@ -1012,19 +1022,22 @@ pub async fn run_worker( Arc::new(AtomicBool::new(matches!(conn, Connection::Http(_)))); let send_result = match (conn, job_completed_rx) { - (Connection::Sql(db), Some(job_completed_rx)) => Some(start_background_processor( - job_completed_rx, - job_completed_tx.clone(), - same_worker_queue_size.clone(), - job_completed_processor_is_done.clone(), - base_internal_url.to_string(), - db.clone(), - worker_dir.clone(), - same_worker_tx.clone(), - worker_name.clone(), - killpill_tx.clone(), - is_dedicated_worker, - )), + (Connection::Sql(db), Some((job_completed_rx, bg_killpill_rx))) => { + Some(start_background_processor( + job_completed_rx, + job_completed_tx.clone(), + same_worker_queue_size.clone(), + job_completed_processor_is_done.clone(), + base_internal_url.to_string(), + db.clone(), + worker_dir.clone(), + same_worker_tx.clone(), + worker_name.clone(), + bg_killpill_rx, + killpill_tx.clone(), + is_dedicated_worker, + )) + } _ => None, }; @@ -1190,11 +1203,11 @@ pub async fn run_worker( jobs_executed += 1; } - #[cfg(any(target_os = "linux"))] - if (jobs_executed as u32 + 1) % DROP_CACHE_PERIOD == 0 { - drop_cache().await; - jobs_executed += 1; - } + // #[cfg(any(target_os = "linux"))] + // if (jobs_executed as u32 + 1) % DROP_CACHE_PERIOD == 0 { + // drop_cache().await; + // jobs_executed += 1; + // } #[cfg(feature = "benchmark")] if benchmark_jobs > 0 && infos.iters == benchmark_jobs as u64 { @@ -1843,7 +1856,6 @@ pub enum SendResult { stop_early_override: Option, token: String, }, - Kill, } async fn do_nativets( From 0b6d017fedc31e790a76cf29a1adaaf2a72acc61 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 11 Apr 2025 23:31:51 +0200 Subject: [PATCH 117/133] feat(python): per import requirement pin (#5520) * implement single line pin * make panic-safe * use pin even if multiple modules imported withing single statement * add repins and make imports respect pins * keep all pins * Allow multiple pins * add comments + handle stuff more safely * fix fully qualified imports * remove ignore * sort nested * apply unique to output requirements list * fix typo * remove mut * update sqlx * sort imports * sort imports * fix formatter and format * refactor * fix comptime error * write tests * perf: do not capture if string is empty --- ...43c98104529879f991c49585cd181e34aa827.json | 23 + ...9e631efb65c3d326b6b6ae8361a2116bff145.json | 23 - .../windmill-parser-py-imports/src/lib.rs | 412 ++++++++++++++---- .../windmill-parser-py-imports/tests/tests.rs | 6 +- backend/tests/fixtures/lockfile_python.sql | 51 +++ backend/tests/worker.rs | 203 +++++++++ .../windmill-worker/src/python_executor.rs | 36 +- .../windmill-worker/src/worker_lockfiles.rs | 1 + 8 files changed, 638 insertions(+), 117 deletions(-) create mode 100644 backend/.sqlx/query-443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827.json delete mode 100644 backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json create mode 100644 backend/tests/fixtures/lockfile_python.sql diff --git a/backend/.sqlx/query-443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827.json b/backend/.sqlx/query-443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827.json new file mode 100644 index 0000000000..65b21050c0 --- /dev/null +++ b/backend/.sqlx/query-443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND\n workspace_id = $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "content", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "443bd83bcea1d37c79cb080095343c98104529879f991c49585cd181e34aa827" +} diff --git a/backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json b/backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json deleted file mode 100644 index 929157b5d7..0000000000 --- a/backend/.sqlx/query-ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT content FROM script WHERE path = $1 AND workspace_id = $2\n AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND\n workspace_id = $2)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "content", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "ac01e45d3335015f53f3d63fe159e631efb65c3d326b6b6ae8361a2116bff145" -} diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index f0c8a4c41a..2f18b74d6b 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -11,6 +11,7 @@ mod mapping; use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; +use std::collections::HashMap; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -20,6 +21,7 @@ use regex_lite::Regex; use rustpython_parser::{ ast::{Stmt, StmtImport, StmtImportFrom, Suite}, + text_size::TextRange, Parse, }; use sqlx::{Pool, Postgres}; @@ -41,9 +43,10 @@ fn replace_full_import(x: &str) -> Option { lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)\s*$").unwrap(); + static ref PIN_RE: Regex = Regex::new(r"(?:\s*#\s*(pin|repin):\s*)(\S*)").unwrap(); } -fn process_import(module: Option, path: &str, level: usize) -> Vec { +fn process_import(module: Option, path: &str, level: usize) -> Vec { if level > 0 { let mut imports = vec![]; let splitted_path = path.split("/"); @@ -52,17 +55,18 @@ fn process_import(module: Option, path: &str, level: usize) -> Vec error::Result Some(path), + _ => None, }) .collect()); } -fn parse_code_for_imports(code: &str, path: &str) -> error::Result> { +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +enum NImport { + // Order matters! First we want to resolve all repins + + // manually repinned requirement + // e.g.: + // import pandas # repin: pandas==x.y.z + Repin { + pin: ImportPin, + key: String, + }, + // manually pinned requirements + // e.g.: + // import pandas # pin: pandas>=x.y.z + // import pandas # pin: pandas<=x.y.z + // + // NOTE: It is possible for multiple pins exist on same import + // That's why we store vector of pins + Pin { + pins: Vec, + key: String, + }, + // Automatically inferred requirement + // e.g.: + // import pandas + Auto { + // Take `x.y.z` for example + // x is going to be the `root` + // and x.y.z is `full` + // + // `full` will be None if it is equal to root + // + // We will use `root` as a requirement name and pass to `uv pip compile` if it was not replaced with any pin + pkg: String, + + // However we still need full, since all pins pin against full import names + key: Option, + }, + // Relative imports + Relative(String), +} +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +enum NImportResolved { + Repin { pin: ImportPin, key: String }, + Pin { pins: Vec, key: String }, + Auto { pkg: String, key: Option }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +struct ImportPin { + pkg: String, + path: String, +} + +fn parse_code_for_imports(code: &str, path: &str) -> error::Result> { let mut code = code.split(DEF_MAIN).next().unwrap_or("").to_string(); // remove main function decorator from end of file if it exists @@ -104,19 +159,56 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> let ast = Suite::parse(&code, "main.py").map_err(|e| { error::Error::ExecutionErr(format!("Error parsing code for imports: {}", e.to_string())) })?; - let nimports: Vec = ast + + let find_pin = |range: TextRange, key: String| { + let hs = code + .chars() + .skip(range.end().to_usize()) + .take_while(|e| *e != '\n') + .collect::(); + + if hs.trim_start().is_empty(){ + return None; + } + + PIN_RE + .captures(&hs) + .and_then(|x| { + x.get(1).zip(x.get(2)).and_then(|(ty_m, pkg_m)| { + let pkg = pkg_m.as_str().to_owned(); + if ty_m.as_str() == "pin" { + Some(vec![NImport::Pin { + pins: vec![ImportPin { pkg, path: path.to_owned() }], + key, + }]) + } else if ty_m.as_str() == "repin" { + Some(vec![NImport::Repin { + pin: ImportPin { pkg, path: path.to_owned() }, + key, + }]) + } else { + None + } + }) + }) + }; + + let mut nimports: Vec = ast .into_iter() .filter_map(|x| match x { - Stmt::Import(StmtImport { names, .. }) => Some( - names - .into_iter() - .map(|x| { - let name = x.name.to_string(); - process_import(Some(name), path, 0) - }) - .flatten() - .collect::>(), - ), + Stmt::Import(StmtImport { names, range }) => names + .get(0) + .and_then(|al| find_pin(range, al.name.to_string())) + .or(Some( + names + .into_iter() + .map(|x| { + let name = x.name.to_string(); + process_import(Some(name), path, 0) + }) + .flatten() + .collect::>(), + )), Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => { Some(process_import( module.map(|x| x.to_string()), @@ -124,15 +216,25 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> i.to_usize(), )) } - Stmt::ImportFrom(StmtImportFrom { level: _, module, .. }) => { - Some(process_import(module.map(|x| x.to_string()), path, 0)) - } + Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin( + range, + module.clone().map(|x| x.to_string()).unwrap_or_default(), + ) + .or(Some(process_import(module.map(|x| x.to_string()), path, 0))), _ => None, }) .flatten() - .filter(|x| !STDIMPORTS.contains(&x.as_str())) + .filter(|x| { + if let NImport::Auto { ref pkg, .. } = x { + !STDIMPORTS.contains(&(*pkg).as_str()) + } else { + true + } + }) .unique() .collect(); + + nimports.sort(); return Ok(nimports); } @@ -143,8 +245,9 @@ pub async fn parse_python_imports( db: &Pool, already_visited: &mut Vec, annotated_pyv_numeric: &mut Option, -) -> error::Result> { - parse_python_imports_inner( +) -> error::Result<(Vec, Option)> { + let mut compile_error_hint: Option = None; + let mut imports = parse_python_imports_inner( code, w_id, path, @@ -153,7 +256,32 @@ pub async fn parse_python_imports( annotated_pyv_numeric, &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())), ) - .await + .await? + .into_values() + .map(|nimport| match nimport { + NImportResolved::Pin { pins, .. } => pins.into_iter().map(|p| { + if let Some(hint) = &mut compile_error_hint{ + hint.push_str(&format!("\n - pin to {} in {}", p.pkg, p.path)); + } else { + compile_error_hint = Some("\n\nMultiple pins can cause problems during lockfile resolution.\nMake sure you checked every pin for conflicts:\n".into()) + }; + Ok(p.pkg) + }).collect_vec(), + NImportResolved::Repin { pin: ImportPin { pkg, .. }, .. } => vec![Ok(pkg)], + NImportResolved::Auto { pkg, ..} => vec![Ok(pkg)], + }) + .flatten() + .collect::>>()? + .into_iter() + .unique() + .collect_vec(); + + imports.sort(); + + compile_error_hint + .as_mut() + .map(|e| e.push_str("\n\nNOTE: You can also `repin` to override all pins")); + Ok((imports, compile_error_hint)) } #[async_recursion] @@ -165,7 +293,7 @@ async fn parse_python_imports_inner( already_visited: &mut Vec, annotated_pyv_numeric: &mut Option, path_where_annotated_pyv: &mut Option, -) -> error::Result> { +) -> error::Result> { let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); // we pass only if there is none or only one annotation @@ -194,7 +322,6 @@ async fn parse_python_imports_inner( } else { *annotated_pyv_numeric = Some(numeric); } - *path_where_annotated_pyv = Some(path.to_owned()); } Ok(()) @@ -209,74 +336,205 @@ async fn parse_python_imports_inner( .lines() .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); if let Some((pos, _)) = find_requirements { - let lines = code - .lines() + let mut requirements = HashMap::new(); + code.lines() .skip(pos + 1) .map_while(|x| { - RE.captures(x) - .map(|x| x.get(1).unwrap().as_str().to_string()) + RE.captures(x).and_then(|x| { + x.get(1).map(|m| { + let requirement = m.as_str().to_string(); + requirements.insert( + requirement.clone(), + NImportResolved::Repin { + pin: ImportPin { pkg: requirement, path: Default::default() }, + key: Default::default(), + }, + ); + }) + }) }) - .collect(); - Ok(lines) + .collect_vec(); + + Ok(requirements) } else { let find_extra_requirements = code.lines().find_position(|x| { x.starts_with("#extra_requirements:") || x.starts_with("# extra_requirements:") }); - let mut imports: Vec = vec![]; + let mut imports: HashMap = HashMap::new(); if let Some((pos, _)) = find_extra_requirements { - let lines: Vec = code - .lines() + code.lines() .skip(pos + 1) .map_while(|x| { - RE.captures(x) - .map(|x| x.get(1).unwrap().as_str().to_string()) + RE.captures(x).and_then(|x| { + x.get(1).map(|m| { + let requirement = m.as_str().to_string(); + imports.insert( + requirement.clone(), + NImportResolved::Auto { key: None, pkg: requirement }, + ); + }) + }) }) - .collect(); - imports.extend(lines); + .collect_vec(); } - let nimports = parse_code_for_imports(code, path)?; - for n in nimports.iter() { - let nested = if n.starts_with("relative:") { - let rpath = n.replace("relative:", ""); - let code = sqlx::query_scalar!( - r#" - SELECT content FROM script WHERE path = $1 AND workspace_id = $2 - AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND - workspace_id = $2) - "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()); + // Will get unsorted vector of imports found in current script + let mut nimports = parse_code_for_imports(code, path)?; - if already_visited.contains(&rpath) { - vec![] - } else { - already_visited.push(rpath.clone()); - parse_python_imports_inner( - &code, - w_id, + // It is important to note, that sorting is important and will always result in this pattern: + // 1. All Repins go first + // 2. All Pins go second + // 3. All Auto go third + // 4. All relative imports go the last + // + // This way we make sure all repins are resolved before (re)pins inside imported relative scripts. + nimports.sort(); + + for n in nimports.into_iter() { + let mut nested = match n { + NImport::Relative(rpath) => { + let code = sqlx::query_scalar!( + r#" + SELECT content FROM script WHERE path = $1 AND workspace_id = $2 + AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND + workspace_id = $2) + "#, &rpath, - db, - already_visited, - annotated_pyv_numeric, - path_where_annotated_pyv, + w_id ) + .fetch_optional(db) .await? + .unwrap_or_else(|| "".to_string()); + + if already_visited.contains(&rpath) { + vec![] + } else { + already_visited.push(rpath.clone()); + // Because the algo goes depth first, this function will never return relative import + // This why we can safely assume later, that there is no relative imports + parse_python_imports_inner( + &code, + w_id, + &rpath, + db, + already_visited, + annotated_pyv_numeric, + path_where_annotated_pyv, + ) + .await? + .into_values() + .collect_vec() + } } - } else { - vec![n.to_string()] + NImport::Repin { pin, key } => vec![NImportResolved::Repin { pin, key }], + NImport::Pin { pins, key } => vec![NImportResolved::Pin { pins, key }], + NImport::Auto { pkg, key } => vec![NImportResolved::Auto { pkg, key }], }; + + // Nested should also be sorted for the same reason + nested.sort(); + + // At this point there should be no NImport::Relative in `nested` for imp in nested { - if !imports.contains(&imp) { - imports.push(imp); + let key = match imp.clone() { + NImportResolved::Pin { key, .. } => key, + NImportResolved::Repin { key, .. } => key, + NImportResolved::Auto { key, pkg } => key.unwrap_or(pkg), + }; + // Handled cases: + // + // 1. + // Error: Imported windmill scripts have different pins + // + // auto + // ├── pin:2 + // └── pin:1 + // + // Fix 1: + // + // auto + // ├── pin:1 + // └── pin:1 + // + // Fix 2: + // + // repin:1 + // ├── pin:2 + // └── pin:1 + // + // 2. + // Error: Imported windmill scripts have different pins + // + // pin:2 + // └── pin:1 + // + // Fix 1: + // + // auto + // └── pin:1 + // + // Fix 2: + // + // repin:2 + // └── pin:1 + // + // 3. repins allowed to be repinned again + // + // repin:2 + // └── repin:1 + // + match imp.clone() { + NImportResolved::Repin { .. } => { + if let Some(existing_import) = imports.get(&key) { + match existing_import { + // replace + p if matches!( + p, + NImportResolved::Pin { .. } | NImportResolved::Auto { .. } + ) => + { + imports.insert(key, imp); + } + // do nothing (older repins have greater precedence) + NImportResolved::Repin { .. } => {} + // Should not be possible + _ => { + return Err(anyhow::anyhow!( + "Internal error: cannot resolve requirement pins", + ) + .into()); + } + } + } else { + imports.insert(key, imp.clone()); + } + } + NImportResolved::Pin { pins: new_pins, .. } => { + if let Some(existing_import) = imports.get_mut(&key) { + match existing_import { + // Check if pin is the same version, if same, do nothing, if not error + NImportResolved::Pin { pins: existing_pins, .. } => { + existing_pins.extend(new_pins) + } + // do nothing + NImportResolved::Repin { .. } => {} + // Replace with new pin + NImportResolved::Auto { .. } => { + imports.insert(key, imp); + } + } + } else { + imports.insert(key, imp.clone()); + } + } + NImportResolved::Auto { .. } => { + if !imports.contains_key(&key) { + imports.insert(key, imp); + } + } } } } - imports.sort(); Ok(imports) } } diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index a634f247dd..d734cb8ead 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -19,7 +19,7 @@ def main(): "; let mut already_visited = vec![]; - let r = parse_python_imports( + let (r, ..) = parse_python_imports( code, "test-workspace", "f/foo/bar", @@ -52,7 +52,7 @@ def main(): "; let mut already_visited = vec![]; - let r = parse_python_imports( + let (r, ..) = parse_python_imports( code, "test-workspace", "f/foo/bar", @@ -83,7 +83,7 @@ def main(): "; let mut already_visited = vec![]; - let r = parse_python_imports( + let (r, ..) = parse_python_imports( code, "test-workspace", "f/foo/bar", diff --git a/backend/tests/fixtures/lockfile_python.sql b/backend/tests/fixtures/lockfile_python.sql new file mode 100644 index 0000000000..27f7b103f0 --- /dev/null +++ b/backend/tests/fixtures/lockfile_python.sql @@ -0,0 +1,51 @@ +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +# requirements: +# microdot==2.2.0 + +import pandas +import requests +import tiny # pin: tiny==0.1.2 + +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/system/requirements', 12346, 'python3', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +# extra_requirements: +# bottle==0.13.2 + +import tiny + +def main(): + pass +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/system/extra_requirements', 12347, 'python3', ''); + + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +' +import tiny # pin: bottle==0.13.2 +import simplejson # pin: simplejson==3.19.3 + +def main(): + return [test1(), test2(), test3(), test4()] +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/system/pins', 12348, 'python3', ''); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index c93256a450..182ed60f2e 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3831,6 +3831,209 @@ def main(): run_preview_relative_imports(&db, content, ScriptLang::Python3).await; } +async fn assert_lockfile( + db: &Pool, + script_content: String, + language: ScriptLang, + expected_lines: Vec<&str>, +) { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await; + let port = server.addr.port(); + let client = windmill_api_client::create_client( + &format!("http://localhost:{port}"), + "SECRET_TOKEN".to_string(), + ); + + client + .create_script( + "test-workspace", + &NewScript { + language: NewScriptLanguage::from_str(language.as_str()).unwrap(), + content: script_content, + path: "f/system/test_import".to_string(), + concurrent_limit: None, + concurrency_time_window_s: None, + cache_ttl: None, + dedicated_worker: None, + description: "".to_string(), + draft_only: None, + envs: vec![], + is_template: None, + kind: None, + parent_hash: None, + lock: None, + summary: "".to_string(), + tag: None, + schema: std::collections::HashMap::new(), + ws_error_handler_muted: Some(false), + priority: None, + delete_after_use: None, + timeout: None, + restart_unless_cancelled: None, + deployment_message: None, + concurrency_key: None, + visible_to_runner_only: None, + no_main_func: None, + codebase: None, + has_preprocessor: None, + on_behalf_of_email: None, + }, + ) + .await + .unwrap(); + + let mut completed = listen_for_completed_jobs(&db).await; + let db2 = db.clone(); + in_test_worker( + &db, + async move { + completed.next().await; // deployed script + + let script = sqlx::query!( + "SELECT hash FROM script WHERE path = $1", + "f/system/test_import".to_string() + ) + .fetch_one(&db2) + .await + .unwrap(); + + let job = RunJob::from(JobPayload::Dependencies { + path: "f/system/test_import".to_string(), + hash: ScriptHash(script.hash), + dedicated_worker: None, + language, + }) + .push(&db2) + .await; + + completed.next().await; // completed job + + let result = completed_job(job, &db2).await.json_result().unwrap(); + + assert_eq!( + result, + json!({ + "lock": expected_lines.join("\n"), + "status": "Successful lock file generation" + }) + ); + }, + port, + ) + .await; +} +#[sqlx::test(fixtures("base", "lockfile_python"))] +async fn test_requirements_python(db: Pool) { + let content = r#" +# py311 +# requirements: +# tiny==0.1.3 + +import bar +import baz # pin: foo +import baz # repin: fee +import bug # repin: free + +def main(): + pass +"# + .to_string(); + + assert_lockfile( + &db, + content, + ScriptLang::Python3, + vec!["# py311", "tiny==0.1.3"], + ) + .await; +} +#[sqlx::test(fixtures("base", "lockfile_python"))] +async fn test_extra_requirements_python(db: Pool) { + { + let content = r#" +# py311 +# extra_requirements: +# tiny + +import f.system.extra_requirements +import tiny # pin: tiny==0.1.0 +import tiny # pin: tiny==0.1.1 +import tiny # repin: tiny==0.1.2 + +def main(): + pass + "# + .to_string(); + + assert_lockfile( + &db, + content, + ScriptLang::Python3, + vec!["# py311", "bottle==0.13.2", "tiny==0.1.2"], + ) + .await; + } +} +#[sqlx::test(fixtures("base", "lockfile_python"))] +async fn test_extra_requirements_python2(db: Pool) { + + let content = r#" +# py311 +# extra_requirements: +# tiny==0.1.3 + +import simplejson # pin: simplejson==3.20.1 +def main(): + pass +"# + .to_string(); + + assert_lockfile( + &db, + content, + ScriptLang::Python3, + vec![ + "# py311", + "simplejson==3.20.1", + "tiny==0.1.3" + ], + ) + .await; + +} + +#[sqlx::test(fixtures("base", "lockfile_python"))] +async fn test_pins_python(db: Pool) { + let content = r#" +# py311 +# extra_requirements: +# tiny==0.1.3 + +import f.system.requirements +import f.system.pins +import tiny # repin: bottle==0.13.0 +import simplejson + +def main(): + pass +"# + .to_string(); + + assert_lockfile( + &db, + content, + ScriptLang::Python3, + vec![ + "# py311", + "bottle==0.13.0", + "microdot==2.2.0", + "simplejson==3.19.3", + "tiny==0.1.3" + ], + ) + .await; +} #[sqlx::test(fixtures("base", "result_format"))] async fn test_result_format(db: Pool) { let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41"; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index f07b9abfe2..c642abae49 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -1439,6 +1439,7 @@ async fn handle_python_deps( .clone(); let mut requirements; + let compilation_error_hint; let mut annotated_pyv = None; let mut annotated_pyv_numeric = None; let is_deployed = requirements_o.is_some(); @@ -1449,23 +1450,26 @@ async fn handle_python_deps( None => { let mut already_visited = vec![]; - requirements = match conn { - Connection::Sql(db) => windmill_parser_py_imports::parse_python_imports( - inner_content, - w_id, - script_path, - db, - &mut already_visited, - &mut annotated_pyv_numeric, - ) - .await? - .join("\n"), + (requirements, compilation_error_hint) = match conn { + Connection::Sql(db) => { + let (r, h) = windmill_parser_py_imports::parse_python_imports( + inner_content, + w_id, + script_path, + db, + &mut already_visited, + &mut annotated_pyv_numeric, + ) + .await?; + + (r.join("\n"), h) + } Connection::Http(_) => match precomputed_agent_info { Some(PrecomputedAgentInfo::Python { py_version, requirements }) => { annotated_pyv_numeric = py_version; - requirements.clone().unwrap_or_else(|| "".to_string()) + (requirements.clone().unwrap_or_else(|| "".to_string()), None) } - _ => "".to_string(), + _ => ("".to_string(), None), }, }; @@ -1487,7 +1491,11 @@ async fn handle_python_deps( ) .await .map_err(|e| { - Error::ExecutionErr(format!("pip compile failed: {}", e.to_string())) + Error::ExecutionErr(format!( + "pip compile failed: {}{}", + e.to_string(), + compilation_error_hint.unwrap_or_default() + )) })?; } &requirements diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 8ba9ee501a..6c56f1acef 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1853,6 +1853,7 @@ async fn capture_dependency_job( &mut annotated_pyv_numeric, ) .await? + .0 .join("\n") }; From 591bb4b6a878fdf6598d6525e69a5de65a5f3863 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Apr 2025 22:35:52 +0000 Subject: [PATCH 118/133] update ee repo ref --- backend/ee-repo-ref.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9c55ceeb81..6081d9fd35 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -44c3e23922097d4386183e56b6b3dc540c70d71b \ No newline at end of file +47869a5803b421a754173250fed79b63fe001ebd \ No newline at end of file From 5010850cdca1d7536c58d80957201d630203aa54 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 11 Apr 2025 23:05:10 +0000 Subject: [PATCH 119/133] fix inline script update effect --- .../InlineScriptEditor.svelte | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte index 8c29c5bdc8..cdf071618e 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptEditor.svelte @@ -63,11 +63,16 @@ return schema } - $: inlineScript && - (inlineScript.path = `${defaultIfEmptyString( - $appPath, - `u/${$userStore?.username ?? 'unknown'}/newapp` - )}/${name?.replaceAll(' ', '_')}`) + $: name && onNameChange() + + function onNameChange() { + if (inlineScript) { + inlineScript.path = `${defaultIfEmptyString( + $appPath, + `u/${$userStore?.username ?? 'unknown'}/newapp` + )}/${name?.replaceAll(' ', '_')}` + } + } onMount(async () => { if (inlineScript && !inlineScript.schema) { @@ -85,6 +90,9 @@ if (inlineScript?.language == 'frontend' && inlineScript.content) { inferSuggestions(inlineScript.content) } + if (!inlineScript?.path) { + onNameChange() + } }) const dispatch = createEventDispatcher() From 12ba15c92863faae50b95f83dc396821cba3bc09 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Apr 2025 11:09:21 +0000 Subject: [PATCH 120/133] improve benchmarks --- backend/pg_log_tail | 0 backend/plot2.py | 33 +++++++++++++++++++ backend/src/monitor.rs | 2 +- .../src/bench.rs | 12 +++---- backend/windmill-common/src/lib.rs | 3 ++ backend/windmill-queue/src/jobs.rs | 14 ++++++++ backend/windmill-worker/src/lib.rs | 2 -- .../windmill-worker/src/result_processor.rs | 4 ++- backend/windmill-worker/src/worker.rs | 18 +++++++--- backend/windmill-worker/src/worker_flow.rs | 4 +-- 10 files changed, 76 insertions(+), 16 deletions(-) delete mode 100644 backend/pg_log_tail create mode 100644 backend/plot2.py rename backend/{windmill-worker => windmill-common}/src/bench.rs (98%) diff --git a/backend/pg_log_tail b/backend/pg_log_tail deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/plot2.py b/backend/plot2.py new file mode 100644 index 0000000000..5a6624bc14 --- /dev/null +++ b/backend/plot2.py @@ -0,0 +1,33 @@ +import json +import matplotlib.pyplot as plt + +# Path to the profiling JSON file +# file_path = "/tmp/windmill/profiling_main.json" +file_path = "/tmp/profiling.json" + +# Load the JSON data +with open(file_path, "r") as f: + data = json.load(f) + +# Extract timings for "pre pull->post pull" +pre_post_pull_timings = [ + timing / 1000000.0 for entry in data["timings"] + for step, timing in entry["timings"] + # if step == "pre pull->post pull" + if step == "->job pulled from DB" +] + +# Plotting the distribution +plt.figure(figsize=(10, 6)) +# plt.hist(pre_post_pull_timings, bins=10, edgecolor='black') +plt.scatter(range(len(pre_post_pull_timings)), pre_post_pull_timings, + alpha=1.0, # Transparency level + s=40) # Size of the dots`) +plt.title("Distribution of 'pre pull->post pull' timings") +# plt.xlabel("Time (ms)") +# plt.ylabel("Frequency") +plt.xlabel("Sample Index") +plt.ylabel("Time (ms)") +plt.grid(True) +plt.tight_layout() +plt.show() \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 6b609dc9a1..8dae3f8c5e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1914,7 +1914,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker worker_name, send_result_never_used, #[cfg(feature = "benchmark")] - &mut windmill_worker::bench::BenchmarkIter::new(), + &mut windmill_common::bench::BenchmarkIter::new(), ) .await; } diff --git a/backend/windmill-worker/src/bench.rs b/backend/windmill-common/src/bench.rs similarity index 98% rename from backend/windmill-worker/src/bench.rs rename to backend/windmill-common/src/bench.rs index 5006d5a34c..c851198b2a 100644 --- a/backend/windmill-worker/src/bench.rs +++ b/backend/windmill-common/src/bench.rs @@ -1,9 +1,9 @@ -use serde::Serialize; -use tokio::time::Instant; -use windmill_common::{ +use crate::{ worker::{write_file, TMP_DIR}, DB, }; +use serde::Serialize; +use tokio::time::Instant; #[derive(Serialize)] pub struct BenchmarkInfo { @@ -41,8 +41,8 @@ impl BenchmarkInfo { self.total_duration = Some(total_duration as u64); println!( - "Writing benchmark {path}, duration of benchmark: {total_duration}s and RPS: {}", - self.iters as f64 / total_duration as f64 + "Writing benchmark {path}, duration of benchmark: {total_duration}ms and RPS: {}", + self.iters as f64 / total_duration as f64 * 1000.0 ); write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling"); Ok(()) @@ -79,7 +79,7 @@ impl BenchmarkIter { } pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) { - use windmill_common::{jobs::JobKind, scripts::ScriptLang}; + use crate::{jobs::JobKind, scripts::ScriptLang}; let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string()); diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index fd43db2f7c..0d2c746d3e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -25,6 +25,8 @@ use sqlx::{Pool, Postgres}; pub mod agent_workers; pub mod apps; pub mod auth; +#[cfg(feature = "benchmark")] +pub mod bench; pub mod cache; pub mod db; pub mod ee; @@ -38,6 +40,7 @@ pub mod indexer; pub mod job_metrics; #[cfg(feature = "parquet")] pub mod job_s3_helpers_ee; + pub mod jobs; pub mod jwt; pub mod more_serde; diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index cf0fb8ce8b..4fb7210048 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -29,7 +29,11 @@ use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; +#[cfg(feature = "benchmark")] +use windmill_common::add_time; use windmill_common::auth::JobPerms; +#[cfg(feature = "benchmark")] +use windmill_common::bench::BenchmarkIter; use windmill_common::utils::now_from_db; use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; use windmill_common::{ @@ -2310,6 +2314,7 @@ pub async fn pull( suspend_first: bool, worker_name: &str, query_o: Option<(String, String)>, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { loop { if let Some((query_suspended, query_no_suspend)) = query_o.as_ref() { @@ -2351,6 +2356,7 @@ pub async fn pull( db, suspend_first, worker_name, + #[cfg(feature = "benchmark")] bench, ) .await?; @@ -2549,6 +2555,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( db: &Pool, suspend_first: bool, worker_name: &str, + #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result<(Option, bool)> { let job_and_suspended: (Option, bool) = { /* Jobs can be started if they: @@ -2589,11 +2596,18 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>( for query in queries.iter() { // tracing::info!("Pulling job with query: {}", query); // let instant = std::time::Instant::now(); + + #[cfg(feature = "benchmark")] + add_time!(bench, "pre pull"); + let r = sqlx::query_as::<_, PulledJob>(query) .bind(worker_name) .fetch_optional(db) .await?; + #[cfg(feature = "benchmark")] + add_time!(bench, "post pull"); + if let Some(pulled_job) = r { // tracing::info!("pulled job: {:?}", instant.elapsed().as_micros()); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index a9380c52e9..2b0a5d6f4a 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -13,8 +13,6 @@ mod bash_executor; #[cfg(feature = "java")] mod java_executor; -#[cfg(feature = "benchmark")] -pub mod bench; mod bun_executor; pub mod common; mod config; diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 389bd067a8..e1b2a763f6 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -26,7 +26,7 @@ use windmill_common::{ }; #[cfg(feature = "benchmark")] -use crate::bench::{BenchmarkInfo, BenchmarkIter}; +use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError, @@ -507,6 +507,8 @@ pub async fn process_completed_job( })?; } + add_time!(bench, "pre add_completed_job"); + add_completed_job( db, &job, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 2081b90f5f..d6b96fd1c8 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -158,7 +158,7 @@ use crate::mssql_executor::do_mssql; use crate::bigquery_executor::do_bigquery; #[cfg(feature = "benchmark")] -use crate::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; +use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; @@ -1002,8 +1002,10 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] { - if let Some(db) = conn.as_sql() { - benchmark_init(benchmark_jobs, db).await; + if i_worker == 1 { + if let Some(db) = conn.as_sql() { + benchmark_init(benchmark_jobs, db).await; + } } } @@ -1322,7 +1324,15 @@ pub async fn run_worker( last_suspend_first = Instant::now(); } - let job = pull(&db, suspend_first, &worker_name, None).await; + let job = pull( + &db, + suspend_first, + &worker_name, + None, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await; add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 27aef9d1f9..63cf414c53 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,8 +11,6 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -#[cfg(feature = "benchmark")] -use crate::bench::BenchmarkIter; use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::{ @@ -30,6 +28,8 @@ use tracing::instrument; use uuid::Uuid; use windmill_common::add_time; use windmill_common::auth::JobPerms; +#[cfg(feature = "benchmark")] +use windmill_common::bench::BenchmarkIter; use windmill_common::cache::{self, RawData}; use windmill_common::db::Authed; use windmill_common::flow_status::{ From 8c6e620f5cce53dac1d99aa5ecb7b562db4b226e Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Apr 2025 12:54:05 +0000 Subject: [PATCH 121/133] nit benchmarks --- backend/windmill-worker/src/worker.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d6b96fd1c8..ef13510a30 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1002,10 +1002,8 @@ pub async fn run_worker( #[cfg(feature = "benchmark")] { - if i_worker == 1 { - if let Some(db) = conn.as_sql() { - benchmark_init(benchmark_jobs, db).await; - } + if let Some(db) = conn.as_sql() { + benchmark_init(benchmark_jobs, db).await; } } From fd47cd60163a736d7b12108a208fb6fd044b4130 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Apr 2025 14:47:48 +0000 Subject: [PATCH 122/133] improve perf of setting api roles --- ...50412144540_improve_perf_api_role.down.sql | 1 + ...0250412144540_improve_perf_api_role.up.sql | 22 ++++ backend/windmill-common/src/db.rs | 105 +++++++++++------- backend/windmill-common/src/tracing_init.rs | 7 +- 4 files changed, 92 insertions(+), 43 deletions(-) create mode 100644 backend/migrations/20250412144540_improve_perf_api_role.down.sql create mode 100644 backend/migrations/20250412144540_improve_perf_api_role.up.sql diff --git a/backend/migrations/20250412144540_improve_perf_api_role.down.sql b/backend/migrations/20250412144540_improve_perf_api_role.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250412144540_improve_perf_api_role.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250412144540_improve_perf_api_role.up.sql b/backend/migrations/20250412144540_improve_perf_api_role.up.sql new file mode 100644 index 0000000000..8a521849fb --- /dev/null +++ b/backend/migrations/20250412144540_improve_perf_api_role.up.sql @@ -0,0 +1,22 @@ +-- Add up migration script here + CREATE OR REPLACE FUNCTION set_session_context( + admin BOOLEAN, + username TEXT, + groups TEXT, + pgroups TEXT, + folders_read TEXT, + folders_write TEXT +) RETURNS void AS $$ +BEGIN + IF admin THEN + SET LOCAL ROLE windmill_admin; + ELSE + SET LOCAL ROLE windmill_user; + END IF; + PERFORM set_config('session.user', username, true); + PERFORM set_config('session.groups', groups, true); + PERFORM set_config('session.pgroups', pgroups, true); + PERFORM set_config('session.folders_read', folders_read, true); + PERFORM set_config('session.folders_write', folders_write, true); +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/windmill-common/src/db.rs b/backend/windmill-common/src/db.rs index 96be430e73..47c698b05c 100644 --- a/backend/windmill-common/src/db.rs +++ b/backend/windmill-common/src/db.rs @@ -72,12 +72,6 @@ impl UserDB { where T: Authable, { - let user = if authed.is_admin() { - "windmill_admin" - } else { - "windmill_user" - }; - let (folders_write, folders_read): &(Vec<_>, Vec<_>) = &authed.folders().into_iter().partition(|x| x.1); @@ -95,10 +89,6 @@ impl UserDB { let mut tx = self.db.begin().await?; - sqlx::query(&format!("SET LOCAL ROLE {}", user)) - .execute(&mut *tx) - .await?; - if let Some(schema) = PG_SCHEMA.as_ref() { sqlx::query(&format!("SET LOCAL search_path TO {}", schema)) .execute(&mut *tx) @@ -106,53 +96,86 @@ impl UserDB { } sqlx::query!( - "SELECT set_config('session.user', $1, true)", - authed.username() - ) - .fetch_optional(&mut *tx) - .await?; - - sqlx::query!( - "SELECT set_config('session.groups', $1, true)", - &authed.groups().join(",") - ) - .fetch_optional(&mut *tx) - .await?; - - sqlx::query!( - "SELECT set_config('session.pgroups', $1, true)", - &authed + "SELECT set_session_context($1, $2, $3, $4, $5, $6)", + authed.is_admin(), + authed.username(), + authed.groups().join(","), + authed .groups() .iter() .map(|x| format!("g/{}", x)) .collect::>() - .join(",") - ) - .fetch_optional(&mut *tx) - .await?; - - sqlx::query!( - "SELECT set_config('session.folders_read', $1, true)", + .join(","), folders_read .iter() .map(|x| x.0.clone()) .collect::>() - .join(",") - ) - .fetch_optional(&mut *tx) - .await?; - - sqlx::query!( - "SELECT set_config('session.folders_write', $1, true)", + .join(","), folders_write .iter() .map(|x| x.0.clone()) .collect::>() .join(",") ) - .fetch_optional(&mut *tx) + .execute(&mut *tx) .await?; + // set_session_context( + // username TEXT, + // groups TEXT, + // pgroups TEXT, + // folders_read TEXT, + // folders_write TEXT + // ) + + // sqlx::query!( + // "SELECT set_config('session.user', $1, true)", + // authed.username() + // ) + // .fetch_optional(&mut *tx) + // .await?; + + // sqlx::query!( + // "SELECT set_config('session.groups', $1, true)", + // &authed.groups().join(",") + // ) + // .fetch_optional(&mut *tx) + // .await?; + + // sqlx::query!( + // "SELECT set_config('session.pgroups', $1, true)", + // &authed + // .groups() + // .iter() + // .map(|x| format!("g/{}", x)) + // .collect::>() + // .join(",") + // ) + // .fetch_optional(&mut *tx) + // .await?; + + // sqlx::query!( + // "SELECT set_config('session.folders_read', $1, true)", + // folders_read + // .iter() + // .map(|x| x.0.clone()) + // .collect::>() + // .join(",") + // ) + // .fetch_optional(&mut *tx) + // .await?; + + // sqlx::query!( + // "SELECT set_config('session.folders_write', $1, true)", + // folders_write + // .iter() + // .map(|x| x.0.clone()) + // .collect::>() + // .join(",") + // ) + // .fetch_optional(&mut *tx) + // .await?; + Ok(tx) } } diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 5e6ef6940e..5ed5cec752 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -60,8 +60,11 @@ pub fn initialize_tracing( "RUST_LOG", &format!("windmill={}", rust_log_env.as_ref().unwrap()), ) - } - let default_env_filter = if rust_log_env.is_ok_and(|x| x == "debug") { + } else if rust_log_env.as_ref().is_ok_and(|x| x == "sqlxdebug") { + std::env::set_var("RUST_LOG", "windmill=debug,sqlx=debug"); + }; + + let default_env_filter = if rust_log_env.is_ok_and(|x| x == "debug" || x == "sqlxdebug") { LevelFilter::DEBUG } else { LevelFilter::INFO From 496960d349885a9dc33165ebf09e593e9773e5e8 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Apr 2025 15:02:43 +0000 Subject: [PATCH 123/133] improve last_job_suspended_history --- backend/windmill-worker/src/worker.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ef13510a30..b92a3bc560 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1127,7 +1127,7 @@ pub async fn run_worker( }; let mut suspend_first_success = false; let mut last_reading = Instant::now() - Duration::from_secs(NUM_SECS_READINGS + 1); - let mut last_30jobs_suspended: Vec = vec![false; 30]; + let mut last_30jobs_suspended = 0; let mut last_suspend_first = Instant::now(); let mut killed_but_draining_same_worker_jobs = false; @@ -1311,9 +1311,7 @@ pub async fn run_worker( match &conn { Connection::Sql(db) => { let pull_time = Instant::now(); - let likelihood_of_suspend = (1.0 - + last_30jobs_suspended.iter().filter(|&&x| x).count() as f64) - / 31.0; + let likelihood_of_suspend = last_30jobs_suspended as f64 / 30.0; let suspend_first = suspend_first_success || rand::random::() < likelihood_of_suspend || last_suspend_first.elapsed().as_secs_f64() > 5.0; @@ -1364,10 +1362,11 @@ pub async fn run_worker( if let Ok(j) = job.as_ref() { let suspend_success = j.suspended; if suspend_first { - last_30jobs_suspended.push(suspend_success); - if last_30jobs_suspended.len() > 30 { - last_30jobs_suspended.remove(0); + if last_30jobs_suspended < 30 { + last_30jobs_suspended += 1; } + } else { + last_30jobs_suspended -= 1; } suspend_first_success = suspend_first && suspend_success; #[cfg(feature = "prometheus")] From 7a4f64b92d9bdd2ff51f437702a897cbfff71f12 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 12 Apr 2025 15:26:56 +0000 Subject: [PATCH 124/133] improve perf of push --- ...24582fdb302883aebd2da187ac0084e767ea3.json | 22 -- ...34546cddaf1029618ed14015fd7b0a7017441.json | 27 +++ ...7b9cc3c82d815d99b3d435adcfbb5a9246124.json | 22 -- ...06caccb849db9e0f71d86da655b01c6a3e8d0.json | 27 --- ...0f0b76dc38476670f9fc0667b057d2766d42e.json | 22 -- ...2c25f8768b220e53bf469550a3a3697ab756a.json | 22 -- ...0a72f5d341b053218e7aec83a834cf7ccc98f.json | 21 -- ...5711a40924ec551a7589a64ee96f8aa7f6a21.json | 22 -- ...5d76ae2223d59e9f321a8b6d0c27adc09f741.json | 14 -- ...98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json} | 16 +- backend/ee-repo-ref.txt | 2 +- backend/windmill-queue/src/jobs.rs | 212 ++++++++++-------- 12 files changed, 158 insertions(+), 271 deletions(-) delete mode 100644 backend/.sqlx/query-122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3.json create mode 100644 backend/.sqlx/query-18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441.json delete mode 100644 backend/.sqlx/query-29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124.json delete mode 100644 backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json delete mode 100644 backend/.sqlx/query-6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e.json delete mode 100644 backend/.sqlx/query-6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a.json delete mode 100644 backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json delete mode 100644 backend/.sqlx/query-9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21.json delete mode 100644 backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json rename backend/.sqlx/{query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json => query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json} (51%) diff --git a/backend/.sqlx/query-122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3.json b/backend/.sqlx/query-122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3.json deleted file mode 100644 index a329998c95..0000000000 --- a/backend/.sqlx/query-122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT set_config('session.pgroups', $1, true)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "set_config", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "122090a0f89e5248a0a0f199ebd24582fdb302883aebd2da187ac0084e767ea3" -} diff --git a/backend/.sqlx/query-18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441.json b/backend/.sqlx/query-18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441.json new file mode 100644 index 0000000000..c1258697c9 --- /dev/null +++ b/backend/.sqlx/query-18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT set_session_context($1, $2, $3, $4, $5, $6)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "set_session_context", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Bool", + "Text", + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "18ddbb9eb4c0ec03e6170f8c70634546cddaf1029618ed14015fd7b0a7017441" +} diff --git a/backend/.sqlx/query-29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124.json b/backend/.sqlx/query-29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124.json deleted file mode 100644 index 72f3f1f469..0000000000 --- a/backend/.sqlx/query-29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT set_config('session.folders_read', $1, true)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "set_config", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "29fbc3a8c35845a997cd548ed417b9cc3c82d815d99b3d435adcfbb5a9246124" -} diff --git a/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json b/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json deleted file mode 100644 index b96c05d674..0000000000 --- a/backend/.sqlx/query-31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) RETURNING id AS \"id!\"", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id!", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Uuid", - "Bool", - "Timestamptz", - "Varchar", - "Int2" - ] - }, - "nullable": [ - false - ] - }, - "hash": "31df83e9eb6078e93ec5fe4168306caccb849db9e0f71d86da655b01c6a3e8d0" -} diff --git a/backend/.sqlx/query-6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e.json b/backend/.sqlx/query-6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e.json deleted file mode 100644 index 2313dd087c..0000000000 --- a/backend/.sqlx/query-6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT set_config('session.groups', $1, true)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "set_config", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6199e8be5cb13db71108e555ea20f0b76dc38476670f9fc0667b057d2766d42e" -} diff --git a/backend/.sqlx/query-6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a.json b/backend/.sqlx/query-6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a.json deleted file mode 100644 index fb5e174ced..0000000000 --- a/backend/.sqlx/query-6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT set_config('session.user', $1, true)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "set_config", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6c90d5ea2a09b47b81fdab465062c25f8768b220e53bf469550a3a3697ab756a" -} diff --git a/backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json b/backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json deleted file mode 100644 index 07c93d0d88..0000000000 --- a/backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Varchar", - "Varchar", - "Bool", - "Bool", - "JsonbArray", - "TextArray", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f" -} diff --git a/backend/.sqlx/query-9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21.json b/backend/.sqlx/query-9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21.json deleted file mode 100644 index abad579224..0000000000 --- a/backend/.sqlx/query-9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT set_config('session.folders_write', $1, true)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "set_config", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "9897705b7c265333559df757a725711a40924ec551a7589a64ee96f8aa7f6a21" -} diff --git a/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json b/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json deleted file mode 100644 index b7a302213a..0000000000 --- a/backend/.sqlx/query-ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "ca8bf3dda133556ee7051d5436a5d76ae2223d59e9f321a8b6d0c27adc09f741" -} diff --git a/backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json similarity index 51% rename from backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json rename to backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json index 3ea932c5ca..412b64c480 100644 --- a/backend/.sqlx/query-29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e.json +++ b/backend/.sqlx/query-cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $21, $22, $23, $24, $25, $26,\n CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)", + "query": "WITH inserted_job AS (\n INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job,\n created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger,\n script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner,\n flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id,\n cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,\n $19, $20, $21, $22, $23, $24, $25, $26,\n CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END,\n ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)\n ),\n inserted_runtime AS (\n INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)\n ),\n inserted_job_perms AS (\n INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $32, $33, $34, $35, $36, $37, $2) \n ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2\n )\n INSERT INTO v2_job_queue\n (workspace_id, id, running, scheduled_for, started_at, tag, priority)\n VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 THEN now() END, $30, $31)", "describe": { "columns": [], "parameters": { @@ -86,10 +86,20 @@ "Varchar", "Int4", "Int2", - "Bool" + "Bool", + "Bool", + "Timestamptz", + "Varchar", + "Int2", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "TextArray" ] }, "nullable": [] }, - "hash": "29624682d687790dd199c4af759132d79fdb2982de111cb5fd43e3d9ecd0f15e" + "hash": "cccdcb7fe7968eadfc04d8957a8e98b2f2d92a6d7f687a9dd5a70edb3d5a63e6" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 6081d9fd35..91f337034b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -47869a5803b421a754173250fed79b63fe001ebd \ No newline at end of file +1475a504133766e005ec551004c2af2283663352 \ No newline at end of file diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 4fb7210048..9c7dd8a4fc 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4190,93 +4190,6 @@ pub async fn push<'c, 'd>( _ => None, }); - sqlx::query!( - "INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job, - created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger, - script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner, - flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, - cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, - $19, $20, $21, $22, $23, $24, $25, $26, - CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END, - ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27)", - job_id, - workspace_id, - raw_code, - raw_lock, - raw_flow as Option>, - tag, - parent_job, - user, - permissioned_as, - script_hash, - script_path.clone(), - Json(args) as Json, - job_kind.clone() as JobKind, - schedule_path, - language as Option, - same_worker, - pre_run_error.map(|e| e.to_string()), - email, - visible_to_owner, - root_job, - concurrent_limit, - if concurrent_limit.is_some() { - concurrency_time_window_s - } else { - None - }, - custom_timeout, - flow_step_id, - cache_ttl, - final_priority, - preprocessed, - ) - .execute(&mut *tx) - .warn_after_seconds(1) - .await?; - - tracing::debug!("Pushing job {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}"); - let uuid = sqlx::query_scalar!( - "INSERT INTO v2_job_queue - (workspace_id, id, running, scheduled_for, started_at, tag, priority) - VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) \ - RETURNING id AS \"id!\"", - workspace_id, - job_id, - is_running, - scheduled_for_o, - tag, - final_priority, - ) - .fetch_one(&mut *tx) - .warn_after_seconds(1) - .await - .map_err(|e| Error::internal_err(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?; - - sqlx::query!( - "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)", - job_id - ) - .execute(&mut *tx) - .await?; - if let Some(flow_status) = flow_status { - sqlx::query!( - "INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2)", - job_id, - Json(flow_status) as Json, - ) - .execute(&mut *tx) - .await?; - } - - tracing::debug!("Pushed {job_id}"); - // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction. - #[cfg(feature = "prometheus")] - if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { - QUEUE_PUSH_COUNT.inc(); - } - let job_authed = match authed { Some(authed) if authed.email == email @@ -4309,21 +4222,130 @@ pub async fn push<'c, 'd>( .filter_map(|x| serde_json::to_value(x).ok()) .collect::>(); - if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - values ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", + // if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + // values ($1, $2, $3, $4, $5, $6, $7, $8) + // ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8", + // job_id, + // job_authed.email, + // job_authed.username, + // job_authed.is_admin, + // job_authed.is_operator, + // folders.as_slice(), + // job_authed.groups.as_slice(), + // workspace_id, + // ).execute(&mut *tx).await { + // tracing::error!("Could not insert job_perms for job {job_id}: {err:#}"); + // } + + + sqlx::query!( + "WITH inserted_job AS ( + INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job, + created_by, permissioned_as, runnable_id, runnable_path, args, kind, trigger, + script_lang, same_worker, pre_run_error, permissioned_as_email, visible_to_owner, + flow_innermost_root_job, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, + cache_ttl, priority, trigger_kind, script_entrypoint_override, preprocessed) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, + $19, $20, $21, $22, $23, $24, $25, $26, + CASE WHEN $14::VARCHAR IS NOT NULL THEN 'schedule'::job_trigger_kind END, + ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27) + ), + inserted_runtime AS ( + INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null) + ), + inserted_job_perms AS ( + INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + values ($1, $32, $33, $34, $35, $36, $37, $2) + ON CONFLICT (job_id) DO UPDATE SET email = $32, username = $33, is_admin = $34, is_operator = $35, folders = $36, groups = $37, workspace_id = $2 + ) + INSERT INTO v2_job_queue + (workspace_id, id, running, scheduled_for, started_at, tag, priority) + VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 THEN now() END, $30, $31)", job_id, + workspace_id, + raw_code, + raw_lock, + raw_flow as Option>, + tag, + parent_job, + user, + permissioned_as, + script_hash, + script_path.clone(), + Json(args) as Json, + job_kind.clone() as JobKind, + schedule_path, + language as Option, + same_worker, + pre_run_error.map(|e| e.to_string()), + email, + visible_to_owner, + root_job, + concurrent_limit, + if concurrent_limit.is_some() { + concurrency_time_window_s + } else { + None + }, + custom_timeout, + flow_step_id, + cache_ttl, + final_priority, + preprocessed, + is_running, + scheduled_for_o, + tag, + final_priority, job_authed.email, job_authed.username, job_authed.is_admin, job_authed.is_operator, folders.as_slice(), job_authed.groups.as_slice(), - workspace_id, - ).execute(&mut *tx).await { - tracing::error!("Could not insert job_perms for job {job_id}: {err:#}"); + ) + .execute(&mut *tx) + .warn_after_seconds(1) + .await?; + +// tracing::debug!("Pushing job {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}"); +// let uuid = sqlx::query_scalar!( +// "INSERT INTO v2_job_queue +// (workspace_id, id, running, scheduled_for, started_at, tag, priority) +// VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) \ +// RETURNING id AS \"id!\"", +// workspace_id, +// job_id, +// , +// ) +// .fetch_one(&mut *tx) +// .warn_after_seconds(1) +// .await +// .map_err(|e| Error::internal_err(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?; + + // sqlx::query!( + // "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)", + // job_id + // ) + // .execute(&mut *tx) + // .await?; + if let Some(flow_status) = flow_status { + sqlx::query!( + "INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2)", + job_id, + Json(flow_status) as Json, + ) + .execute(&mut *tx) + .await?; } - + + tracing::debug!("Pushed {job_id}"); + // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction. + #[cfg(feature = "prometheus")] + if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) { + QUEUE_PUSH_COUNT.inc(); + } + + { let uuid_string = job_id.to_string(); @@ -4384,7 +4406,7 @@ pub async fn push<'c, 'd>( .await?; } - Ok((uuid, tx)) + Ok((job_id, tx)) } pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value { From 44f99653ebb6938c0b30cd060a252974efceaff3 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:18:08 +0200 Subject: [PATCH 125/133] fix app input not handling change when value is empty (#5615) --- .../components/apps/components/inputs/AppNumberInput.svelte | 6 +++--- .../components/apps/components/inputs/AppTextInput.svelte | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/lib/components/apps/components/inputs/AppNumberInput.svelte b/frontend/src/lib/components/apps/components/inputs/AppNumberInput.svelte index 29dccf3b88..567094e224 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppNumberInput.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppNumberInput.svelte @@ -54,12 +54,12 @@ $: handleDefault(resolvedConfig.defaultValue) - $: value && onChangeValue() + $: value, onChangeValue() function onChangeValue() { - outputs?.result.set(value) + outputs?.result.set(value ?? undefined) if (iterContext && listInputs) { - listInputs.set(id, value) + listInputs.set(id, value ?? undefined) } } diff --git a/frontend/src/lib/components/apps/components/inputs/AppTextInput.svelte b/frontend/src/lib/components/apps/components/inputs/AppTextInput.svelte index dadc2d81d1..731ce20d07 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppTextInput.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppTextInput.svelte @@ -62,7 +62,7 @@ let initialHandleDefault = true $: handleDefault(resolvedConfig.defaultValue) - $: value && onValueChange() + $: value, onValueChange() function onValueChange() { let val = value ?? '' From 4aae6ab634280adc1de9abd890100b7c12c89158 Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 14 Apr 2025 16:24:15 +0200 Subject: [PATCH 126/133] fix: number input in app multiselect yields NOT_NUMBER (#5616) * fix: number input in app multiselect yields NOT_NUMBER * wrong type cast --- .../components/inputs/AppMultiSelectV2.svelte | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte b/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte index fb21e6ba07..e7b8e82728 100644 --- a/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte +++ b/frontend/src/lib/components/apps/components/inputs/AppMultiSelectV2.svelte @@ -42,14 +42,16 @@ result: [] as string[] }) - let selectedItems: (string | { value: string; label: any })[] | undefined = [ + let selectedItems: (number | string | { value: string; label: any })[] | undefined = [ ...new Set(outputs?.result.peak()) - ] as string[] + ] as (number | string | { value: string; label: any })[] function setResultsFromSelectedItems() { outputs?.result.set([ ...(selectedItems?.map((item) => { - if (typeof item == 'object' && item.value != undefined && item.label != undefined) { + if (typeof item == 'number') { + return item.toString() + } else if (typeof item == 'object' && item.value != undefined && item.label != undefined) { return item?.value ?? `NOT_STRING` } else if (typeof item == 'string') { return item @@ -80,6 +82,9 @@ if (typeof item == 'object' && item.value != undefined && item.label != undefined) { return item } + if (typeof item == 'number') { + return item.toString() + } return typeof item === 'string' ? item : `NOT_STRING` }) } @@ -97,11 +102,13 @@ return deepEqual(item.value, value) } return item == value - }) ?? (typeof value == 'string' ? value : undefined) + }) ?? + (typeof value == 'string' ? value : undefined) ?? + (typeof value == 'number' ? value.toString() : undefined) ) }) .filter((item) => item != undefined) - selectedItems = [...new Set(nvalue)] as (string | { value: string; label: any })[] + selectedItems = [...new Set(nvalue)] setResultsFromSelectedItems() } } @@ -215,7 +222,7 @@ e.target?.['parentElement']?.dispatchEvent(newe) }} > - {typeof option == 'object' ? option?.label ?? 'NO_LABEL' : option} + {typeof option == 'object' ? (option?.label ?? 'NO_LABEL') : option}
From 54c1ed1d4fe9b8040fc69e4cbe6f35fba6973df0 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Mon, 14 Apr 2025 16:25:20 +0200 Subject: [PATCH 127/133] nit app runnable s3 input (#5609) * nit app runnable s3 input * n --- .../settingsPanel/inputEditor/StaticInputEditor.svelte | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte index da0f6fbf88..078dbeb06c 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/inputEditor/StaticInputEditor.svelte @@ -200,11 +200,13 @@ bind:this={s3FilePicker} readOnlyMode={false} on:close={(e) => { - if (componentInput?.value?.s3) { + if (e.detail) { + if (componentInput) { + componentInput.value = e.detail + } s3FileUploadRawMode = true } }} - bind:selectedFileKey={componentInput.value} /> {:else if format?.startsWith('resource-') && (componentInput.value == undefined || typeof componentInput.value == 'string')} Date: Mon, 14 Apr 2025 16:25:41 +0200 Subject: [PATCH 128/133] feat: Batch re-run (#5553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * runs on svelte 5 * Line component from svelte-chartjs * Replaced all svelte-chartjs occurrences with custom wrapper * Fix props mistake * Fix illegal table structures * self-closing-tags fix * aria labels * Fixed trivial warnings and errors * @tanstack/svelte-table fix * upgrade to vite 6 * svelte-kit sync before running svelte-check * Remove on:clear which is actually on:removeAll and already handled by on:change * fix worker tags not displaying in Autoscaling * Try to fix svelte-kit sync not working during CI * remove warnings * Fix add flow page crashing * access worldStore before assignment fix * fix infinite recursions in App Editor * Replaced JSON.stringify with proper deepEqual * component mount api changed (no longer classes) * fix ci errors * Fix infinite loops in background runnable panel * factored effect on deep equal logic in onObjChange * fix "Add" not working in AgGrid Table * Replaced legacy component.$set api * Fix multiselect infinite value reaction * Fix flow input fields resetting when opening their edit tab * fix date input resetting when typing year * Remove !p-0 affecting subgrid dotted borders * fix missing debounceTemplate causing hundreds of updates * Fix AgGrid action refreshes and disppearing * resolve getItems generating random ids every rerun * fix cannot access items before init * fix sort lambda arguments being undefined * Revert "Remove !p-0 affecting subgrid dotted borders" This reverts commit c62809bb45d682a48376b071680645ed4e1c601b. * fix input not updating in decision tree editor * Update frontend/src/lib/components/schema/EditableSchemaWrapper.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * Re-added padding affecting subgrid dotted borders (#5479) * remove !p-0 in preset components * removed extra padding on accordion tabs subgrid * Fix non-reactive SchemaForm * dirty fix for the oneOf bug * feat: add nu-lang support (#5217) * feat: add nu (nushell) support * add worker tests * deactivate tables and non-any types below top-level full support will come in V1 for V0 it's better to keep things minimal and simple * add syntax highlighting used python's grammar, since nushell isn't supported by monaco nor svelte-highlights for V1 nu will get it`s own grammar * add logo * partially implement plugin support * change logo + ability to deploy + nsjail draft * static variables + get_resource + get_variable * lsp/dev.nu + initial nu lsp (not working yet) * make it work with nsjail * nullguard * Much more flexible signature parsing and better error-messages * add init script * rename nulsp to nu * install nu to dockerfile * fix merge * implement Default for MainArgSignature * stage NU_CACHE_DIR * improve dockerfiles * dev.nu for parser-wasm + flake.nix * update code for windows * add nushell to flake * upload Cargo.lock * make build.sh work on nixos * build wasm cli parsers * add docs to README_DEV.md * add helper script docker/dev.nu * improve docker/dev.nu * fix windows * commit frontend/package(lock).json * update cargo.lock * correctly update cargo.lock * remove lsp * update flake.nix to include svelte server and nushell * Revert base.sql to main * remove PLUGIN_USE_RE * make CARGO_PATH private * add nu to cli * Change flags to build wasm-nu-parser * remove flake.nix from parser-wasm * update wasm-build target * remove unused import * add cli support for nu * update github workflows * wasm-build 0.17 -> 0.19 * update build script * update cargo.lock * Fix typographical error * update Cargo.lock * update ENV_SETTINGS * use published nu parser * update package.lock * rewrite parser in tree-sitter * implement parser from scratch * polishing * change init script to match new parser * fix imports * fix cli build * fix cli build * merge * update wasm * use MiniPulledJob * update cli * change cli wasm schema * change cli * update deno.json * make wasm modules load lazily * regenerate parsers * remove leftover * update cargo.lock * clean up dnt.ts * add docs to cli/test.nu * add schema validation option * add Nu to try_validate_schema * reference frontend to new parser version * feat: unsafe parameters for sql queries (table names, column names) (#5488) * Make schema validation struct Schema Validation rules that are constructed from the schema or from the MainArgSig(TODO). * Make other validator builder * Fail dependency job like with lockfile failing for schema validator * Add last types + tests * Remove unused dependency * fix typos * Migration ID was colliding with another, changed it manually * Add Oneof + other fixes * fix: cache for querying scripts correclty handles ScriptMetadata * Add cache for schema validation from main arg sig * Prepare sqlx * Remove default features * Feature flags * WIP: unsafe sql params for sql langauges * Fix down migration table name * cleanup: put validation logic inside a function * Refactor to cache the should_validate boolean Changed the schemavalidators cache to take in an Option, effectively storing the `should_validate_schema` information. Also pass the schema when avaialble to construct the schema validator * Add other job kinds to u8 cache key just in case * Change sql languages to all get arguments as Values instead of RawValue * Only cache if not preview * Add last sql languages and some CI fixes * Rename after typo on `sanitized` * Finish rename * Remove unused import * Fix wrong test * Add newly published regex parser version * Remove default features from cargo.toml * Change to a cleaner syntax for the interpolated args * Update republished parser * fix win build (#5494) * add sysinfoapi feature flag for winapi dependency * add ff * add ff at the right place * fix(frontend): use stable path for capture tables + nits (#5495) * add missing capture move on first time deploy (#5496) * avoid regen client as build step * perf: cache workspace env variables to avoid one query (#5499) * perf: optimize number of queries needed for job run (#5504) * optPerf * update sqlx * update sqlx * fix: improve cancel for flows with many substeps * feat: list references upon renaming a script or a flow (#5487) * Refactored flow_workspace_runnables to more generic workspace_runnable_dependencies * list flows referencing an item upon renaming it * Refactor with two exclusive columns to avoid breaking FK constraints * Show apps depending on item upon renaming * sqlx prepare * list-disc instead of • * on delete and on update cascade * displayPathChangedWarning oneOf check instead noneOf * combine migrations + add "on update cascade" to flow fk * unique index on app dependencies to avoid duplicates * create new workspace_runnable_dependencies instead of renaming old table * Add "looking for references" loading msg * Revert "create new workspace_runnable_dependencies instead of renaming old table" This reverts commit 015c38ca8f9fbd8b4a4e817f1ad105434bfd68c5. * flow_workspace_runnables view for backwards compatibility * Add warning for script imports on rename * support import dependency tracking in deno * number of using scripts / flows / apps tooltip * forgot sqlx prepare * delete app-related rows in down migration * Made selection more generic * RunsBatchActionsDropdown refactor * started BatchReRunOptionsPane * fix overflow quirk * fetch schema * refactor to group jobs by (kind, path) * auto select * computePropertyMap * InputTransformForm works * Pickable properties * remove PropPickerWrapper and make it optional in InputTransformForm * hide help btn * available expressions info alert * extraLib for editor linting * fix selected group not updating * nit * Refactor async logic in script tag * persist changes in state * correct typing * count for each (path, kind) group * support flows * use dot operator when possible * count jobs and fix wrong number * fix selectedJobs recomputing periodically * (v0) individual api requests to re-run jobs * move batchReRunChangedArgs state upwards * Support static arg * mistake * Single confirmation modal + removed unnecessary state * change confirmation modal color * use runes in confirmation modal * listSelectedJobsSchema API endpoint * refactored batch rerun pane for listSelectedJobsSchema * eliminated selectedJobs * batch rerun works backend (v0 same args) * Static input transforms * simpler list_selected_jobs_schemas sql query with coalesce * use latest schema UI + refactor * run latest version in backend * add deno_core dependency to windmill-api * stream jobs from db * basic js evaluation * sqlx prepare * add id path and hash in editor lint * js works with job object! * moved deno_core logic to separate function * openapi yaml mistake * unnecessary bind * fix date as string * Stream re-ran uuids * handle SSE multiple values at once * don't select all by default on batch action * nit ui * check that schema has property backend * Better JobGroup query + cache * handle multi type properties * Notify user on error * stupid mistake * Fix warnings and update svelte-exmarkdown for svelte 5 * regen package-lock to fix crash on vite preview * batch re-run all filtered jobs * merge schemas to common type * more explicit tooltips * changed sse counter ui * typos * fix tutorial first part * nit mistake * package lock + elipsis nit * fix: latest_schema option still checked on the job original schema * always gotta forget sqlx prepare * fix flashing loading screen * fix batch re-run select all filtered * better tooltip * fix batch actions btn growing on wide screen * revert disableBatchActions * fix selectable step jobs --------- Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> Co-authored-by: pyranota <92104930+pyranota@users.noreply.github.com> Co-authored-by: wendrul <53628737+wendrul@users.noreply.github.com> Co-authored-by: Alexander Petric Co-authored-by: HugoCasa Co-authored-by: Ruben Fiszel --- ...4cdb6eae2e71e56bc8e5215559af967c81bd8.json | 79 ++++ ...953e3b32a9a7a73784219d48f8e8932cdb0c8.json | 23 + ...53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json | 85 ++++ ...3b077c957da8455ea747b0680923d8282425b.json | 73 +++ ...7e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8.json | 73 +++ ...b1ec3fc844ed286ff563f0c9a5c4778cfb718.json | 67 +++ ...bd8f537e68fabceb094fe535693fbd146eff2.json | 23 + backend/Cargo.lock | 4 + backend/Cargo.toml | 4 +- backend/windmill-api/Cargo.toml | 9 +- backend/windmill-api/openapi.yaml | 310 +++++++++---- backend/windmill-api/src/jobs.rs | 367 ++++++++++++++- .../src/lib/components/FieldHeader.svelte | 10 +- .../lib/components/InputTransformForm.svelte | 72 +-- .../src/lib/components/S3FilePicker.svelte | 4 +- .../ConfirmationModal.svelte | 54 ++- .../src/lib/components/jobs/batchReruns.ts | 110 +++++ .../runs/BatchReRunOptionsPane.svelte | 275 ++++++++++++ .../src/lib/components/runs/RunRow.svelte | 11 +- .../runs/RunsBatchActionsDropdown.svelte | 92 ++++ .../src/lib/components/runs/RunsTable.svelte | 22 +- frontend/src/lib/schema.ts | 18 +- frontend/src/lib/utils.ts | 20 +- .../(logged)/runs/[...path]/+page.svelte | 422 ++++++++++-------- 24 files changed, 1872 insertions(+), 355 deletions(-) create mode 100644 backend/.sqlx/query-3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8.json create mode 100644 backend/.sqlx/query-5ea1d8c87a17690942ca9e70e0b953e3b32a9a7a73784219d48f8e8932cdb0c8.json create mode 100644 backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json create mode 100644 backend/.sqlx/query-b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b.json create mode 100644 backend/.sqlx/query-c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8.json create mode 100644 backend/.sqlx/query-ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718.json create mode 100644 backend/.sqlx/query-d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2.json create mode 100644 frontend/src/lib/components/jobs/batchReruns.ts create mode 100644 frontend/src/lib/components/runs/BatchReRunOptionsPane.svelte create mode 100644 frontend/src/lib/components/runs/RunsBatchActionsDropdown.svelte diff --git a/backend/.sqlx/query-3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8.json b/backend/.sqlx/query-3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8.json new file mode 100644 index 0000000000..7229d4053f --- /dev/null +++ b/backend/.sqlx/query-3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8.json @@ -0,0 +1,79 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "script_hash!: _", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "scheduled_for!: _", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "input", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false, + false, + null, + null, + null, + true + ] + }, + "hash": "3376b42d273c2499f3517c7754f4cdb6eae2e71e56bc8e5215559af967c81bd8" +} diff --git a/backend/.sqlx/query-5ea1d8c87a17690942ca9e70e0b953e3b32a9a7a73784219d48f8e8932cdb0c8.json b/backend/.sqlx/query-5ea1d8c87a17690942ca9e70e0b953e3b32a9a7a73784219d48f8e8932cdb0c8.json new file mode 100644 index 0000000000..304814543b --- /dev/null +++ b/backend/.sqlx/query-5ea1d8c87a17690942ca9e70e0b953e3b32a9a7a73784219d48f8e8932cdb0c8.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT jsonb_build_object(\n 'kind', jb.kind,\n 'script_path', jb.runnable_path,\n 'latest_schema', COALESCE(\n (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow')\n ),\n 'schemas', ARRAY(\n SELECT jsonb_build_object(\n 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'),\n 'job_ids', ARRAY_AGG(DISTINCT j.id),\n 'schema', ANY_VALUE(COALESCE(s.schema, f.schema))\n ) FROM v2_job j\n LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow'\n WHERE j.id = ANY(ARRAY_AGG(jb.id))\n GROUP BY COALESCE(s.hash, f.id)\n )\n ) FROM v2_job jb\n WHERE (jb.kind = 'flow' OR jb.kind = 'script')\n AND jb.workspace_id = $1 AND jb.id = ANY($2)\n GROUP BY jb.kind, jb.runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jsonb_build_object", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + null + ] + }, + "hash": "5ea1d8c87a17690942ca9e70e0b953e3b32a9a7a73784219d48f8e8932cdb0c8" +} diff --git a/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json b/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json new file mode 100644 index 0000000000..8d51b798e9 --- /dev/null +++ b/backend/.sqlx/query-ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9.json @@ -0,0 +1,85 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n j.id,\n j.kind AS \"kind: _\",\n COALESCE(s.path, f.path) AS \"script_path!\",\n COALESCE(s.hash, f.id) AS \"script_hash!: _\",\n COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS \"scheduled_for!: _\",\n args AS input,\n COALESCE(s.schema, f.schema) AS \"schema: _\"\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n LEFT JOIN v2_job_completed jc ON jc.id = j.id\n LEFT JOIN v2_job_queue jq ON jq.id = j.id\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "script_hash!: _", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "scheduled_for!: _", + "type_info": "Timestamptz" + }, + { + "ordinal": 5, + "name": "input", + "type_info": "Jsonb" + }, + { + "ordinal": 6, + "name": "schema: _", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false, + false, + null, + null, + null, + true, + null + ] + }, + "hash": "ad88831c9021b79c9037a925c2b53be3a93dbe2a67bdc4d4342c84f439b5eaf9" +} diff --git a/backend/.sqlx/query-b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b.json b/backend/.sqlx/query-b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b.json new file mode 100644 index 0000000000..80f0549739 --- /dev/null +++ b/backend/.sqlx/query-b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n j.kind AS \"kind!: JobKind\",\n j.runnable_id AS \"script_hash: _\",\n j.runnable_path AS script_path,\n COUNT(*) AS \"count!\",\n ANY_VALUE(COALESCE(f.schema, s.schema)) AS schema\n FROM v2_job j\n LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.id = j.runnable_id AND f.path = j.runnable_path AND j.kind = 'flow'\n WHERE COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL\n AND j.workspace_id = $1 AND j.id = ANY($2)\n GROUP BY j.runnable_id, j.runnable_path, j.kind", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "script_hash: _", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "schema", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false, + true, + true, + null, + null + ] + }, + "hash": "b61a00e6a8ca0a1d24e64fdc9223b077c957da8455ea747b0680923d8282425b" +} diff --git a/backend/.sqlx/query-c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8.json b/backend/.sqlx/query-c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8.json new file mode 100644 index 0000000000..e976cd0dd9 --- /dev/null +++ b/backend/.sqlx/query-c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8.json @@ -0,0 +1,73 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT DISTINCT ON (j.runnable_path, j.kind) \n j.kind AS \"kind!: JobKind\",\n j.runnable_path AS script_path,\n NULL as \"script_hash: _\",\n -1::bigint as \"count!: _\",\n COALESCE(f.schema, s.schema) AS schema\n FROM v2_job j\n LEFT JOIN script s ON s.path = j.runnable_path AND j.kind = 'script'\n LEFT JOIN flow_version f ON f.path = j.runnable_path AND j.kind = 'flow'\n WHERE COALESCE(s.hash, f.id) IS NOT NULL\n AND j.workspace_id = $1 AND j.id = ANY($2)\n ORDER BY j.runnable_path, j.kind, COALESCE(f.created_at, s.created_at) DESC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!: JobKind", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_hash: _", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "count!: _", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "schema", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false, + true, + null, + null, + null + ] + }, + "hash": "c4f382045e5c47986e02f1e57667e6b3cb1d60f15fed9a24e3f05b1bd60c0fc8" +} diff --git a/backend/.sqlx/query-ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718.json b/backend/.sqlx/query-ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718.json new file mode 100644 index 0000000000..a457b8b2bf --- /dev/null +++ b/backend/.sqlx/query-ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718.json @@ -0,0 +1,67 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT j.kind AS \"kind: _\", COALESCE(s.path, f.path) AS \"script_path!\", COALESCE(s.hash, f.id) AS \"script_hash!: _\", args\n FROM v2_job j\n LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script'\n LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow'\n WHERE j.id = ANY($1)\n AND j.workspace_id = $2\n AND COALESCE(s.hash, f.id) IS NOT NULL\n AND COALESCE(s.path, f.path) IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind: _", + "type_info": { + "Custom": { + "name": "job_kind", + "kind": { + "Enum": [ + "script", + "preview", + "flow", + "dependencies", + "flowpreview", + "script_hub", + "identity", + "flowdependencies", + "http", + "graphql", + "postgresql", + "noop", + "appdependencies", + "deploymentcallback", + "singlescriptflow", + "flowscript", + "flownode", + "appscript" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "script_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_hash!: _", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "args", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false, + null, + null, + true + ] + }, + "hash": "ccd4c7fe5fbdf1ab4bbbbd7d2a9b1ec3fc844ed286ff563f0c9a5c4778cfb718" +} diff --git a/backend/.sqlx/query-d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2.json b/backend/.sqlx/query-d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2.json new file mode 100644 index 0000000000..5ebf571855 --- /dev/null +++ b/backend/.sqlx/query-d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(\n (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC),\n (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow')\n ) FROM v2_job jb\n WHERE jb.id = $1 AND jb.workspace_id = $2\n GROUP BY jb.kind, jb.runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "d1fdfabd4f3bbe93cf9667e8e76bd8f537e68fabceb094fe535693fbd146eff2" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 4737d318f6..0f9558966f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -13941,6 +13941,7 @@ dependencies = [ "tikv-jemalloc-sys", "tikv-jemallocator", "tokio", + "tokio-stream", "tracing", "url", "uuid", @@ -13983,6 +13984,8 @@ dependencies = [ "cookie 0.17.0", "cron", "datafusion", + "deno_core", + "deno_error", "futures", "git-version", "google-cloud-googleapis", @@ -14034,6 +14037,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-postgres 0.7.11", + "tokio-stream", "tokio-tar", "tokio-tungstenite", "tokio-util", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ef4d7c5712..ed4155f495 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -64,6 +64,7 @@ cloud = ["windmill-queue/cloud", "windmill-worker/cloud", "windmill-common/cloud jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] tantivy = ["dep:windmill-indexer", "windmill-api/tantivy", "windmill-indexer/enterprise", "windmill-indexer/parquet", "windmill-common/tantivy", "enterprise", "parquet"] sqlx = ["windmill-worker/sqlx"] +deno_core = ["windmill-worker/deno_core", "windmill-api/deno_core", "dep:deno_core", "dep:v8"] kafka = ["windmill-api/kafka"] nats = ["windmill-api/nats"] otel = ["windmill-common/otel", "windmill-worker/otel"] @@ -82,7 +83,6 @@ static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] # Languages python = ["windmill-worker/python"] -deno_core = ["windmill-worker/deno_core", "dep:deno_core", "dep:v8"] rust = ["windmill-worker/rust"] mysql = ["windmill-worker/mysql"] oracledb = ["windmill-worker/oracledb"] @@ -98,6 +98,7 @@ all_languages = [ "python", "deno_core", "rust", "mysql", "oracledb", "mssql", " [dependencies] anyhow.workspace = true tokio.workspace = true +tokio-stream.workspace = true dotenv.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } @@ -181,6 +182,7 @@ axum = { version = "^0.7", features = ["multipart"] } headers = "^0" hyper = { version = "^1", features = ["full"] } tokio = { version = "^1.42.0", features = ["full", "tracing", "time"] } +tokio-stream = { version = "0.1.17" } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors"] } tower-cookies = "^0.10" diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index c3d695f96e..0996b5956d 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -32,6 +32,7 @@ static_frontend = ["dep:rust-embed"] postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"] mqtt_trigger = ["dep:thiserror", "dep:rumqttc"] sqs_trigger = ["dep:aws-sdk-sqs", "dep:thiserror", "dep:aws-config"] +deno_core = ["dep:deno_core", "dep:deno_error"] gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] cloud = ["windmill-common/cloud"] @@ -47,6 +48,7 @@ windmill-git-sync.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-worker.workspace = true tokio.workspace = true +tokio-stream.workspace = true anyhow.workspace = true argon2.workspace = true axum.workspace = true @@ -133,4 +135,9 @@ aws-sdk-sqs = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } google-cloud-pubsub = { workspace = true, optional = true } google-cloud-googleapis = { workspace = true , optional = true } -tonic = { workspace = true, optional = true } \ No newline at end of file +tonic = { workspace = true, optional = true } +deno_error = { workspace = true, optional = true } +deno_core = { workspace = true, optional = true } + +[build-dependencies] +deno_core = { workspace = true, optional = true } \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index f1c3e0a195..5ed6537931 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1378,7 +1378,7 @@ paths: format: int64 description: The ID of the GitHub installation to delete responses: - '200': + "200": description: Installation successfully deleted /w/{workspace}/github_app/export/{installationId}: @@ -1400,7 +1400,7 @@ paths: schema: type: integer responses: - '200': + "200": description: Successfully exported the JWT token content: application/json: @@ -1435,7 +1435,7 @@ paths: jwt_token: type: string responses: - '200': + "200": description: Successfully imported the installation /users/accept_invite: @@ -1791,7 +1791,7 @@ paths: schema: $ref: "#/components/schemas/OperatorSettings" responses: - '200': + "200": description: Operator settings updated successfully content: text/plain: @@ -1997,7 +1997,7 @@ paths: required: - premium - owner - + /w/{workspace}/workspaces/threshold_alert: get: summary: get threshold alert info @@ -2368,7 +2368,6 @@ paths: application/json: schema: $ref: "#/components/schemas/AIConfig" - /w/{workspace}/workspaces/edit_error_handler: post: @@ -3538,14 +3537,14 @@ paths: tags: - teams responses: - '200': + "200": description: Teams information successfully synchronized content: application/json: schema: type: array items: - $ref: '#/components/schemas/TeamInfo' + $ref: "#/components/schemas/TeamInfo" /teams/activities: post: @@ -3579,7 +3578,7 @@ paths: description: The card block to be sent in the Teams card responses: - '200': + "200": description: Activity processed successfully /w/{workspace}/resources/create: @@ -4946,8 +4945,7 @@ paths: /scripts_u/tokened_raw/{workspace}/{token}/{path}: get: - summary: - raw script by path with a token (mostly used by lsp to be used with + summary: raw script by path with a token (mostly used by lsp to be used with import maps to resolve scripts) operationId: rawScriptByPathTokened tags: @@ -5041,6 +5039,63 @@ paths: lock_error_logs: type: string + /w/{workspace}/jobs/list_selected_job_groups: + # We use post because sending a huge array as a query param can produce + # URLs that may be too long + post: + summary: list selected jobs script/flow schemas grouped by (kind, path) + operationId: listSelectedJobGroups + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: script args + required: true + content: + application/json: + schema: + type: array + items: + type: string + format: uuid + responses: + "200": + description: result + content: + text/plain: + schema: + type: array + items: + type: object + properties: + kind: + type: string + enum: ["script", "flow"] + script_path: + type: string + latest_schema: + type: object + schemas: + type: array + items: + type: object + properties: + schema: + type: object + script_hash: + type: string + job_ids: + type: array + items: + type: string + required: [schema, script_hash, job_ids] + required: + - kind + - script_path + - latest_schema + - schemas + /w/{workspace}/jobs/run/p/{path}: post: summary: run script by path @@ -5525,7 +5580,6 @@ paths: lock_error_logs: type: string - /w/{workspace}/flows/get_triggers_count/{path}: get: summary: get triggers count of flow @@ -6523,7 +6577,6 @@ paths: in: query schema: type: boolean - requestBody: description: flow args required: true @@ -6531,7 +6584,6 @@ paths: application/json: schema: $ref: "#/components/schemas/ScriptArgs" - responses: "201": description: job created @@ -6541,6 +6593,60 @@ paths: type: string format: uuid + /w/{workspace}/jobs/run/batch_rerun_jobs: + post: + summary: re-run multiple jobs + operationId: batchReRunJobs + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: list of job ids to re run and arg tranforms + required: true + content: + application/json: + schema: + type: object + required: [job_ids, script_options_by_path, flow_options_by_path] + properties: + job_ids: + type: array + items: + type: string + script_options_by_path: + type: object + additionalProperties: + type: object + properties: + input_transforms: + type: object + additionalProperties: + $ref: "#/components/schemas/InputTransform" + use_latest_version: boolean + flow_options_by_path: + type: object + additionalProperties: + type: object + properties: + input_transforms: + type: object + additionalProperties: + $ref: "#/components/schemas/InputTransform" + use_latest_version: boolean + responses: + "201": + description: stream of created job uuids separated by \n. Lines may start with 'Error:' + example: | + a1a74c0d-708e-4539-9768-e8b3d37996bd + f0949132-5b30-48fe-bac8-873f047df810 + Error: Could not re-run 0b885808-ae89-4458-af95-c1ca3a13b0a5 + 52b9c01d-1125-4bbb-8bee-d41f26b70066 + content: + text/event-stream: + schema: + type: string + /w/{workspace}/jobs/restart/f/{id}/from/{step_id}/{branch_or_iteration_n}: post: summary: restart a completed flow at a given step @@ -6557,8 +6663,7 @@ paths: schema: type: string - name: branch_or_iteration_n - description: - for branchall or loop, the iteration at which the flow should + description: for branchall or loop, the iteration at which the flow should restart required: true in: path @@ -6926,10 +7031,82 @@ paths: schema: type: integer - /w/{workspace}/jobs/queue/list_filtered_uuids: + /w/{workspace}/jobs/list_filtered_uuids: get: summary: get the ids of all jobs matching the given filters - operationId: listFilteredUuids + operationId: listFilteredJobsUuids + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/CreatedBy" + - $ref: "#/components/parameters/Label" + - $ref: "#/components/parameters/Worker" + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/ScriptExactPath" + - $ref: "#/components/parameters/ScriptStartPath" + - $ref: "#/components/parameters/SchedulePath" + - $ref: "#/components/parameters/ScriptExactHash" + - $ref: "#/components/parameters/StartedBefore" + - $ref: "#/components/parameters/StartedAfter" + - $ref: "#/components/parameters/CreatedBefore" + - $ref: "#/components/parameters/CreatedAfter" + - $ref: "#/components/parameters/CreatedOrStartedBefore" + - $ref: "#/components/parameters/Running" + - $ref: "#/components/parameters/ScheduledForBeforeNow" + - $ref: "#/components/parameters/CreatedOrStartedAfter" + - $ref: "#/components/parameters/CreatedOrStartedAfterCompletedJob" + - $ref: "#/components/parameters/JobKinds" + - $ref: "#/components/parameters/Suspended" + - $ref: "#/components/parameters/ArgsFilter" + - $ref: "#/components/parameters/Tag" + - $ref: "#/components/parameters/ResultFilter" + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: is_skipped + description: is the job skipped + in: query + schema: + type: boolean + - name: is_flow_step + description: is the job a flow step + in: query + schema: + type: boolean + - name: has_null_parent + description: has null parent + in: query + schema: + type: boolean + - name: success + description: filter on successful jobs + in: query + schema: + type: boolean + - name: all_workspaces + description: get jobs from all workspaces (only valid if request come from the `admins` workspace) + in: query + schema: + type: boolean + - name: is_not_schedule + description: is not a scheduled job + in: query + schema: + type: boolean + responses: + "200": + description: uuids of jobs + content: + application/json: + schema: + type: array + items: + type: string + + /w/{workspace}/jobs/queue/list_filtered_uuids: + get: + summary: get the ids of all queued jobs matching the given filters + operationId: listFilteredQueueUuids tags: - job parameters: @@ -8739,7 +8916,7 @@ paths: summary: delete nats trigger operationId: deleteNatsTrigger tags: - - nats_trigger + - nats_trigger parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/Path" @@ -8768,7 +8945,6 @@ paths: schema: $ref: "#/components/schemas/NatsTrigger" - /w/{workspace}/nats_triggers/list: get: summary: list nats triggers @@ -8803,7 +8979,6 @@ paths: items: $ref: "#/components/schemas/NatsTrigger" - /w/{workspace}/nats_triggers/exists/{path}: get: summary: does nats trigger exists @@ -8850,7 +9025,6 @@ paths: schema: type: string - /w/{workspace}/nats_triggers/test: post: summary: test NATS connection @@ -9523,8 +9697,8 @@ paths: tags: - postgres_trigger parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" responses: "200": description: boolean that indicates if postgres is set to logical level or not @@ -9727,7 +9901,6 @@ paths: schema: type: string - /w/{workspace}/postgres_triggers/publication/delete/{publication}/{path}: delete: summary: delete postgres publication @@ -10840,7 +11013,8 @@ paths: required: true schema: type: string - enum: [ + enum: + [ script, group_, resource, @@ -11078,7 +11252,6 @@ paths: items: $ref: "#/components/schemas/Capture" - /w/{workspace}/capture/move/{runnable_kind}/{path}: post: summary: move captures and configs for a script or flow @@ -11336,8 +11509,7 @@ paths: /w/{workspace}/job_helpers/duckdb_connection_settings: post: - summary: - Converts an S3 resource to the set of instructions necessary to connect + summary: Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettings tags: @@ -11366,8 +11538,7 @@ paths: type: string /w/{workspace}/job_helpers/v2/duckdb_connection_settings: post: - summary: - Converts an S3 resource to the set of instructions necessary to connect + summary: Converts an S3 resource to the set of instructions necessary to connect DuckDB to an S3 bucket operationId: duckdbConnectionSettingsV2 tags: @@ -11403,8 +11574,7 @@ paths: /w/{workspace}/job_helpers/polars_connection_settings: post: - summary: - Converts an S3 resource to the set of arguments necessary to connect + summary: Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettings tags: @@ -11448,8 +11618,7 @@ paths: - client_kwargs /w/{workspace}/job_helpers/v2/polars_connection_settings: post: - summary: - Converts an S3 resource to the set of arguments necessary to connect + summary: Converts an S3 resource to the set of arguments necessary to connect Polars to an S3 bucket operationId: polarsConnectionSettingsV2 tags: @@ -11526,8 +11695,7 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" requestBody: - description: - S3 resource path to use. If empty, the S3 resource defined in the + description: S3 resource path to use. If empty, the S3 resource defined in the workspace settings will be used required: true content: @@ -12571,8 +12739,7 @@ components: type: string ParentJob: name: parent_job - description: - The parent job that is at the origin and responsible for the execution + description: The parent job that is at the origin and responsible for the execution of this script if any in: query schema: @@ -12592,8 +12759,7 @@ components: type: string NewJobId: name: job_id - description: - The job id to assign to the created job. if missing, job is chosen + description: The job id to assign to the created job. if missing, job is chosen randomly using the ULID scheme. If a job id already exists in the queue or as a completed job, the request to create one will fail (Bad Request) in: query @@ -12684,8 +12850,7 @@ components: format: date-time CreatedOrStartedAfter: name: created_or_started_after - description: - filter on created_at for non non started job and started_at otherwise + description: filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp in: query schema: @@ -12693,8 +12858,7 @@ components: format: date-time CreatedOrStartedAfterCompletedJob: name: created_or_started_after_completed_jobs - description: - filter on created_at for non non started job and started_at otherwise + description: filter on created_at for non non started job and started_at otherwise after (exclusive) timestamp but only for the completed jobs in: query schema: @@ -12702,8 +12866,7 @@ components: format: date-time CreatedOrStartedBefore: name: created_or_started_before - description: - filter on created_at for non non started job and started_at otherwise + description: filter on created_at for non non started job and started_at otherwise before (inclusive) timestamp in: query schema: @@ -12791,8 +12954,7 @@ components: enum: [Create, Update, Delete, Execute] JobKinds: name: job_kinds - description: - filter on job kind (values 'preview', 'script', 'dependencies', 'flow') + description: filter on job kind (values 'preview', 'script', 'dependencies', 'flow') separated by, in: query schema: @@ -13200,7 +13362,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript", + "appscript" ] schedule_path: type: string @@ -13307,7 +13469,7 @@ components: "singlescriptflow", "flowscript", "flownode", - "appscript", + "appscript" ] schedule_path: type: string @@ -13757,7 +13919,7 @@ components: "bytes", "dict", "datetime", - "sql", + "sql" ] - type: object properties: @@ -13797,7 +13959,7 @@ components: "bytes", "dict", "datetime", - "sql", + "sql" ] - type: object properties: @@ -13823,7 +13985,7 @@ components: "bytes", "dict", "datetime", - "sql", + "sql" ] - type: object properties: @@ -13855,8 +14017,7 @@ components: ScriptLang: type: string - enum: - [ + enum: [ python3, deno, go, @@ -14293,16 +14454,15 @@ components: - edited_at - is_flow - AuthenticationMethod: type: string enum: - - none - - windmill - - api_key - - basic_http - - custom_script - - signature + - none + - windmill + - api_key + - basic_http + - custom_script + - signature HttpTrigger: allOf: @@ -14644,7 +14804,7 @@ components: MqttQoS: type: string - enum: ['qos0', 'qos1', 'qos2'] + enum: ["qos0", "qos1", "qos2"] MqttV3Config: type: object @@ -15024,7 +15184,6 @@ components: - is_flow - enabled - Slot: type: object properties: @@ -15119,8 +15278,8 @@ components: error: type: string last_server_ping: - type: string - format: date-time + type: string + format: date-time required: - enabled - postgres_resource_path @@ -15916,13 +16075,7 @@ components: properties: type: type: string - enum: - [ - "S3Storage", - "AzureBlobStorage", - "AzureWorkloadIdentity", - "S3AwsOidc", - ] + enum: ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"] s3_resource_path: type: string azure_blob_resource_path: @@ -15937,12 +16090,7 @@ components: type: type: string enum: - [ - "S3Storage", - "AzureBlobStorage", - "AzureWorkloadIdentity", - "S3AwsOidc", - ] + ["S3Storage", "AzureBlobStorage", "AzureWorkloadIdentity", "S3AwsOidc"] s3_resource_path: type: string azure_blob_resource_path: @@ -16422,7 +16570,7 @@ components: type: array description: List of channels within the team items: - $ref: '#/components/schemas/ChannelInfo' + $ref: "#/components/schemas/ChannelInfo" ChannelInfo: type: object diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 4111f28b6b..be015f5929 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -8,7 +8,9 @@ use axum::body::Body; use axum::http::HeaderValue; -use futures::TryFutureExt; +#[cfg(feature = "deno_core")] +use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; +use futures::{StreamExt, TryFutureExt}; use http::{HeaderMap, HeaderName}; use itertools::Itertools; use quick_cache::sync::Cache; @@ -140,6 +142,13 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run/batch_rerun_jobs", + post(batch_rerun_jobs) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run/workflow_as_code/:job_id/:entrypoint", post(run_workflow_as_code) @@ -203,6 +212,13 @@ pub fn workspaced_service() -> Router { "/list", get(list_jobs).layer(Extension(api_list_jobs_query_duration)), ) + .route( + "/list_selected_job_groups", + // We use post because sending a huge array as a query param can produce + // URLs that may be too long + post(list_selected_job_groups), + ) + .route("/list_filtered_uuids", get(list_filtered_job_uuids)) .route("/queue/list", get(list_queue_jobs)) .route("/queue/count", get(count_queue_jobs)) .route("/queue/list_filtered_uuids", get(list_filtered_uuids)) @@ -645,6 +661,48 @@ async fn get_flow_job_debug_info( } } +async fn list_selected_job_groups( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(uuids): Json>, +) -> error::Result { + let mut tx = user_db.begin(&authed).await?; + + let results = sqlx::query_scalar!( + r#"SELECT jsonb_build_object( + 'kind', jb.kind, + 'script_path', jb.runnable_path, + 'latest_schema', COALESCE( + (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), + (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow') + ), + 'schemas', ARRAY( + SELECT jsonb_build_object( + 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'), + 'job_ids', ARRAY_AGG(DISTINCT j.id), + 'schema', ANY_VALUE(COALESCE(s.schema, f.schema)) + ) FROM v2_job j + LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script' + LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow' + WHERE j.id = ANY(ARRAY_AGG(jb.id)) + GROUP BY COALESCE(s.hash, f.id) + ) + ) FROM v2_job jb + WHERE (jb.kind = 'flow' OR jb.kind = 'script') + AND jb.workspace_id = $1 AND jb.id = ANY($2) + GROUP BY jb.kind, jb.runnable_path"#, + &w_id, + &uuids + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(Json(results).into_response()) +} + #[derive(Deserialize)] struct GetJobQuery { pub no_logs: Option, @@ -1745,6 +1803,37 @@ async fn cancel_selection( cancel_jobs(jobs_to_cancel, &db, authed.username.as_str(), w_id.as_str()).await } +async fn list_filtered_job_uuids( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(lq): Query, +) -> error::JsonResult> { + require_admin(authed.is_admin, &authed.username)?; + check_scopes(&authed, || format!("jobs:listjobs"))?; + + let mut sqlb = list_completed_jobs_query( + w_id.as_str(), + None, + 0, + &lq, + &["v2_job.id"], + false, + get_scope_tags(&authed), + ); + let sqlb2 = list_queue_jobs_query( + w_id.as_str(), + &lq.into(), + &["v2_job.id"], + Pagination { page: None, per_page: None }, + false, + get_scope_tags(&authed), + ); + let query = sqlb.union_all(sqlb2.subquery()?).subquery()?; + let ids = sqlx::query_scalar(query.as_str()).fetch_all(&db).await?; + Ok(Json(ids)) +} + async fn list_filtered_uuids( authed: ApiAuthed, Extension(db): Extension, @@ -1901,7 +1990,7 @@ async fn list_jobs( let sqlc = if lq.running.is_none() { Some(list_completed_jobs_query( &w_id, - per_page + offset, + Some(per_page + offset), 0, &ListCompletedQuery { order_desc: Some(true), ..lqc }, UnifiedJob::completed_job_fields(), @@ -1940,7 +2029,9 @@ async fn list_jobs( } else { if sqlc.is_none() { return Err(error::Error::BadRequest( - "cannot specify success, label, created_or_started_before, or started_before with running".to_string(), + "cannot specify success, label, created_or_started_before, or starte + d_before with running" + .to_string(), )); } sqlc.unwrap().limit(per_page).offset(offset).query()? @@ -3152,6 +3243,268 @@ pub async fn check_license_key_valid() -> error::Result<()> { Ok(()) } +use windmill_common::flows::InputTransform; + +#[derive(Deserialize)] +struct BatchReRunJobsBodyArgs { + job_ids: Vec, + script_options_by_path: HashMap, + flow_options_by_path: HashMap, +} + +#[derive(Deserialize)] +struct BatchReRunOptions { + input_transforms: Option>, + use_latest_version: Option, +} + +#[derive(sqlx::FromRow, Serialize, Clone)] +struct BatchReRunQueryReturnType { + id: Uuid, + kind: JobKind, + script_path: String, + script_hash: ScriptHash, + input: serde_json::Value, + scheduled_for: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + schema: Option, +} + +#[cfg(feature = "deno_core")] +#[op2] +#[string] +fn get_deno_core_job_value(state: &mut OpState) -> Option { + let obj = state.borrow::(); + let str = serde_json::to_string(&obj).ok()?; + Some(str) +} + +#[cfg(feature = "deno_core")] +async fn batch_rerun_compute_js_expression( + expr: String, + job: BatchReRunQueryReturnType, +) -> error::Result> { + let ext = deno_core::Extension { + name: "batch_rerun_arg_transform_ext", + ops: vec![get_deno_core_job_value()].into(), + ..Default::default() + }; + let mut isolate = + JsRuntime::new(deno_core::RuntimeOptions { extensions: vec![ext], ..Default::default() }); + + { + let op_state = isolate.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(BatchReRunQueryReturnType { schema: None, ..job }); + } + isolate + .execute_script( + "", + "let job = JSON.parse(Deno.core.ops.get_deno_core_job_value());", + ) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + + // Run user expr + let result = isolate + .execute_script("", expr) + .map_err(|e| Error::ExecutionErr(e.to_string()))?; + let mut scope = isolate.handle_scope(); + let result = v8::Local::new(&mut scope, result); + let result: serde_json::Value = + serde_v8::from_v8(&mut scope, result).map_err(|e| Error::ExecutionErr(e.to_string()))?; + let result = JsonRawValue::from_string(result.to_string())?; + Ok(result) +} + +async fn batch_rerun_jobs( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(body): Json, +) -> Response { + let stream = batch_rerun_jobs_inner(authed, db, user_db, w_id, body); + + let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); + + Response::builder() + .status(201) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .body(body) + .unwrap() +} + +fn batch_rerun_jobs_inner( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + w_id: String, + body: BatchReRunJobsBodyArgs, +) -> impl futures::Stream { + let (tx, rx) = tokio::sync::mpsc::channel(10); + tokio::spawn(async move { + let mut job_stream = sqlx::query_as!( + BatchReRunQueryReturnType, + r#"SELECT + j.id, + j.kind AS "kind: _", + COALESCE(s.path, f.path) AS "script_path!", + COALESCE(s.hash, f.id) AS "script_hash!: _", + COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS "scheduled_for!: _", + args AS input, + COALESCE(s.schema, f.schema) AS "schema: _" + FROM v2_job j + LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script' + LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow' + LEFT JOIN v2_job_completed jc ON jc.id = j.id + LEFT JOIN v2_job_queue jq ON jq.id = j.id + WHERE j.id = ANY($1) + AND j.workspace_id = $2 + AND COALESCE(s.hash, f.id) IS NOT NULL + AND COALESCE(s.path, f.path) IS NOT NULL"#, + &body.job_ids, + w_id + ).fetch(&db); + while let Some(Ok(job)) = job_stream.next().await { + let job_result = + batch_rerun_handle_job(&job, &authed, &db, &user_db, &w_id, &body).await; + let send_to_stream_result = tx + .send(match job_result { + Ok(uuid) => format!("{}\n", uuid), + Err(err) => format!("Error: {}\n", err.to_string()), + }) + .await; + match send_to_stream_result { + Ok(_) => {} + Err(e) => tracing::error!("Couldn't re-run job {}: {}", job.id, e.to_string()), + } + } + }); + tokio_stream::wrappers::ReceiverStream::new(rx) +} + +async fn batch_rerun_handle_job( + job: &BatchReRunQueryReturnType, + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &String, + body: &BatchReRunJobsBodyArgs, +) -> error::Result { + let options = if matches!(job.kind, JobKind::Script) { + &body.script_options_by_path + } else { + &body.flow_options_by_path + } + .get(&job.script_path); + + let mut args: HashMap> = serde_json::from_value(job.input.clone())?; + let use_latest_version = options.and_then(|o| o.use_latest_version).unwrap_or(false); + let input_transforms = options + .and_then(|o| o.input_transforms.as_ref()) + .map(|t| t.iter()) + .into_iter() + .flatten(); + + let latest_schema; + let schema = if use_latest_version { + latest_schema = sqlx::query_scalar!( + r#"SELECT COALESCE( + (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), + (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow') + ) FROM v2_job jb + WHERE jb.id = $1 AND jb.workspace_id = $2 + GROUP BY jb.kind, jb.runnable_path"#, + &job.id, + &w_id + ).fetch_optional(db).await?.flatten(); + latest_schema.as_ref() + } else { + job.schema.as_ref() + }; + let schema = schema + .and_then(serde_json::Value::as_object) + .and_then(|s| s.get("properties")) + .and_then(serde_json::Value::as_object); + for (property_name, transform) in input_transforms { + let schema_has_key = schema + .map(|s| s.contains_key(property_name)) + .unwrap_or(false); + if !schema_has_key { + continue; + } + match transform { + InputTransform::Static { value } => { + args.insert(property_name.clone(), value.clone()); + } + InputTransform::Javascript { expr } => { + #[cfg(not(feature = "deno_core"))] + tracing::error!("deno_core feature is not activated, cannot evaluate: {expr}"); + #[cfg(feature = "deno_core")] + args.insert( + property_name.clone(), + batch_rerun_compute_js_expression(expr.clone(), job.clone()).await?, + ); + } + } + } + + // Call appropriate function to push job to queue + match job.kind { + JobKind::Flow => { + let result = run_flow_by_path_inner( + authed.clone(), + db.clone(), + user_db.clone(), + w_id.clone(), + StripPath(job.script_path.clone()), + RunJobQuery { ..Default::default() }, + PushArgsOwned { extra: None, args }, + None, + ) + .await; + if let Ok((_, uuid)) = result { + return Ok(uuid); + } + } + JobKind::Script => { + let result = if use_latest_version { + run_script_by_path_inner( + authed.clone(), + db.clone(), + user_db.clone(), + w_id.clone(), + StripPath(job.script_path.clone()), + RunJobQuery { ..Default::default() }, + PushArgsOwned { extra: None, args }, + None, + ) + .await + } else { + run_job_by_hash_inner( + authed.clone(), + db.clone(), + user_db.clone(), + w_id.clone(), + job.script_hash, + RunJobQuery { ..Default::default() }, + PushArgsOwned { extra: None, args }, + None, + ) + .await + }; + if let Ok((_, uuid)) = result { + return Ok(uuid); + } + } + _ => {} + } + Err(error::Error::ExecutionErr( + format!("Couldn't re-run job {}", job.id).to_string(), + )) +} + pub async fn run_flow_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -5609,7 +5962,7 @@ pub fn filter_list_completed_query( pub fn list_completed_jobs_query( w_id: &str, - per_page: usize, + per_page: Option, offset: usize, lq: &ListCompletedQuery, fields: &[&str], @@ -5620,8 +5973,10 @@ pub fn list_completed_jobs_query( .fields(fields) .order_by("v2_job.created_at", lq.order_desc.unwrap_or(true)) .offset(offset) - .limit(per_page) .clone(); + if let Some(per_page) = per_page { + sqlb.limit(per_page); + } if let Some(tags) = tags { sqlb.and_where_in( @@ -5682,7 +6037,7 @@ async fn list_completed_jobs( let sql = list_completed_jobs_query( &w_id, - per_page, + Some(per_page), offset, &lq, &[ diff --git a/frontend/src/lib/components/FieldHeader.svelte b/frontend/src/lib/components/FieldHeader.svelte index 2bbbacf198..9c465f73fe 100644 --- a/frontend/src/lib/components/FieldHeader.svelte +++ b/frontend/src/lib/components/FieldHeader.svelte @@ -1,8 +1,9 @@
@@ -51,8 +54,9 @@ {/if} {#if !emptyString(simpleTooltip)} - - + + + {simpleTooltip} diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 4d67f10ba3..67f88534c5 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -29,7 +29,7 @@ import type { InputTransform } from '$lib/gen' import TemplateEditor from './TemplateEditor.svelte' import { setInputCat as computeInputCat, isCodeInjection } from '$lib/utils' - import { FunctionSquare } from 'lucide-svelte' + import { FunctionSquare, InfoIcon } from 'lucide-svelte' import { getResourceTypes } from './resourceTypesStore' import type { FlowCopilotContext } from './copilot/flow' import StepInputGen from './copilot/StepInputGen.svelte' @@ -39,6 +39,9 @@ export let schema: Schema | { properties?: Record; required?: string[] } export let arg: InputTransform | any export let argName: string + export let headerTooltip: string | undefined = undefined + export let headerTooltipIconClass = '' + export let HeaderTooltipIcon = InfoIcon export let extraLib: string = 'missing extraLib' export let inputCheck: boolean = true export let previousModuleId: string | undefined @@ -49,6 +52,7 @@ export let argExtra: Record = {} export let pickableProperties: PickableProperties | undefined = undefined export let enableAi = false + export let hideHelpButton = false let monaco: SimpleEditor | undefined = undefined let monacoTemplate: TemplateEditor | undefined = undefined @@ -70,8 +74,9 @@ const { shouldUpdatePropertyType, exprsToSet } = getContext('FlowCopilotContext') || {} - const { inputMatches, focusProp, propPickerConfig, clearFocus } = + const propPickerWrapperContext: PropPickerWrapperContext | undefined = getContext('PropPickerWrapper') + const { inputMatches, focusProp, propPickerConfig, clearFocus } = propPickerWrapperContext ?? {} function setExpr() { const newArg = $exprsToSet?.[argName] @@ -221,7 +226,7 @@ function onFocus() { focused = true if (isStaticTemplate(inputCat)) { - focusProp(argName, 'append', (path) => { + focusProp?.(argName, 'append', (path) => { const toAppend = `\$\{${path}}` arg.value = `${arg.value ?? ''}${toAppend}` monacoTemplate?.setCode(arg.value) @@ -230,7 +235,7 @@ return false }) } else { - focusProp(argName, 'insert', (path) => { + focusProp?.(argName, 'insert', (path) => { arg.expr = path arg.type = 'javascript' propertyType = 'javascript' @@ -253,7 +258,7 @@ if (propertyType == 'static') { setPropertyType(arg?.value) codeInjectionDetected = checkCodeInjection(arg?.value) != undefined - } else if (propertyType == 'javascript' && focused) { + } else if (propertyType == 'javascript' && focused && inputMatches) { // setPropertyType(arg?.expr) $inputMatches = checkCodeInjection(arg?.expr) } @@ -266,7 +271,7 @@ } function updateFocused(newFocused: boolean) { - if (focusedPrev && !newFocused) { + if (focusedPrev && !newFocused && inputMatches) { $inputMatches = undefined } focusedPrev = focused @@ -304,6 +309,9 @@
- { - if ( - $propPickerConfig?.propName == argName && - $propPickerConfig?.insertionMode == 'connect' - ) { - clearFocus() - } else { - focusProp?.(argName, 'connect', (path) => { - connectProperty(path) - dispatch('change', { argName }) - return true - }) - } - }} - /> + {#if propPickerWrapperContext} + { + if ( + $propPickerConfig?.propName == argName && + $propPickerConfig?.insertionMode == 'connect' + ) { + clearFocus() + } else { + focusProp?.(argName, 'connect', (path) => { + connectProperty(path) + dispatch('change', { argName }) + return true + }) + } + }} + /> + {/if}
{/if}
@@ -503,7 +513,7 @@ bind:code={arg.value} fontSize={14} on:change={() => { - dispatch('change', { argName }) + dispatch('change', { argName, arg }) }} /> {/if} @@ -520,7 +530,7 @@ }} shouldDispatchChanges on:change={() => { - dispatch('change', { argName }) + dispatch('change', { argName, arg }) }} label={argName} bind:editor={monaco} @@ -554,20 +564,20 @@ bind:this={monaco} bind:code={arg.expr} on:change={() => { - dispatch('change', { argName }) + dispatch('change', { argName, arg }) }} {extraLib} lang="javascript" shouldBindKey={false} on:focus={() => { focused = true - focusProp(argName, 'insert', (path) => { + focusProp?.(argName, 'insert', (path) => { monaco?.insertAtCursor(path) return false }) }} on:change={() => { - dispatch('change', { argName }) + dispatch('change', { argName, arg }) }} on:blur={() => { focused = false @@ -575,7 +585,9 @@ autoHeight />
- + {#if !hideHelpButton} + + {/if}
{:else} Not recognized input type {argName} ({arg.expr}, {propertyType}) diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index 3fd2e3a309..6a184e5ab9 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -799,7 +799,7 @@ deleteFileFromS3(fileMetadata?.fileKey) }} keyListen={false} - bind:loading={fileDeletionInProgress} + loading={fileDeletionInProgress} >
diff --git a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte index 49f6089a63..d334dc8c4c 100644 --- a/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/ConfirmationModal.svelte @@ -3,14 +3,26 @@ import { createEventDispatcher } from 'svelte' import { fade } from 'svelte/transition' import Button from '../button/Button.svelte' - import { AlertTriangle, CornerDownLeft, Loader2 } from 'lucide-svelte' + import { AlertTriangle, CornerDownLeft, Loader2, RefreshCcw } from 'lucide-svelte' - export let title: string - export let confirmationText: string - export let keyListen: boolean = true - export let loading: boolean = false + type Props = { + title: string + confirmationText: string + keyListen?: boolean + loading?: boolean + open?: boolean + type?: 'danger' | 'reload' + } - export let open: boolean = false + const { + title, + confirmationText, + keyListen = true, + loading = false, + open = false, + type: _type + }: Props = $props() + const type = $derived(_type ?? 'danger') const dispatch = createEventDispatcher() @@ -31,6 +43,27 @@ function fadeFast(node: HTMLElement) { return fade(node, { duration: 100 }) } + + const theme = { + danger: { + Icon: AlertTriangle, + color: 'red', + classes: { + icon: 'text-red-500 dark:text-red-400', + iconWrapper: 'bg-red-100 dark:bg-red-800/50' + } + }, + + reload: { + Icon: RefreshCcw, + color: 'dark', + classes: { + icon: 'text-blue-500 dark:text-blue-400', + iconWrapper: 'bg-blue-100 dark:bg-blue-800/50' + } + } + } satisfies { [type in typeof type]: any } + const Icon = $derived(theme[type].Icon ?? AlertTriangle) @@ -60,15 +93,16 @@ >
- +

{title}

+
@@ -77,14 +111,14 @@ + {/each} + {/await} +
+ + + + + {#if selected} +
+ + Use the job object to access data about the original job + +
+ { + if (!selected) return + ;(options[selected.kind][selected.script_path] ??= {}).use_latest_version = + e.detail as boolean + }} + size="sm" + options={{ + right: 'Always use latest version', + rightTooltip: + selected.kind === 'flow' + ? 'Flow jobs will always run on the latest version of the flow' + : 'Run all jobs with the latest version of the script even if they originally ran an older version' + }} + /> + + + + {@const displayedSchema = selectedUsesLatestSchema + ? (selected.latest_schema as Schema) + : mergeSchemasForBatchReruns(selected.schemas.map((s) => s.schema as Schema))} + {@const extraLib = buildExtraLibForBatchReruns({ + schemas: selected.schemas, + script_path: selected.script_path + })} +
+ {#key [selected, displayedSchema]} + {#each Object.keys(displayedSchema.properties) as propertyName} + { + if (!selected) return + const newArg = e.detail.arg as InputTransform + ;((options[selected.kind][selected.script_path] ??= {}).input_transforms ??= + {})[propertyName] = newArg + }} + argName={propertyName} + schema={displayedSchema} + {extraLib} + previousModuleId={undefined} + pickablepropertyMap={{ + hasResume: false, + previousId: undefined, + priorIds: {}, + flow_input: {} + }} + hideHelpButton + {...propertyAlwaysExists(propertyName, selected) + ? {} + : { + headerTooltip: + 'This property does not exist on all versions of the script. You can handle different cases in the code below', + HeaderTooltipIcon: TriangleAlert, + headerTooltipIconClass: 'text-orange-500' + }} + {...propertyAlwaysHasSameType(propertyName, selected) + ? {} + : { + headerTooltip: + 'This property does not always have the same type depending on the version of the script. You can handle different cases in the code below', + HeaderTooltipIcon: TriangleAlert, + headerTooltipIconClass: 'text-orange-500' + }} + /> + {/each} + {/key} +
+ {/if} + + + +
+
diff --git a/frontend/src/lib/components/runs/RunRow.svelte b/frontend/src/lib/components/runs/RunRow.svelte index 4430708091..32c2e16f3a 100644 --- a/frontend/src/lib/components/runs/RunRow.svelte +++ b/frontend/src/lib/components/runs/RunRow.svelte @@ -7,9 +7,9 @@ msToReadableTime, truncateHash, truncateRev, - isJobCancelable, isFlowPreview, - isScriptPreview + isScriptPreview, + isJobSelectable } from '$lib/utils' import { Badge, Button } from '../common' import ScheduleEditor from '../ScheduleEditor.svelte' @@ -33,6 +33,7 @@ import Portal from '$lib/components/Portal.svelte' import WaitTimeWarning from '../common/waitTimeWarning/WaitTimeWarning.svelte' + import type { RunsSelectionMode } from './RunsBatchActionsDropdown.svelte' const dispatch = createEventDispatcher() @@ -41,7 +42,7 @@ export let containerWidth: number = 0 export let containsLabel: boolean = false export let activeLabel: string | null - export let isSelectingJobsToCancel: boolean = false + export let selectionMode: RunsSelectionMode | false = false let scheduleEditor: ScheduleEditor @@ -61,13 +62,13 @@ )} style="width: {containerWidth}px" on:click={() => { - if (!isSelectingJobsToCancel || isJobCancelable(job)) { + if (!selectionMode || isJobSelectable(selectionMode)(job)) { dispatch('select') } }} >
- {#if isSelectingJobsToCancel && isJobCancelable(job)} + {#if selectionMode && isJobSelectable(selectionMode)(job)}
diff --git a/frontend/src/lib/components/runs/RunsBatchActionsDropdown.svelte b/frontend/src/lib/components/runs/RunsBatchActionsDropdown.svelte new file mode 100644 index 0000000000..1f682c4fa5 --- /dev/null +++ b/frontend/src/lib/components/runs/RunsBatchActionsDropdown.svelte @@ -0,0 +1,92 @@ + + + + +{#if isLoading} + +{:else if selectionMode} +
+ + {/if} + {#if selectionMode == 're-run'} + + {/if} +
+{:else} + onSetSelectionMode('cancel') + }, + ...($userStore?.is_admin || $superadmin + ? [{ displayName: 'Cancel all jobs matching filters', action: onCancelFilteredJobs }] + : []), + { + displayName: 'Select jobs to re-run', + action: () => onSetSelectionMode('re-run') + }, + ...($userStore?.is_admin || $superadmin + ? [{ displayName: 'Re-run all jobs matching filters', action: onReRunFilteredJobs }] + : []) + ]} + > + +
+ Batch actions + +
+ + +{/if} diff --git a/frontend/src/lib/components/runs/RunsTable.svelte b/frontend/src/lib/components/runs/RunsTable.svelte index 112a9aa5e6..17171bb669 100644 --- a/frontend/src/lib/components/runs/RunsTable.svelte +++ b/frontend/src/lib/components/runs/RunsTable.svelte @@ -8,14 +8,15 @@ import Popover from '../Popover.svelte' import { workspaceStore } from '$lib/stores' import { twMerge } from 'tailwind-merge' - import { isJobCancelable } from '$lib/utils' + import { isJobSelectable } from '$lib/utils' + import type { RunsSelectionMode } from './RunsBatchActionsDropdown.svelte' //import InfiniteLoading from 'svelte-infinite-loading' export let jobs: Job[] | undefined = undefined export let externalJobs: Job[] = [] export let omittedObscuredJobs: boolean export let showExternalJobs: boolean = false - export let isSelectingJobsToCancel: boolean = false + export let selectionMode: RunsSelectionMode | false = false export let selectedIds: string[] = [] export let selectedWorkspace: string | undefined = undefined export let activeLabel: string | null = null @@ -143,17 +144,20 @@ let allSelected: boolean = false function selectAll() { + if (!selectionMode) return if (allSelected) { allSelected = false selectedIds = [] } else { allSelected = true - selectedIds = jobs?.filter(isJobCancelable).map((j) => j.id) ?? [] + selectedIds = jobs?.filter(isJobSelectable(selectionMode)).map((j) => j.id) ?? [] } } - let cancelableJobCount: number = 0 - $: isSelectingJobsToCancel && (allSelected = selectedIds.length === cancelableJobCount) - $: isSelectingJobsToCancel && (cancelableJobCount = jobs?.filter(isJobCancelable).length ?? 0) + $: selectionMode && (allSelected = selectedIds.length === selectableJobCount) + + let selectableJobCount: number = 0 + $: selectionMode && + (selectableJobCount = jobs?.filter(isJobSelectable(selectionMode)).length ?? 0) function jobCountString(jobCount: number | undefined, lastFetchWentToEnd: boolean): string { if (jobCount === undefined) { @@ -195,7 +199,7 @@ bind:clientWidth={containerWidth} >
- {#if isSelectingJobsToCancel && cancelableJobCount != 0} + {#if selectionMode && selectableJobCount}
{ const jobId = jobOrDate.job.id - if (isSelectingJobsToCancel) { + if (selectionMode) { if (selectedIds.includes(jobOrDate.job.id)) { selectedIds = selectedIds.filter((id) => id != jobId) } else { diff --git a/frontend/src/lib/schema.ts b/frontend/src/lib/schema.ts index ae23a678ba..b01c18a628 100644 --- a/frontend/src/lib/schema.ts +++ b/frontend/src/lib/schema.ts @@ -1,15 +1,17 @@ -import type { Schema } from './common' +import type { Schema, SchemaProperty } from './common' -export function schemaToTsType(schema: Schema): string { - if (!schema || !schema.properties) { +export function schemaToTsType(schema: Schema | SchemaProperty): string { + const schemaProperties = schema.properties + const schemaRequired = schema.required + if (!schema || !schemaProperties) { return 'any' } - const propKeys = Object.keys(schema.properties) + const propKeys = Object.keys(schemaProperties) const types = propKeys .map((key: string) => { - const prop = schema.properties[key] - const isOptional = !schema.required.includes(key) + const prop = schemaProperties[key] + const isOptional = !schemaRequired?.includes(key) const prefix = `${key}${isOptional ? '?' : ''}` let type: string = 'any' if (prop.type === 'string') { @@ -19,11 +21,13 @@ export function schemaToTsType(schema: Schema): string { } else if (prop.type === 'boolean') { type = 'boolean' } else if (prop.type === 'array') { - let type = prop.items?.type ?? 'any' + type = prop.items?.type ?? 'any' if (type === 'integer') { type = 'number' } type = `${type}[]` + } else if (prop.type === 'object' && prop.properties) { + type = schemaToTsType(prop) } return `${prefix}: ${type}` diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index 1e0d9cf6dc..fd1c26bb69 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -16,10 +16,22 @@ import type { EnumType, SchemaProperty } from './common' import type { Schema } from './common' export { sendUserToast } import type { AnyMeltElement } from '@melt-ui/svelte' +import type { RunsSelectionMode } from './components/runs/RunsBatchActionsDropdown.svelte' export function isJobCancelable(j: Job): boolean { return j.type === 'QueuedJob' && !j.schedule_path && !j.canceled } +export function isJobReRunnable(j: Job): boolean { + return (j.job_kind === 'script' || j.job_kind === 'flow') && j.parent_job === undefined +} + +export function isJobSelectable(selectionType: RunsSelectionMode) { + const f: (j: Job) => boolean = { + cancel: isJobCancelable, + 're-run': isJobReRunnable + }[selectionType] + return f +} export function validateUsername(username: string): string { if (username != '' && !/^[a-zA-Z]\w+$/.test(username)) { @@ -85,7 +97,7 @@ export function displayDate( ? { day: 'numeric', month: 'numeric' - } + } : {} return date.toLocaleString(undefined, { ...timeChoices, @@ -229,7 +241,7 @@ export function clickOutside( } } - const capture = typeof options === 'boolean' ? options : options?.capture ?? true + const capture = typeof options === 'boolean' ? options : (options?.capture ?? true) document.addEventListener('click', handleClick, capture ?? true) return { @@ -605,6 +617,10 @@ export function pluralize(quantity: number, word: string, customPlural?: string) } } +export function addDeterminant(word: string): string { + return (/^[aeiou]/i.test(word) ? 'an ' : 'a ') + word +} + export function capitalize(word: string): string { return word ? word.charAt(0).toUpperCase() + word.slice(1) : '' } diff --git a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte index 11fc0732ac..4f57d3d1ed 100644 --- a/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/runs/[...path]/+page.svelte @@ -7,12 +7,13 @@ FolderService, ScriptService, FlowService, - type ExtendedJobs + type ExtendedJobs, + OpenAPI } from '$lib/gen' import { page } from '$app/stores' import { sendUserToast } from '$lib/toast' - import { superadmin, userStore, workspaceStore, userWorkspaces } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Button, Drawer, DrawerContent, Skeleton } from '$lib/components/common' import RunChart from '$lib/components/RunChart.svelte' @@ -31,19 +32,28 @@ import { twMerge } from 'tailwind-merge' import ManuelDatePicker from '$lib/components/runs/ManuelDatePicker.svelte' import JobLoader from '$lib/components/runs/JobLoader.svelte' - import { AlertTriangle, Calendar, Check, ChevronDown, Clock, X } from 'lucide-svelte' + import { AlertTriangle, Calendar, ChevronDown, Clock } from 'lucide-svelte' import ConcurrentJobsChart from '$lib/components/ConcurrentJobsChart.svelte' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { goto } from '$app/navigation' import { base } from '$app/paths' - import { isJobCancelable } from '$lib/utils' + import type { RunsSelectionMode } from '$lib/components/runs/RunsBatchActionsDropdown.svelte' + import RunsBatchActionsDropdown from '$lib/components/runs/RunsBatchActionsDropdown.svelte' + import { isJobSelectable } from '$lib/utils' + import BatchReRunOptionsPane, { + type BatchReRunOptions + } from '$lib/components/runs/BatchReRunOptionsPane.svelte' let jobs: Job[] | undefined let selectedIds: string[] = [] + let loadingSelectedIds = false + $: loadingSelectedIds && selectedIds.length && setTimeout(() => (loadingSelectedIds = false), 250) let selectedWorkspace: string | undefined = undefined + let batchReRunOptions: BatchReRunOptions = { flow: {}, script: {} } + // All Filters // Filter by let path: string | null = $page.params.path @@ -158,9 +168,17 @@ let selectedManualDate = 0 let autoRefresh: boolean = getAutoRefresh() let runDrawer: Drawer - let isCancelingVisibleJobs = false - let isCancelingFilteredJobs = false let lookback: number = 1 + let askingForConfirmation: + | undefined + | { + title: string + confirmBtnText: string + loading?: boolean + preContent?: string + onConfirm?: () => void + type?: ConfirmationModal['$$prop_def']['type'] + } = undefined function getAutoRefresh() { try { @@ -353,8 +371,8 @@ lastFetchWentToEnd = false selectedManualDate = 0 selectedIds = [] - jobIdsToCancel = [] - isSelectingJobsToCancel = false + batchReRunOptions = { flow: {}, script: {} } + selectionMode = false selectedWorkspace = undefined jobLoader?.loadJobs(minTs, maxTs, true) } @@ -484,22 +502,29 @@ } } - let jobIdsToCancel: string[] = [] - let isSelectingJobsToCancel = false - let fetchingFilteredJobs = false - let selectedFiltersString: string | undefined = undefined + let selectionMode: RunsSelectionMode | false = false - async function cancelVisibleJobs() { - isSelectingJobsToCancel = true - selectedIds = jobs?.filter(isJobCancelable).map((j) => j.id) ?? [] - if (selectedIds.length === 0) { - sendUserToast('There are no visible jobs that can be canceled', true) + async function onSetSelectionMode(mode: RunsSelectionMode | false) { + selectionMode = mode + if (!mode) { + selectedIds = [] + batchReRunOptions = { flow: {}, script: {} } + return + } + const selectableIds = jobs?.filter(isJobSelectable(mode)).map((j) => j.id) ?? [] + selectedIds = [] + + if (!selectableIds?.length) { + sendUserToast( + 'There are no visible jobs that can be ' + + { cancel: 'cancelled', 're-run': 're-ran' }[mode], + true + ) } } - async function cancelFilteredJobs() { - isCancelingFilteredJobs = true - fetchingFilteredJobs = true - const selectedFilters = { + + function getSelectedFilters() { + return { workspace: $workspaceStore ?? '', startedBefore: maxTs, startedAfter: minTs, @@ -536,19 +561,148 @@ allWorkspaces: allWorkspaces ? true : undefined, allowWildcards: allowWildcards ? true : undefined } - - selectedFiltersString = JSON.stringify(selectedFilters, null, 4) - jobIdsToCancel = await JobService.listFilteredUuids(selectedFilters) - fetchingFilteredJobs = false } - async function cancelSelectedJobs() { - jobIdsToCancel = selectedIds - isCancelingVisibleJobs = true + async function cancelJobs(uuidsToCancel: string[]) { + const uuids = await JobService.cancelSelection({ + workspace: $workspaceStore ?? '', + requestBody: uuidsToCancel + }) + selectedIds = [] + jobLoader?.loadJobs(minTs, maxTs, true, true) + sendUserToast(`Canceled ${uuids.length} jobs`) + selectionMode = false } - function jobCountString(count: number) { - return `${count} ${count == 1 ? 'job' : 'jobs'}` + async function onCancelFilteredJobs() { + askingForConfirmation = { + title: 'Confirm cancelling all jobs corresponding to the selected filters', + confirmBtnText: 'Loading...', + loading: true + } + + const selectedFilters = getSelectedFilters() + const selectedFiltersString = JSON.stringify(selectedFilters, null, 4) + const jobIdsToCancel = await JobService.listFilteredQueueUuids(selectedFilters) + + askingForConfirmation = { + title: `Confirm cancelling all jobs corresponding to the selected filters (${jobIdsToCancel.length} jobs)`, + confirmBtnText: `Cancel ${jobIdsToCancel.length} jobs that matched the filters`, + preContent: selectedFiltersString, + onConfirm: () => { + cancelJobs(jobIdsToCancel) + } + } + } + + async function onCancelSelectedJobs() { + askingForConfirmation = { + confirmBtnText: `Cancel ${selectedIds.length} jobs`, + title: 'Confirm cancelling the selected jobs', + onConfirm: () => { + cancelJobs(selectedIds) + } + } + } + + async function reRunJobs(jobIdsToReRun: string[]) { + if (!$workspaceStore) return + + if (askingForConfirmation) { + askingForConfirmation.loading = true + } + + const body: Parameters[0]['requestBody'] = { + job_ids: jobIdsToReRun, + script_options_by_path: batchReRunOptions.script, + flow_options_by_path: batchReRunOptions.flow + } + + // workaround because EventSource does not support POST requests + // https://medium.com/@david.richards.tech/sse-server-sent-events-using-a-post-request-without-eventsource-1c0bd6f14425 + const response = await fetch(`${OpenAPI.BASE}/w/${$workspaceStore}/jobs/run/batch_rerun_jobs`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }) + await new Promise(async (resolve) => { + const reader = response?.body?.pipeThrough(new TextDecoderStream()).getReader() + let reRanUuids: string[] = [] + if (reader) { + while (true) { + const { value, done } = await reader.read() + if (value) { + // It is possible get multiple values at once in case of buffering + const receivedUuids: string[] = [] + for (const line of value.split('\n')) { + if (!line) continue + else if (line.startsWith('Error:')) { + console.error(line) + } else { + receivedUuids.push(line) + } + } + if (receivedUuids.length) { + reRanUuids.push(...receivedUuids) + if (askingForConfirmation) { + askingForConfirmation.confirmBtnText = `${reRanUuids.length}/${jobIdsToReRun.length}` + } + } + } + + if (done || !value) { + if (reRanUuids.length) { + sendUserToast(`Re-ran ${reRanUuids.length}/${jobIdsToReRun.length} jobs`) + } + if (reRanUuids.length !== jobIdsToReRun.length) { + sendUserToast( + `Failed to re-run ${jobIdsToReRun.length - reRanUuids.length} jobs. Check console for details`, + true + ) + // We do not get explicit error from backend if the job script don't exist + for (const jobId of jobIdsToReRun) { + if (reRanUuids.includes(jobId)) continue + console.error('Could not re-run job ' + jobId) + } + } + break + } + } + } + resolve(undefined) + }) + + selectedIds = [] + batchReRunOptions = { flow: {}, script: {} } + jobLoader?.loadJobs(minTs, maxTs, true, true) + selectionMode = false + } + + async function onReRunFilteredJobs() { + const selectedFilters = getSelectedFilters() + selectedIds = [] + loadingSelectedIds = true + + if (jobKindsCat !== 'runs') { + sendUserToast('Batch re-run is only supported for scripts and flows', true) + } + selectedIds = await JobService.listFilteredJobsUuids({ + ...selectedFilters, + jobKinds: 'script,flow' + }) + selectionMode = 're-run' + } + + async function onReRunSelectedJobs() { + const jobIdsToReRun = selectedIds + askingForConfirmation = { + title: `Confirm re-running the selected jobs`, + confirmBtnText: `Re-run ${jobIdsToReRun.length} jobs`, + type: 'reload', + onConfirm: async () => { + await reRunJobs(jobIdsToReRun) + } + } } function setLookback(lookbackInDays: number) { @@ -628,50 +782,25 @@ /> { - isCancelingFilteredJobs = false - let uuids = await JobService.cancelSelection({ - workspace: $workspaceStore ?? '', - requestBody: jobIdsToCancel - }) - jobIdsToCancel = [] - selectedIds = [] - jobLoader?.loadJobs(minTs, maxTs, true, true) - sendUserToast(`Canceled ${uuids.length} jobs`) - isSelectingJobsToCancel = false + const func = askingForConfirmation?.onConfirm + await func?.() + askingForConfirmation = undefined }} - loading={fetchingFilteredJobs} + type={askingForConfirmation?.type} + loading={askingForConfirmation?.loading} on:canceled={() => { - isCancelingFilteredJobs = false + askingForConfirmation = undefined }} > - 
{selectedFiltersString}
+ {#if askingForConfirmation?.preContent} + 
{askingForConfirmation.preContent}
+ {/if} - { - isCancelingVisibleJobs = false - let uuids = await JobService.cancelSelection({ - workspace: $workspaceStore ?? '', - requestBody: jobIdsToCancel - }) - jobIdsToCancel = [] - selectedIds = [] - jobLoader?.loadJobs(minTs, maxTs, true, true) - sendUserToast(`Canceled ${uuids.length} jobs`) - isSelectingJobsToCancel = false - }} - on:canceled={() => { - isCancelingVisibleJobs = false - }} -/> - {#if selectedIds.length === 1} @@ -815,7 +944,7 @@ -
- {#if isSelectingJobsToCancel} -
- -
- {:else if !$userStore?.is_admin && !$superadmin} - - -
- Cancel jobs - -
- - - {:else} - - -
- Cancel jobs - -
- - - {/if} -
+
{/if} - - {#if selectedIds.length === 1} + + {#if selectionMode === 're-run'} + + {:else if selectedIds.length === 1} {#if selectedIds[0] === '-'}
There is no information available for this job
{:else} @@ -1195,7 +1273,7 @@ {/if}
- {#if isSelectingJobsToCancel} -
- -
- {:else if !$userStore?.is_admin && !$superadmin} - - -
- Cancel jobs - -
- - - {:else} - - -
- Cancel jobs - -
- - - {/if} +
@@ -1427,13 +1453,13 @@ externalJobs={externalJobs ?? []} omittedObscuredJobs={extendedJobs?.omitted_obscured_jobs ?? false} showExternalJobs={!graphIsRunsChart} - {isSelectingJobsToCancel} + {selectionMode} bind:selectedIds bind:selectedWorkspace bind:lastFetchWentToEnd on:loadExtra={loadExtra} on:select={() => { - if (!isSelectingJobsToCancel) runDrawer.openDrawer() + if (!selectionMode) runDrawer.openDrawer() }} on:filterByPath={filterByPath} on:filterByUser={filterByUser} From 8d062c47ecd9e84a81140d5c59814da9217dd434 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Apr 2025 10:14:05 +0200 Subject: [PATCH 129/133] fix(cli): wmill-locks improvement --- cli/main.ts | 36 ++++++++-------- cli/metadata.ts | 111 ++++++++++++++++++++++++++++-------------------- 2 files changed, 84 insertions(+), 63 deletions(-) diff --git a/cli/main.ts b/cli/main.ts index 000f4c6e4c..92131ce63b 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -33,6 +33,7 @@ import { pull, push } from "./sync.ts"; import { add as workspaceAdd } from "./workspace.ts"; import workers from "./workers.ts"; import queues from "./queues.ts"; +import { readLockfile } from "./metadata.ts"; export { flow, @@ -96,24 +97,25 @@ const command = new Command() .command("init", "Bootstrap a windmill project with a wmill.yaml file") .action(async () => { if (await Deno.stat("wmill.yaml").catch(() => null)) { - log.error(colors.red("wmill.yaml already exists")); - return; + log.error(colors.green("wmill.yaml already exists")); + } else { + await Deno.writeTextFile( + "wmill.yaml", + yamlStringify({ + defaultTs: "bun", + includes: ["f/**"], + excludes: [], + codebases: [], + skipVariables: true, + skipResources: true, + skipSecrets: true, + includeSchedules: false, + includeTriggers: false, + }) + ); + log.info(colors.green("wmill.yaml created")); } - await Deno.writeTextFile( - "wmill.yaml", - yamlStringify({ - defaultTs: "bun", - includes: ["f/**"], - excludes: [], - codebases: [], - skipVariables: true, - skipResources: true, - skipSecrets: true, - includeSchedules: false, - includeTriggers: false, - }) - ); - log.info(colors.green("wmill.yaml created")); + await readLockfile(); }) .command("app", app) .command("flow", flow) diff --git a/cli/metadata.ts b/cli/metadata.ts index d00ef071da..3a83143182 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -33,11 +33,11 @@ import { FlowValue } from "./gen/types.gen.ts"; export class LockfileGenerationError extends Error { constructor(message: string) { super(message); - this.name = 'LockfileGenerationError'; + this.name = "LockfileGenerationError"; } } -export async function generateAllMetadata() { } +export async function generateAllMetadata() {} function findClosestRawReqs( lang: "bun" | "python3" | "php" | undefined, @@ -259,7 +259,7 @@ export async function generateScriptMetadataInternal( rawReqs ); } else { - metadataParsedContent.lock = ''; + metadataParsedContent.lock = ""; } } else { metadataParsedContent.lock = @@ -369,11 +369,7 @@ async function updateScriptLock( ); } throw new LockfileGenerationError( - `Failed to generate lockfile: ${JSON.stringify( - response, - null, - 2 - )}` + `Failed to generate lockfile: ${JSON.stringify(response, null, 2)}` ); } const lockPath = remotePath + ".script.lock"; @@ -385,7 +381,7 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch { } + } catch {} metadataContent.lock = ""; } } catch (e) { @@ -427,7 +423,7 @@ export async function updateFlow( | { error: { message: string } } | undefined; if (rawResponse.status != 200) { - const msg = (res as any)?.["error"]?.["message"] + const msg = (res as any)?.["error"]?.["message"]; if (msg) { throw new LockfileGenerationError( `Failed to generate lockfile: ${msg}` @@ -441,7 +437,7 @@ export async function updateFlow( } catch (e) { try { responseText = await rawResponse.text(); - } catch { } + } catch {} throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` ); @@ -463,22 +459,26 @@ export async function inferSchema( }> { let inferedSchema: any; if (language === "python3") { - const { parse_python } = await import('./wasm/python/windmill_parser_wasm.js') + const { parse_python } = await import( + "./wasm/python/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_python(content)); } else if (language === "nativets") { - const { parse_deno } = await import('./wasm/ts/windmill_parser_wasm.js') + const { parse_deno } = await import("./wasm/ts/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "bun") { - const { parse_deno } = await import('./wasm/ts/windmill_parser_wasm.js') + const { parse_deno } = await import("./wasm/ts/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "deno") { - const { parse_deno } = await import('./wasm/ts/windmill_parser_wasm.js') + const { parse_deno } = await import("./wasm/ts/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "go") { const { parse_go } = await import("./wasm/go/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_go(content)); } else if (language === "mysql") { - const { parse_mysql } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_mysql } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_mysql(content)); inferedSchema.args = [ @@ -486,28 +486,36 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "bigquery") { - const { parse_bigquery } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_bigquery } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_bigquery(content)); inferedSchema.args = [ { name: "database", typ: { resource: "bigquery" } }, ...inferedSchema.args, ]; } else if (language === "oracledb") { - const { parse_oracledb } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_oracledb } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_oracledb(content)); inferedSchema.args = [ { name: "database", typ: { resource: "oracledb" } }, ...inferedSchema.args, ]; } else if (language === "snowflake") { - const { parse_snowflake } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_snowflake } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_snowflake(content)); inferedSchema.args = [ { name: "database", typ: { resource: "snowflake" } }, ...inferedSchema.args, ]; } else if (language === "mssql") { - const { parse_mssql } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_mssql } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_mssql(content)); inferedSchema.args = [ { name: "database", typ: { resource: "ms_sql_server" } }, @@ -521,7 +529,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "graphql") { - const { parse_graphql } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_graphql } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_graphql(content)); inferedSchema.args = [ { name: "api", typ: { resource: "graphql" } }, @@ -531,7 +541,9 @@ export async function inferSchema( const { parse_bash } = await import("./wasm/regex/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_bash(content)); } else if (language === "powershell") { - const { parse_powershell } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_powershell } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_powershell(content)); } else if (language === "php") { const { parse_php } = await import("./wasm/php/windmill_parser_wasm.js"); @@ -540,18 +552,22 @@ export async function inferSchema( const { parse_rust } = await import("./wasm/rust/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_rust(content)); } else if (language === "csharp") { - const { parse_csharp } = await import("./wasm/csharp/windmill_parser_wasm.js"); + const { parse_csharp } = await import( + "./wasm/csharp/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_csharp(content)); } else if (language === "nu") { const { parse_nu } = await import("./wasm/nu/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_nu(content)); } else if (language === "ansible") { - const { parse_ansible } = await import("./wasm/yaml/windmill_parser_wasm.js"); + const { parse_ansible } = await import( + "./wasm/yaml/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_ansible(content)); } else if (language === "java") { const { parse_java } = await import("./wasm/java/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_java(content)); - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -569,10 +585,12 @@ export async function inferSchema( } if (!currentSchema) { - currentSchema = {} + currentSchema = {}; } currentSchema.required = []; - const oldProperties = JSON.parse(JSON.stringify(currentSchema?.properties ?? {})); + const oldProperties = JSON.parse( + JSON.stringify(currentSchema?.properties ?? {}) + ); currentSchema.properties = {}; for (const arg of inferedSchema.args) { @@ -594,7 +612,6 @@ export async function inferSchema( } } - return { schema: currentSchema, has_preprocessor: inferedSchema.has_preprocessor, @@ -620,23 +637,23 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null; - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null; + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { - label: string; - properties: { key: string; typ: any }[]; - } - ]; - }, + oneof: [ + { + label: string; + properties: { key: string; typ: any }[]; + } + ]; + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: "" }; @@ -839,10 +856,10 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + }) | undefined, globalDeps: GlobalDeps, codebases: SyncCodebase[] @@ -932,7 +949,7 @@ const WMILL_LOCKFILE = "wmill-lock.yaml"; export async function readLockfile(): Promise { try { const read = await yamlParseFile(WMILL_LOCKFILE); - if (typeof read == "object") { + if (typeof read == "object" && read != null) { return read as Lock; } else { throw new Error("Invalid lockfile"); @@ -940,6 +957,8 @@ export async function readLockfile(): Promise { } catch { const lock = { locks: {} }; await Deno.writeTextFile(WMILL_LOCKFILE, yamlStringify(lock, yamlOptions)); + log.info(colors.green("wmill-lock.yaml created")); + return lock; } } From 7ede048bb5d66100f4e59d32d5b785c2b8f06295 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Apr 2025 10:20:10 +0200 Subject: [PATCH 130/133] nit cli --- cli/sync.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/sync.ts b/cli/sync.ts index 884d0e667e..f31cf3f328 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -41,6 +41,7 @@ import { SyncCodebase, listSyncCodebases } from "./codebase.ts"; import { generateFlowLockInternal, generateScriptMetadataInternal, + readLockfile, } from "./metadata.ts"; import { FlowModule, OpenFlow, RawScript } from "./gen/types.gen.ts"; import { pushResource } from "./resource.ts"; @@ -343,7 +344,7 @@ export function newPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { else if (language == "nu") ext = "nu"; else if (language == "ansible") ext = "playbook.yml"; else if (language == "java") ext = "java"; - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG else ext = "no_ext"; return [`${name}.inline_script.`, ext]; @@ -659,9 +660,8 @@ export async function elementsToMap( path.endsWith(".nats_trigger" + ext) || path.endsWith(".postgres_trigger" + ext) || path.endsWith(".mqtt_trigger" + ext) || - path.endsWith(".sqs_trigger" + ext) || - path.endsWith(".gcp_trigger" + ext) - ) + path.endsWith(".sqs_trigger" + ext) || + path.endsWith(".gcp_trigger" + ext)) ) continue; if (!skips.includeUsers && path.endsWith(".user" + ext)) continue; @@ -695,7 +695,7 @@ export async function elementsToMap( "yml", "nu", "java", - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG ].includes(path.split(".").pop() ?? "") && !isFileResource(path) ) @@ -1273,7 +1273,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { } } log.info("All local changes pulled, now updating wmill-lock.yaml"); - + await readLockfile(); // ensure wmill-lock.yaml exists const globalDeps = await findGlobalDeps(); const tracker: ChangeTracker = await buildTracker(changes); From cd7ab5165e0ad2884f280d20f4d2624b37e5c417 Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Tue, 15 Apr 2025 13:07:20 +0200 Subject: [PATCH 131/133] Change port for indexer (to remove collision with prometheus) (#5619) --- Caddyfile | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Caddyfile b/Caddyfile index 67925e6492..933407b98d 100644 --- a/Caddyfile +++ b/Caddyfile @@ -12,7 +12,7 @@ bind {$ADDRESS} reverse_proxy /ws/* http://lsp:3001 # reverse_proxy /ws_mp/* http://multiplayer:3002 - # reverse_proxy /api/srch/* http://windmill_indexer:8001 + # reverse_proxy /api/srch/* http://windmill_indexer:8002 reverse_proxy /* http://windmill_server:8000 # tls /certs/cert.pem /certs/key.pem } diff --git a/docker-compose.yml b/docker-compose.yml index a8ac7ba565..82c66a9e69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -135,9 +135,9 @@ services: replicas: 0 # set to 1 to enable full-text job and log search restart: unless-stopped expose: - - 8001 + - 8002 environment: - - PORT=8001 + - PORT=8002 - DATABASE_URL=${DATABASE_URL} - MODE=indexer depends_on: From 8a4ef022c828599a7ee2de5414730f2c60b4c080 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 15 Apr 2025 16:12:14 +0200 Subject: [PATCH 132/133] remove rgossiaux dep --- frontend/package-lock.json | 9 --------- frontend/package.json | 4 ---- 2 files changed, 13 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4ca3ed9a53..207f5b22f5 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -89,7 +89,6 @@ "@melt-ui/pp": "^0.3.2", "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.34.3", - "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.16.0", "@sveltejs/package": "^2.3.7", @@ -1209,14 +1208,6 @@ "version": "0.0.1", "license": "SEE LICENSE IN LICENSE" }, - "node_modules/@rgossiaux/svelte-headlessui": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "peerDependencies": { - "svelte": "^3.47.0" - } - }, "node_modules/@rollup/pluginutils": { "version": "5.1.4", "dev": true, diff --git a/frontend/package.json b/frontend/package.json index f8ceffa287..5d0e13fe3f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,7 +22,6 @@ "@melt-ui/pp": "^0.3.2", "@melt-ui/svelte": "^0.86.2", "@playwright/test": "^1.34.3", - "@rgossiaux/svelte-headlessui": "^2.0.0", "@sveltejs/adapter-static": "^3.0.6", "@sveltejs/kit": "^2.16.0", "@sveltejs/package": "^2.3.7", @@ -73,9 +72,6 @@ "yootils": "^0.3.1" }, "overrides": { - "@rgossiaux/svelte-headlessui": { - "svelte": "$svelte" - }, "monaco-graphql": { "monaco-editor": "$monaco-editor" } From 841391923755af5846466502bbfb3db2781520ee Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Tue, 15 Apr 2025 17:19:57 +0200 Subject: [PATCH 133/133] replace on click wt on pointer down on flow node click (#5620) * replace on click wt on pointer down on flow node click * pointerdown on virtualitems --------- Co-authored-by: Diego Imbert --- .../components/flows/map/FlowModuleSchemaItem.svelte | 2 +- frontend/src/lib/components/flows/map/MapItem.svelte | 12 ++++++------ .../components/flows/map/VirtualItemWrapper.svelte | 2 +- .../graph/renderers/triggers/TriggersWrapper.svelte | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index e0ad53a894..84f59161da 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -203,7 +203,7 @@ style="width: 275px; height: 38px; background-color: {bgColor};" on:mouseenter={() => (hover = true)} on:mouseleave={() => (hover = false)} - on:click|preventDefault|stopPropagation + on:pointerdown|preventDefault|stopPropagation >
{#if retry} diff --git a/frontend/src/lib/components/flows/map/MapItem.svelte b/frontend/src/lib/components/flows/map/MapItem.svelte index 1ce5f2c918..dceff7d539 100644 --- a/frontend/src/lib/components/flows/map/MapItem.svelte +++ b/frontend/src/lib/components/flows/map/MapItem.svelte @@ -122,7 +122,7 @@ on:changeId on:move={() => dispatch('move')} on:delete={onDelete} - on:click={() => dispatch('select', mod.id)} + on:pointerdown={() => dispatch('select', mod.id)} on:updateMock={({ detail }) => { mod.mock = detail dispatch('updateMock') @@ -148,7 +148,7 @@ on:changeId on:delete={onDelete} on:move={() => dispatch('move')} - on:click={() => dispatch('select', mod.id)} + on:pointerdown={() => dispatch('select', mod.id)} {...itemProps} id={mod.id} label={mod.summary || 'Run one branch'} @@ -165,7 +165,7 @@ on:changeId on:delete={onDelete} on:move={() => dispatch('move')} - on:click={() => dispatch('select', mod.id)} + on:pointerdown={() => dispatch('select', mod.id)} id={mod.id} {...itemProps} label={mod.summary || `Run all branches${mod.value.parallel ? ' (parallel)' : ''}`} @@ -180,7 +180,7 @@ {retries} {editMode} on:changeId - on:click={() => dispatch('select', mod.id)} + on:pointerdown={() => dispatch('select', mod.id)} on:delete={onDelete} on:move={() => dispatch('move')} on:updateMock={({ detail }) => { @@ -196,8 +196,8 @@ (mod.id === 'preprocessor' ? 'Preprocessor' : mod.id.startsWith('failure') - ? 'Error Handler' - : undefined) || + ? 'Error Handler' + : undefined) || (`path` in mod.value ? mod.value.path : undefined) || (mod.value.type === 'rawscript' ? `Inline ${prettyLanguage(mod.value.language)}` diff --git a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte index f4a136885e..f1ce5ee1bb 100644 --- a/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte +++ b/frontend/src/lib/components/flows/map/VirtualItemWrapper.svelte @@ -33,7 +33,7 @@ onTop ? 'z-[901]' : '' )} style="width: 275px; max-height: 38px; background-color: {bgColor} !important;" - on:click={() => { + on:pointerdown={() => { if (selectable) { if (id) { dispatch('select', id) diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte index 3eab0f4d7a..ced0a81d86 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte @@ -25,7 +25,7 @@ class="flex w-full flex-row gap-1 px-2 p-1 items-center {selected ? 'outline outline-2 outline-gray-600 rounded-sm dark:bg-white/5 dark:outline-gray-400' : ''}" - on:click={() => { + on:pointerdown={() => { dispatch('select') }} >