diff --git a/.github/workflows/git-commands.yaml b/.github/workflows/git-commands.yaml index cafc4fed05..8a5d94df64 100644 --- a/.github/workflows/git-commands.yaml +++ b/.github/workflows/git-commands.yaml @@ -29,10 +29,16 @@ jobs: --health-retries 5 steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -44,6 +50,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -70,21 +77,25 @@ jobs: - name: Run update-sqlx script env: DATABASE_URL: postgres://postgres:postgres@localhost:5432/windmill - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + set -e # Exit on any command failure PR_NUMBER=${{ github.event.issue.number }} BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME - mkdir frontend/build + mkdir -p frontend/build cd backend cargo install sqlx-cli --version 0.8.5 sqlx migrate run - ./update_sqlx.sh --dir ./windmill-ee-private + if ! ./update_sqlx.sh --dir ./windmill-ee-private; then + gh pr comment $PR_NUMBER --body "❌ SQLx update failed. Please check the workflow logs for details." + exit 1 + fi # Pass the branch name to the next step echo "BRANCH_NAME=$BRANCH_NAME" >> $GITHUB_ENV @@ -97,7 +108,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -114,10 +125,16 @@ jobs: pull-requests: write issues: write steps: + - uses: actions/create-github-app-token@v2 + id: app + with: + app-id: ${{ vars.INTERNAL_APP_ID }} + private-key: ${{ secrets.INTERNAL_APP_KEY }} + - name: Comment on PR - Starting uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, @@ -129,6 +146,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 with: + token: ${{ steps.app.outputs.token }} ref: ${{ github.event.issue.pull_request.head.ref }} fetch-depth: 0 @@ -149,19 +167,19 @@ jobs: - name: Update ee-repo-ref.txt env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app.outputs.token }} run: | + PR_NUMBER=${{ github.event.issue.number }} + BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) echo "Checking out PR branch: $BRANCH_NAME" git checkout $BRANCH_NAME - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" + git config --local user.email "windmill-internal-app[bot]@users.noreply.github.com" + git config --local user.name "windmill-internal-app[bot]" git config pull.rebase true git pull origin $BRANCH_NAME echo "${{ steps.get-commit-hash.outputs.commit_hash }}" > backend/ee-repo-ref.txt echo "Updated backend/ee-repo-ref.txt with commit hash: ${{ steps.get-commit-hash.outputs.commit_hash }}" # commit and push the changes - PR_NUMBER=${{ github.event.issue.number }} - BRANCH_NAME=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName) git add backend/ee-repo-ref.txt git commit -m "Update ee-repo-ref.txt" || echo "No changes to commit" git push origin $BRANCH_NAME @@ -169,7 +187,7 @@ jobs: - name: Comment on PR - Completed uses: actions/github-script@v6 with: - github-token: ${{ secrets.GITHUB_TOKEN }} + github-token: ${{ steps.app.outputs.token }} script: | github.rest.issues.createComment({ issue_number: context.issue.number, diff --git a/CHANGELOG.md b/CHANGELOG.md index fcf81139a7..6320014cf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,35 @@ # Changelog +## [1.544.1](https://github.com/windmill-labs/windmill/compare/v1.544.0...v1.544.1) (2025-09-18) + + +### Bug Fixes + +* fix onLoad auth issue ([bb4699b](https://github.com/windmill-labs/windmill/commit/bb4699bdc6d4e458175594db92b34fdf12fadf1d)) + +## [1.544.0](https://github.com/windmill-labs/windmill/compare/v1.543.0...v1.544.0) (2025-09-18) + + +### Features + +* **ai agent:** allow multiple images input for ai agent + code cleaning ([#6591](https://github.com/windmill-labs/windmill/issues/6591)) ([3199f9f](https://github.com/windmill-labs/windmill/commit/3199f9fffd36c84d7cbb4a512935f7eb19fa5049)) +* **aichat:** add max tokens settings ([#6613](https://github.com/windmill-labs/windmill/issues/6613)) ([d837bad](https://github.com/windmill-labs/windmill/commit/d837badf2c70c483e260b099a663fdda3f1f509a)) +* allow operator to use script/flow with dynselect input ([#6616](https://github.com/windmill-labs/windmill/issues/6616)) ([e98bde6](https://github.com/windmill-labs/windmill/commit/e98bde6be6633e9a529b67672fcde5e4ce5f9129)) +* **backend:** flow streaming ([#6520](https://github.com/windmill-labs/windmill/issues/6520)) ([993baf4](https://github.com/windmill-labs/windmill/commit/993baf46bd46524b5cefe1d6ffc037e9c6ec32d2)) +* fix gcp cleanup and add ack-deadline for gcp push delivery [#6631](https://github.com/windmill-labs/windmill/issues/6631) ([4b71495](https://github.com/windmill-labs/windmill/commit/4b7149527b52ac42094af580e3788326c30a0c70)) +* **flow:** Add helper to add expression to arrays ([#6629](https://github.com/windmill-labs/windmill/issues/6629)) ([56ddad2](https://github.com/windmill-labs/windmill/commit/56ddad2d5a960c782fad89c2c86ad9cc2c4dd8cb)) +* **frontend:** allow publishing script to hub from list view ([#6634](https://github.com/windmill-labs/windmill/issues/6634)) ([39b2f54](https://github.com/windmill-labs/windmill/commit/39b2f547799a6d75012ca2595d532e87a3035e85)) +* simplify sync vs promotion mode ui in git sync settings ([#6615](https://github.com/windmill-labs/windmill/issues/6615)) ([7707bb8](https://github.com/windmill-labs/windmill/commit/7707bb8fecd85cc65a4e511f3033170a1c24fb1d)) +* update git sync script for email triggers ([#6582](https://github.com/windmill-labs/windmill/issues/6582)) ([e97c535](https://github.com/windmill-labs/windmill/commit/e97c535376177b8681fbb6b6553d6f241462f7a9)) + + +### Bug Fixes + +* add ack deadline gcp ([#6625](https://github.com/windmill-labs/windmill/issues/6625)) ([426065e](https://github.com/windmill-labs/windmill/commit/426065efee5e5e775dde403f7e7f7c78d002909a)) +* **aiagent:** fix endpoint for azure ([#6633](https://github.com/windmill-labs/windmill/issues/6633)) ([709a937](https://github.com/windmill-labs/windmill/commit/709a937ac2dab3d8a18b01f200976fa2d4625e89)) +* Don't reencrypt secrets on workspace forking ([#6622](https://github.com/windmill-labs/windmill/issues/6622)) ([9325f56](https://github.com/windmill-labs/windmill/commit/9325f5636c2e957a724bb051d41e15d2966b2899)) +* jumpcloud scim support + instance settings ui bug (nextcloud oauth) ([#6618](https://github.com/windmill-labs/windmill/issues/6618)) ([9ff4ca0](https://github.com/windmill-labs/windmill/commit/9ff4ca06629a0cf7da2996f2b22ea7915cc4705e)) + ## [1.543.0](https://github.com/windmill-labs/windmill/compare/v1.542.4...v1.543.0) (2025-09-15) diff --git a/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json b/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json new file mode 100644 index 0000000000..ef5348cf50 --- /dev/null +++ b/backend/.sqlx/query-0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT parent_job FROM v2_job WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "parent_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "0997b46bae6e2374b568e8367898d6fc79c331431326250c1a59674054ceaabd" +} diff --git a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json similarity index 83% rename from backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json rename to backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json index 141f92e271..dd40aabbc1 100644 --- a/backend/.sqlx/query-1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd.json +++ b/backend/.sqlx/query-17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17\n WHERE \n workspace_id = $12 AND \n path = $13\n ", + "query": "\n UPDATE \n gcp_trigger \n SET \n gcp_resource_path = $1,\n subscription_id = $2,\n topic_id = $3,\n delivery_type = $4,\n delivery_config = $5,\n is_flow = $6, \n edited_by = $7, \n email = $8,\n script_path = $9,\n path = $10,\n enabled = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16,\n auto_acknowledge_msg = $17,\n ack_deadline = $18\n WHERE \n workspace_id = $12 AND \n path = $13\n ", "describe": { "columns": [], "parameters": { @@ -31,10 +31,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "1a9ba16c90d3d65c4ff39aaddb3079009e03af711e7f6b53332537cf4cb0e8dd" + "hash": "17ca259e1c78e1317fdd19436e15bef428fc4f0d52776d7a5fca64f17225ef30" } diff --git a/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json b/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json new file mode 100644 index 0000000000..6ba559e644 --- /dev/null +++ b/backend/.sqlx/query-2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE v2_job_status\n SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT))\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "2e589e039986e7a2c75e328868874669b32cbe0dae6822b2d2fad0635c5f6087" +} diff --git a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json b/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json deleted file mode 100644 index e1bf471dd5..0000000000 --- a/backend/.sqlx/query-31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable \n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "value", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "is_secret", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "description", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "account", - "type_info": "Int4" - }, - { - "ordinal": 7, - "name": "is_oauth", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "expires_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - false, - true - ] - }, - "hash": "31869c5dba5cefd4ffaae7720617a40ef42ca940ca4ff7f9fb2e69e63d830d27" -} diff --git a/backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json b/backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json similarity index 61% rename from backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json rename to backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json index 82333559eb..e7d66a3d75 100644 --- a/backend/.sqlx/query-70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393.json +++ b/backend/.sqlx/query-40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n SUBSTR(rs.stream, $8) AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", + "query": "SELECT\n c.id IS NOT NULL AS completed,\n CASE\n WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END)\n ELSE false\n END AS running,\n CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs,\n SUBSTR(rs.stream, $8) AS new_result_stream,\n COALESCE(r.memory_peak, c.memory_peak) AS mem_peak,\n COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\",\n (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job,\n COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\",\n CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset,\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n created_by AS \"created_by!\",\n CASE WHEN $4::BOOLEAN THEN (\n SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc'\n ) END AS progress,\n rs.stream AS \"result_stream: Option\"\n FROM v2_job j\n LEFT JOIN v2_job_queue q USING (id)\n LEFT JOIN v2_job_runtime r USING (id)\n LEFT JOIN v2_job_status f USING (id)\n LEFT JOIN v2_job_completed c USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $3\n LEFT JOIN job_logs ON job_logs.job_id = $3\n WHERE j.workspace_id = $2 AND j.id = $3\n AND ($6::text[] IS NULL OR j.tag = ANY($6))", "describe": { "columns": [ { @@ -35,31 +35,36 @@ }, { "ordinal": 6, + "name": "stream_job", + "type_info": "Uuid" + }, + { + "ordinal": 7, "name": "workflow_as_code_status: sqlx::types::Json>", "type_info": "Jsonb" }, { - "ordinal": 7, + "ordinal": 8, "name": "log_offset", "type_info": "Int4" }, { - "ordinal": 8, + "ordinal": 9, "name": "stream_offset", "type_info": "Int4" }, { - "ordinal": 9, + "ordinal": 10, "name": "created_by!", "type_info": "Varchar" }, { - "ordinal": 10, + "ordinal": 11, "name": "progress", "type_info": "Int4" }, { - "ordinal": 11, + "ordinal": 12, "name": "result_stream: Option", "type_info": "Text" } @@ -86,10 +91,11 @@ null, null, null, + null, false, null, false ] }, - "hash": "70ddcf86865a315934843285e8ec618c47a5acbe400d79840c4a2d86f8886393" + "hash": "40999264f09a781c4393b50c2c41ae5a5e64086198cb67aba72345bb3cdf7773" } diff --git a/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json b/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json deleted file mode 100644 index f25cfd9eba..0000000000 --- a/backend/.sqlx/query-69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT result as \"result: sqlx::types::Json>\", v2_job.tag,\n v2_job_queue.running as \"running: Option\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", CHAR_LENGTH(rs.stream) AS stream_offset\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 1, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "running: Option", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "result_stream: Option", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "stream_offset", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Uuid", - "Int4" - ] - }, - "nullable": [ - true, - false, - false, - null, - null - ] - }, - "hash": "69924462c788dbc8f31aacc7f8ae588d76bf1f25d631833ce4b194818a7d1437" -} diff --git a/backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json b/backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json similarity index 53% rename from backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json rename to backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json index 76c37b7a08..7254b7e84f 100644 --- a/backend/.sqlx/query-d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d.json +++ b/backend/.sqlx/query-80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", + "query": "SELECT\n COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\",\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $2::uuid as job_id, $1::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $2", "describe": { "columns": [ { @@ -17,20 +17,33 @@ "ordinal": 2, "name": "stream_offset", "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "flow_status: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" } ], "parameters": { "Left": [ "Text", "Uuid", - "Int4" + "Int4", + "Bool" ] }, "nullable": [ + null, + null, null, null, null ] }, - "hash": "d8c209b177da2e147a3549c888969478cd80aa157700e5c1f3b9f4d12dd31a1d" + "hash": "80809d397cf84f7278ebb276078871b371663257a127eb35512695c487066fd7" } diff --git a/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json b/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json new file mode 100644 index 0000000000..abc63b0e90 --- /dev/null +++ b/backend/.sqlx/query-8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34.json @@ -0,0 +1,55 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n jc.result as \"result: sqlx::types::Json>\",\n v2_job.tag,\n v2_job_queue.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM v2_job\n LEFT JOIN v2_job_queue USING (id)\n LEFT JOIN v2_job_completed jc USING (id)\n LEFT JOIN v2_job_status js USING (id)\n LEFT JOIN job_result_stream rs ON rs.job_id = $2\n WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result: sqlx::types::Json>", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "running: Option", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "result_stream: Option", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "stream_offset", + "type_info": "Int4" + }, + { + "ordinal": 5, + "name": "stream_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid", + "Int4", + "Bool" + ] + }, + "nullable": [ + true, + false, + false, + null, + null, + null + ] + }, + "hash": "8126b118704341846e88bd289f1afe83c07b7a8b422f48022994370b3e433f34" +} diff --git a/backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json b/backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json similarity index 51% rename from backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json rename to backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json index 2f14e723e4..4cca74dd4b 100644 --- a/backend/.sqlx/query-6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d.json +++ b/backend/.sqlx/query-a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", + "query": "SELECT\n COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\",\n jq.running as \"running: Option\",\n SUBSTR(rs.stream, $3) AS \"result_stream: Option\",\n CHAR_LENGTH(rs.stream) + 1 AS stream_offset,\n CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job\n FROM (\n SELECT $1::uuid as job_id, $2::text as workspace_id\n ) base\n LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id\n LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id\n LEFT JOIN v2_job_status js ON js.id = base.job_id\n LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id\n WHERE base.job_id = $1", "describe": { "columns": [ { @@ -22,21 +22,28 @@ "ordinal": 3, "name": "stream_offset", "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "stream_job", + "type_info": "Uuid" } ], "parameters": { "Left": [ "Uuid", "Text", - "Int4" + "Int4", + "Bool" ] }, "nullable": [ null, false, null, + null, null ] }, - "hash": "6907eb134dc5dbf118387e073897f86574c92de16252b2b1c475ab8146e5343d" + "hash": "a58a345f7082181f89e7f88929b7149791de48bc2e489edb55d63f67702cce05" } diff --git a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json b/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json deleted file mode 100644 index 63301d439c..0000000000 --- a/backend/.sqlx/query-a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Jsonb", - "Int4", - "Bool", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "a6d1b80e1b407610987c98521f8e36dc8e96a63c4690721ae0bc169a3d83aff1" -} diff --git a/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json new file mode 100644 index 0000000000..9fb02ee9e8 --- /dev/null +++ b/backend/.sqlx/query-bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO workspace_key (workspace_id, kind, key)\n SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "bd5d39d1ef26ac0526a2ae834b45bcf902d143eec8faec306c651e74ad14c68e" +} diff --git a/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json new file mode 100644 index 0000000000..a0d0a5dfcb --- /dev/null +++ b/backend/.sqlx/query-c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at)\n SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at\n FROM variable\n WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c4121e4d1de409f66f1984ad2a962dd86568e89f278b93320911f8ae5475f038" +} diff --git a/backend/.sqlx/query-d8ef35b4990eb9b2a306494d5b9acde9f57cfaba047c5b7680e0dccd2c1507df.json b/backend/.sqlx/query-d8ef35b4990eb9b2a306494d5b9acde9f57cfaba047c5b7680e0dccd2c1507df.json new file mode 100644 index 0000000000..43ce3fada4 --- /dev/null +++ b/backend/.sqlx/query-d8ef35b4990eb9b2a306494d5b9acde9f57cfaba047c5b7680e0dccd2c1507df.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT \n (flow_status->'step')::integer as step,\n jsonb_array_length(flow_status->'modules') as len,\n flow_status->'modules'->-1->>'branch_chosen' IS NOT NULL as is_branch_one,\n parent_job as ppp_job\n FROM v2_job \n LEFT JOIN v2_job_status USING (id)\n WHERE v2_job.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "step", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "len", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "is_branch_one", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "ppp_job", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null, + null, + null, + true + ] + }, + "hash": "d8ef35b4990eb9b2a306494d5b9acde9f57cfaba047c5b7680e0dccd2c1507df" +} diff --git a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json similarity index 67% rename from backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json rename to backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json index 2874c70c1c..6d41284328 100644 --- a/backend/.sqlx/query-e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0.json +++ b/backend/.sqlx/query-e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16\n )", + "query": "\n INSERT INTO gcp_trigger (\n gcp_resource_path,\n subscription_id,\n topic_id,\n delivery_type,\n delivery_config,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry,\n auto_acknowledge_msg,\n ack_deadline\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4,\n $5,\n $6, \n $7, \n $8, \n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16,\n $17\n )", "describe": { "columns": [], "parameters": { @@ -30,10 +30,11 @@ "Varchar", "Jsonb", "Jsonb", - "Bool" + "Bool", + "Int4" ] }, "nullable": [] }, - "hash": "e619fa013528a6275f98e14ae1727c55b0d4f4a5e4ee87c29251042e2916f0a0" + "hash": "e7c61bbdcf882f6e1e9b11df03e7a2ee318c72b365c0fe0b9fbec886a461f5e4" } diff --git a/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json b/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json new file mode 100644 index 0000000000..5afdd9726a --- /dev/null +++ b/backend/.sqlx/query-f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "new_result_stream", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "stream_offset", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Uuid" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "f17f914d2522bf7cb5de9d7ba5557ee0dce940039ab42fd39bf079d87b6cad8a" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 005bec2f26..3e064ffbd5 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -620,7 +620,7 @@ dependencies = [ "thiserror 1.0.69", "time", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tokio-util", "tokio-websockets", "tracing", @@ -3613,7 +3613,7 @@ dependencies = [ "serde_json", "thiserror 2.0.16", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tokio-socks", "tokio-util", "tower 0.5.2", @@ -5879,7 +5879,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "wasi 0.14.6+wasi-0.2.4", + "wasi 0.14.7+wasi-0.2.4", "wasm-bindgen", ] @@ -6608,7 +6608,7 @@ dependencies = [ "pin-project-lite", "rustls-native-certs 0.7.3", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tower-service", ] @@ -6657,7 +6657,7 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tower-service", "webpki-roots 1.0.2", ] @@ -7176,9 +7176,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.78" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" +checksum = "852f13bec5eba4ba9afbeb93fd7c13fe56147f055939ae21c43a29a0ecb2702e" dependencies = [ "once_cell", "wasm-bindgen", @@ -7186,9 +7186,9 @@ dependencies = [ [[package]] name = "json-patch" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159294d661a039f7644cea7e4d844e6b25aaf71c1ffe9d73a96d768c24b0faf4" +checksum = "f300e415e2134745ef75f04562dd0145405c2f7fd92065db029ac4b16b57fe90" dependencies = [ "jsonptr", "serde", @@ -7910,7 +7910,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "webpki-roots 0.26.11", ] @@ -9911,11 +9911,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.22.27", + "toml_edit 0.23.4", ] [[package]] @@ -10719,7 +10719,7 @@ dependencies = [ "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tokio-util", "tower 0.5.2", "tower-http", @@ -13738,9 +13738,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ "rustls 0.23.29", "tokio", @@ -13832,7 +13832,7 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tokio-util", ] @@ -13844,7 +13844,7 @@ checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" dependencies = [ "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "toml_edit 0.19.15", ] @@ -13857,6 +13857,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + [[package]] name = "toml_edit" version = "0.19.15" @@ -13866,18 +13875,28 @@ dependencies = [ "indexmap 2.11.1", "serde", "serde_spanned", - "toml_datetime", + "toml_datetime 0.6.11", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.27" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "7211ff1b8f0d3adae1663b7da9ffe396eabe1ca25f0b0bee42b0da29a9ddce93" dependencies = [ "indexmap 2.11.1", - "toml_datetime", + "toml_datetime 0.7.0", + "toml_parser", + "winnow 0.7.13", +] + +[[package]] +name = "toml_parser" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +dependencies = [ "winnow 0.7.13", ] @@ -13907,7 +13926,7 @@ dependencies = [ "rustls-pemfile 2.2.0", "socket2 0.5.10", "tokio", - "tokio-rustls 0.26.2", + "tokio-rustls 0.26.3", "tokio-stream", "tower 0.4.13", "tower-layer", @@ -14740,9 +14759,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.6+wasi-0.2.4" +version = "0.14.7+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f71243a3f320c00a8459e455c046ce571229c2f31fd11645d9dc095e3068ca0" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" dependencies = [ "wasip2", ] @@ -14764,9 +14783,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" +checksum = "ab10a69fbd0a177f5f649ad4d8d3305499c42bab9aef2f7ff592d0ec8f833819" dependencies = [ "cfg-if", "once_cell", @@ -14777,9 +14796,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" +checksum = "0bb702423545a6007bbc368fde243ba47ca275e549c8a28617f56f6ba53b1d1c" dependencies = [ "bumpalo", "log", @@ -14791,9 +14810,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.51" +version = "0.4.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" +checksum = "a0b221ff421256839509adbb55998214a70d829d3a28c69b4a6672e9d2a42f67" dependencies = [ "cfg-if", "js-sys", @@ -14804,9 +14823,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" +checksum = "fc65f4f411d91494355917b605e1480033152658d71f722a90647f56a70c88a0" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -14814,9 +14833,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" +checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", @@ -14827,18 +14846,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.101" +version = "0.2.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" +checksum = "293c37f4efa430ca14db3721dfbe48d8c33308096bd44d80ebaa775ab71ba1cf" dependencies = [ "unicode-ident", ] [[package]] name = "wasm-bindgen-test" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80cc7f8a4114fdaa0c58383caf973fc126cf004eba25c9dc639bccd3880d55ad" +checksum = "aee0a0f5343de9221a0d233b04520ed8dc2e6728dce180b1dcd9288ec9d9fa3c" dependencies = [ "js-sys", "minicov", @@ -14849,9 +14868,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ada2ab788d46d4bda04c9d567702a79c8ced14f51f221646a16ed39d0e6a5d" +checksum = "a369369e4360c2884c3168d22bded735c43cccae97bbc147586d4b480edd138d" dependencies = [ "proc-macro2", "quote", @@ -14898,9 +14917,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.78" +version = "0.3.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" dependencies = [ "js-sys", "wasm-bindgen", @@ -15109,7 +15128,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "aws-sdk-config", @@ -15169,7 +15188,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "argon2", @@ -15289,7 +15308,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.543.0" +version = "1.544.1" dependencies = [ "base64 0.22.1", "chrono", @@ -15304,7 +15323,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.543.0" +version = "1.544.1" dependencies = [ "chrono", "serde", @@ -15317,7 +15336,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "axum", @@ -15336,7 +15355,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15420,7 +15439,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.543.0" +version = "1.544.1" dependencies = [ "regex", "serde", @@ -15435,7 +15454,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "bytes", @@ -15459,7 +15478,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.543.0" +version = "1.544.1" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15471,7 +15490,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.543.0" +version = "1.544.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -15480,7 +15499,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15492,7 +15511,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15504,7 +15523,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "gosyn", @@ -15516,7 +15535,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15528,7 +15547,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15540,7 +15559,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "nu-parser", @@ -15551,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15562,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15574,7 +15593,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15597,7 +15616,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15611,7 +15630,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15628,7 +15647,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15642,7 +15661,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "lazy_static", @@ -15660,7 +15679,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "getrandom 0.2.16", @@ -15685,7 +15704,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "serde_json", @@ -15695,7 +15714,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "async-recursion", @@ -15728,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.543.0" +version = "1.544.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15738,12 +15757,13 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.543.0" +version = "1.544.1" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", + "async-trait", "backon", "base64 0.22.1", "bit-vec 0.6.3", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7191214ef8..290c3389b7 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.543.0" +version = "1.544.1" authors.workspace = true edition.workspace = true @@ -34,7 +34,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.543.0" +version = "1.544.1" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index d9e0f1c4b1..7d8849c19a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -08f610ecc5486ad2568e0abd22405baa90a93481 +daa1c26c268c778e55756f02a459b6c7628c9267 diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql new file mode 100644 index 0000000000..144aab482d --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE gcp_trigger DROP COLUMN ack_deadline; \ No newline at end of file diff --git a/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql new file mode 100644 index 0000000000..7a85bd7c82 --- /dev/null +++ b/backend/migrations/20250917172504_add_ack_deadline_to_gcp_trigger.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE gcp_trigger ADD COLUMN ack_deadline INTEGER; \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0e05dfbe9b..11dcb6c220 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.543.0 + version: 1.544.1 title: Windmill API contact: @@ -1306,8 +1306,6 @@ paths: type: string endpoint_sync: type: string - endpoint_openai_sync: - type: string summary: type: string description: @@ -1318,7 +1316,6 @@ paths: - workspace - endpoint_async - endpoint_sync - - endpoint_openai_sync - summary - kind @@ -5606,35 +5603,6 @@ paths: type: string format: uuid - /w/{workspace}/jobs/openai_sync/p/{path}: - post: - summary: run script by path in openai format - operationId: openaiSyncScriptByPath - tags: - - job - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/ScriptPath" - - $ref: "#/components/parameters/ParentJob" - - $ref: "#/components/parameters/NewJobId" - - $ref: "#/components/parameters/IncludeHeader" - - $ref: "#/components/parameters/QueueLimit" - - requestBody: - description: script args - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ScriptArgs" - - responses: - "200": - description: job result - content: - application/json: - schema: {} - /w/{workspace}/jobs/run_wait_result/p/{path}: post: summary: run script by path @@ -5650,6 +5618,7 @@ paths: - $ref: "#/components/parameters/NewJobId" - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/SkipPreprocessor" requestBody: description: script args @@ -5681,34 +5650,7 @@ paths: - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" - $ref: "#/components/parameters/Payload" - - responses: - "200": - description: job result - content: - application/json: - schema: {} - - /w/{workspace}/jobs/openai_sync/f/{path}: - post: - summary: run flow by path and wait until completion in openai format - operationId: openaiSyncFlowByPath - tags: - - job - parameters: - - $ref: "#/components/parameters/WorkspaceId" - - $ref: "#/components/parameters/ScriptPath" - - $ref: "#/components/parameters/IncludeHeader" - - $ref: "#/components/parameters/QueueLimit" - - $ref: "#/components/parameters/NewJobId" - - requestBody: - description: script args - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ScriptArgs" + - $ref: "#/components/parameters/SkipPreprocessor" responses: "200": @@ -5729,6 +5671,7 @@ paths: - $ref: "#/components/parameters/IncludeHeader" - $ref: "#/components/parameters/QueueLimit" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" requestBody: description: script args @@ -7114,11 +7057,7 @@ paths: in: query schema: type: integer - - name: skip_preprocessor - description: skip the preprocessor - in: query - schema: - type: boolean + - $ref: "#/components/parameters/SkipPreprocessor" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/NewJobId" @@ -7276,11 +7215,7 @@ paths: in: query schema: type: integer - - name: skip_preprocessor - description: skip the preprocessor - in: query - schema: - type: boolean + - $ref: "#/components/parameters/SkipPreprocessor" - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -14137,6 +14072,12 @@ components: in: query schema: type: string + SkipPreprocessor: + name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean Payload: name: payload description: | @@ -14514,6 +14455,12 @@ components: type: object additionalProperties: type: string + max_tokens_per_model: + type: object + additionalProperties: + type: integer + minimum: 1 + maximum: 2000000 Alert: type: object @@ -16818,6 +16765,12 @@ components: type: boolean auto_acknowledge_msg: type: boolean + ack_deadline: + type: integer + format: int32 + minimum: 10 + maximum: 600 + description: "Time in seconds within which the message must be acknowledged. If not provided, defaults to the subscription's acknowledgment deadline (600 seconds)." error_handler_path: type: string error_handler_args: diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 9f13d32cf5..3c3ba77bb5 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -10,8 +10,8 @@ use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; -use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel, AZURE_API_VERSION}; use windmill_common::error::{to_anyhow, Error, Result}; lazy_static::lazy_static! { @@ -25,9 +25,6 @@ lazy_static::lazy_static! { pub static ref AI_REQUEST_CACHE: Cache<(String, AIProvider), ExpiringAIRequestConfig> = Cache::new(500); } -const AZURE_API_VERSION: &str = "2025-04-01-preview"; -const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; - #[derive(Deserialize, Debug)] struct AIOAuthResource { client_id: String, @@ -154,21 +151,13 @@ impl AIRequestConfig { let base_url = self.base_url.trim_end_matches('/'); - let is_azure = matches!(provider, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL - || matches!(provider, AIProvider::AzureOpenAI); + let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); let url = if is_azure && method != Method::GET { - if base_url.ends_with("/deployments") { - let model = Self::get_azure_model(&body)?; - format!("{}/{}/{}", base_url, model, path) - } else if base_url.ends_with("/openai") { - let model = Self::get_azure_model(&body)?; - format!("{}/deployments/{}/{}", base_url, model, path) - } else { - format!("{}/{}", base_url, path) - } + let model = AIProvider::extract_model_from_body(&body)?; + AIProvider::build_azure_openai_url(base_url, &model, path) } else if is_anthropic_sdk { let truncated_base_url = base_url.trim_end_matches("/v1"); format!("{}/{}", truncated_base_url, path) @@ -233,18 +222,6 @@ impl AIRequestConfig { .map_err(|e| Error::internal_err(format!("Failed to reserialize request body: {}", e)))? .into()) } - - fn get_azure_model(body: &Bytes) -> Result { - #[derive(Deserialize, Debug)] - struct AzureModel { - model: String, - } - - let azure_model: AzureModel = serde_json::from_slice(body) - .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; - - Ok(azure_model.model) - } } #[derive(Clone, Debug)] @@ -272,6 +249,8 @@ pub struct AIConfig { pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens_per_model: Option>, } pub fn global_service() -> Router { diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 5f66a0deaf..00148a2961 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -415,6 +415,7 @@ pub struct Tokened { pub token: String, } +#[derive(Clone, Debug)] pub struct OptTokened { #[allow(dead_code)] pub token: Option, diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 603c040475..3283dea123 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -183,6 +183,7 @@ pub struct GcpTriggerConfig { pub create_update: Option, pub topic_id: String, pub auto_acknowledge_msg: Option, + pub ack_deadline: Option, } #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] @@ -397,6 +398,7 @@ async fn set_gcp_trigger_config( gcp_config.create_update, false, capture_config.is_flow, + gcp_config.ack_deadline ) .await?; gcp_config.create_update = Some(config); diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b8d51c3020..b00a08cdb5 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -44,6 +44,7 @@ use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH; use windmill_common::variables::get_workspace_key; +use crate::triggers::trigger_helpers::ScriptId; use crate::{ add_webhook_allowed_origin, args::{self, RawWebhookArgs}, @@ -171,6 +172,27 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run_and_stream/f/*script_path", + get(stream_flow_by_path) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) + .route( + "/run_and_stream/p/*script_path", + get(stream_script_by_path) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) + .route( + "/run_and_stream/h/:hash", + get(stream_script_by_hash) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run/h/:hash", post(run_job_by_hash) @@ -5145,6 +5167,174 @@ pub async fn run_wait_result_flow_by_path( .await } +pub async fn stream_flow_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, flow_path)): Path<(String, StripPath)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_flow_path(flow_path.to_path()), + args, + run_query, + ) + .await +} + +pub async fn stream_script_by_path( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, script_path)): Path<(String, StripPath)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_script_path(script_path.to_path()), + args, + run_query, + ) + .await +} + +pub async fn stream_script_by_hash( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, script_hash)): Path<(String, ScriptHash)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_script_hash(script_hash), + args, + run_query, + ) + .await +} + +pub async fn stream_job( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + w_id: String, + runnable_id: RunnableId, + args: RawWebhookArgs, + run_query: RunJobQuery, +) -> error::Result { + let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))) + }); + + let payload_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_args); + + let args = args + .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) + .await?; + + let uuid = match runnable_id { + RunnableId::ScriptId(ScriptId::ScriptPath(script_path)) + | RunnableId::HubScript(script_path) => { + run_script_by_path_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + StripPath(script_path), + run_query, + args, + ) + .await? + .0 + } + RunnableId::ScriptId(ScriptId::ScriptHash(script_hash)) => { + run_job_by_hash_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + script_hash, + run_query, + args, + ) + .await? + .0 + } + RunnableId::FlowPath(flow_path) => { + run_flow_by_path_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + StripPath(flow_path), + run_query, + args, + ) + .await? + .0 + } + }; + + let opt_authed = Some(authed.clone()); + let opt_tokened = OptTokened { token: None }; // ignored when authed is some + let (tx, rx) = tokio::sync::mpsc::channel(32); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { + format!( + "data: {}\n\n", + serde_json::to_string(&x).unwrap_or_default() + ) + }); + + start_job_update_sse_stream( + opt_authed, + opt_tokened, + db, + w_id, + uuid, + None, + None, + None, + None, + Some(true), + Some(true), + None, + None, + tx, + ); + + let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); + + Ok(Response::builder() + .status(200) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .header("Connection", "keep-alive") + .body(body) + .unwrap()) +} + pub async fn run_wait_result_flow_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, @@ -6303,6 +6493,7 @@ pub struct JobUpdateQuery { pub no_logs: Option, pub only_result: Option, pub fast: Option, + pub is_flow: Option, } #[derive(Serialize, Debug)] @@ -6331,6 +6522,8 @@ pub struct JobUpdate { pub job: Option, #[serde(skip_serializing_if = "Option::is_none")] pub only_result: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub flow_stream_job_id: Option, } impl JobUpdate { @@ -6349,6 +6542,7 @@ impl Hash for JobUpdate { self.mem_peak.hash(state); self.progress.hash(state); self.stream_offset.hash(state); + self.flow_stream_job_id.hash(state); if !self.completed.unwrap_or(false) { self.flow_status.as_ref().map(|x| x.get().hash(state)); self.workflow_as_code_status @@ -6421,6 +6615,7 @@ async fn get_job_update( running, only_result, no_logs, + is_flow, .. }): Query, ) -> JsonResult { @@ -6439,6 +6634,8 @@ async fn get_job_update( false, only_result, no_logs, + is_flow, + None, ) .await?, )) @@ -6457,9 +6654,12 @@ async fn get_job_update_sse( no_logs, only_result, fast, + is_flow, }): Query, -) -> Response { - let stream = get_job_update_sse_stream( +) -> error::Result { + let (tx, rx) = tokio::sync::mpsc::channel(32); + + start_job_update_sse_stream( opt_authed, opt_tokened, db, @@ -6472,8 +6672,11 @@ async fn get_job_update_sse( only_result, fast, no_logs, - ) - .map(|x| { + is_flow, + tx, + ); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { format!( "data: {}\n\n", serde_json::to_string(&x).unwrap_or_default() @@ -6482,26 +6685,26 @@ async fn get_job_update_sse( let body = axum::body::Body::from_stream(stream.map(Result::<_, std::convert::Infallible>::Ok)); - Response::builder() + Ok(Response::builder() .status(200) .header("Content-Type", "text/event-stream") .header("Cache-Control", "no-cache") .header("Connection", "keep-alive") .body(body) - .unwrap() + .unwrap()) } #[derive(Serialize)] #[serde(tag = "type", rename_all = "lowercase")] enum JobUpdateSSEStream { Update(JobUpdate), - Error(String), + Error { error: String }, NotFound, Timeout, Ping, } -fn get_job_update_sse_stream( +fn start_job_update_sse_stream( opt_authed: Option, opt_tokened: OptTokened, db: DB, @@ -6514,18 +6717,18 @@ fn get_job_update_sse_stream( only_result: Option, fast: Option, no_logs: Option, -) -> impl futures::Stream { - let (tx, rx) = tokio::sync::mpsc::channel(32); - + is_flow: Option, + tx: tokio::sync::mpsc::Sender, +) -> () { tokio::spawn(async move { let mut log_offset = initial_log_offset; let mut stream_offset = initial_stream_offset; let mut last_update_hash: Option = None; + let mut flow_stream_job_id = None; // Send initial update immediately let mut running = running; let mut mem_peak = 0; - match get_job_update_data( &opt_authed, &opt_tokened, @@ -6540,6 +6743,8 @@ fn get_job_update_sse_stream( true, only_result, no_logs, + is_flow, + flow_stream_job_id, ) .await { @@ -6566,6 +6771,9 @@ fn get_job_update_sse_stream( update.stream_offset = None; } } + if update.flow_stream_job_id.is_some() { + flow_stream_job_id = update.flow_stream_job_id; + } if tx.send(JobUpdateSSEStream::Update(update)).await.is_err() { tracing::warn!("Failed to send initial job update for job {job_id}"); return; @@ -6576,7 +6784,7 @@ fn get_job_update_sse_stream( } Err(e) => { if tx - .send(JobUpdateSSEStream::Error(e.to_string())) + .send(JobUpdateSSEStream::Error { error: e.to_string() }) .await .is_err() { @@ -6630,6 +6838,8 @@ fn get_job_update_sse_stream( true, only_result, no_logs, + is_flow, + flow_stream_job_id, ) .await { @@ -6665,6 +6875,13 @@ fn get_job_update_sse_stream( update.stream_offset = None; } } + if update.flow_stream_job_id.is_some() { + if flow_stream_job_id.is_none() { + flow_stream_job_id = update.flow_stream_job_id; + } else { + update.flow_stream_job_id = None; + } + } if let Some(new_mem_peak) = update.mem_peak { if new_mem_peak != mem_peak { mem_peak = new_mem_peak; @@ -6693,8 +6910,29 @@ fn get_job_update_sse_stream( } } }); +} - tokio_stream::wrappers::ReceiverStream::new(rx) +async fn get_flow_stream_delta( + db: &DB, + flow_stream_job_id: Option, + stream_offset: Option, +) -> error::Result, Option)>> { + if let Some(job_id) = flow_stream_job_id { + let record = sqlx::query!( + "SELECT SUBSTR(rs.stream, $1) AS new_result_stream, CHAR_LENGTH(rs.stream) + 1 AS stream_offset FROM job_result_stream rs WHERE rs.job_id = $2", + stream_offset.unwrap_or(0), + job_id, + ) + .fetch_optional(db) + .await?; + if let Some(record) = record { + Ok(Some((record.new_result_stream, record.stream_offset))) + } else { + Ok(None) + } + } else { + Ok(None) + } } async fn get_job_update_data( @@ -6711,6 +6949,8 @@ async fn get_job_update_data( get_full_job_on_completion: bool, only_result: Option, no_logs: Option, + is_flow: Option, + flow_stream_job_id: Option, ) -> error::Result { let tags = if log_view { log_job_view( @@ -6729,115 +6969,146 @@ async fn get_job_update_data( None }; + let ignore_flow_stream_job_id = is_flow.is_some_and(|x| !x) || flow_stream_job_id.is_some(); + if only_result.unwrap_or(false) { - let result = if let Some(tags) = tags { - let r = - sqlx::query!( - "SELECT result as \"result: sqlx::types::Json>\", v2_job.tag, - v2_job_queue.running as \"running: Option\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", CHAR_LENGTH(rs.stream) AS stream_offset + let (result, running, mut result_stream, mut new_stream_offset, new_flow_stream_job_id) = + if let Some(tags) = tags { + let r = sqlx::query!( + "SELECT + jc.result as \"result: sqlx::types::Json>\", + v2_job.tag, + v2_job_queue.running as \"running: Option\", + SUBSTR(rs.stream, $3) AS \"result_stream: Option\", + CHAR_LENGTH(rs.stream) AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM v2_job LEFT JOIN v2_job_queue USING (id) - LEFT JOIN v2_job_completed USING (id) + LEFT JOIN v2_job_completed jc USING (id) + LEFT JOIN v2_job_status js USING (id) LEFT JOIN job_result_stream rs ON rs.job_id = $2 WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - w_id, - job_id, - stream_offset.unwrap_or(0), - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; - - if !tags.contains(&r.tag.as_str()) { - return Err(Error::NotAuthorized(format!( - "Job tag {} is not in the scope tags: {}", - r.tag, - tags.join(", ") - ))); - } - let running = r.running.as_ref().map(|x| *x); - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, + w_id, + job_id, + stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) - } else { - if running.is_some_and(|x| !x) { - let r = sqlx::query!( + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; + + if !tags.contains(&r.tag.as_str()) { + return Err(Error::NotAuthorized(format!( + "Job tag {} is not in the scope tags: {}", + r.tag, + tags.join(", ") + ))); + } + let running = r.running.as_ref().map(|x| *x); + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + if running.is_some_and(|x| !x) { + let r = sqlx::query!( "SELECT COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\", jq.running as \"running: Option\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset + CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( SELECT $1::uuid as job_id, $2::text as workspace_id ) base LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id + LEFT JOIN v2_job_status js ON js.id = base.job_id LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $1", job_id, w_id, stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ).fetch_optional(db).await?; - if let Some(r) = r { - let running = r.running.as_ref().map(|x| *x); - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, - ) + if let Some(r) = r { + let running = r.running.as_ref().map(|x| *x); + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + (None, None, None, None, None) + } } else { - (None, None, None, None) - } - } else { - let q = sqlx::query!( + let q = sqlx::query!( "SELECT COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\", SUBSTR(rs.stream, $3) AS \"result_stream: Option\", - CHAR_LENGTH(rs.stream) + 1 AS stream_offset + CHAR_LENGTH(rs.stream) + 1 AS stream_offset, + COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\", + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job FROM ( SELECT $2::uuid as job_id, $1::text as workspace_id ) base LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id + LEFT JOIN v2_job_status js ON js.id = base.job_id LEFT JOIN job_result_stream rs ON rs.job_id = base.job_id WHERE base.job_id = $2", w_id, job_id, stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) .fetch_optional(db) .await?; - if let Some(r) = q { - ( - r.result.map(|x| x.0), - running, - r.result_stream.flatten(), - r.stream_offset, - ) - } else { - (None, None, None, None) + if let Some(r) = q { + ( + r.result.map(|x| x.0), + running, + r.result_stream.flatten(), + r.stream_offset, + r.stream_job, + ) + } else { + (None, None, None, None, None) + } } - } - }; + }; + + let flow_stream_job_id = flow_stream_job_id.or(new_flow_stream_job_id); + + let flow_stream_delta = + get_flow_stream_delta(db, flow_stream_job_id, stream_offset).await?; + + if let Some((flow_result_stream, flow_stream_offset)) = flow_stream_delta { + result_stream = flow_result_stream; + new_stream_offset = flow_stream_offset; + } + Ok(JobUpdate { - running: result.1, - completed: if result.0.is_some() { Some(true) } else { None }, + running, + completed: if result.is_some() { Some(true) } else { None }, log_offset: None, new_logs: None, - new_result_stream: result.2, - stream_offset: result.3, + new_result_stream: result_stream, + stream_offset: new_stream_offset, mem_peak: None, progress: None, job: None, flow_status: None, workflow_as_code_status: None, - only_result: result.0, + only_result: result, + flow_stream_job_id, }) } else { - let record = sqlx::query!( + let mut record = sqlx::query!( "SELECT c.id IS NOT NULL AS completed, CASE @@ -6848,6 +7119,7 @@ async fn get_job_update_data( SUBSTR(rs.stream, $8) AS new_result_stream, COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\", + (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job, COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\", CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset, CHAR_LENGTH(rs.stream) + 1 AS stream_offset, @@ -6891,6 +7163,16 @@ async fn get_job_update_data( None }; + let flow_stream_job_id = flow_stream_job_id.or(record.stream_job); + + let flow_stream_delta = + get_flow_stream_delta(db, flow_stream_job_id, stream_offset).await?; + + if let Some((new_result_stream, stream_offset)) = flow_stream_delta { + record.new_result_stream = new_result_stream; + record.stream_offset = stream_offset; + } + Ok(JobUpdate { running: record.running, completed: record.completed, @@ -6908,6 +7190,7 @@ async fn get_job_update_data( .flow_status .map(|x: sqlx::types::Json>| x.0), only_result: None, + flow_stream_job_id, }) } } diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api/src/scopes.rs index ef20131659..7984e4b627 100644 --- a/backend/windmill-api/src/scopes.rs +++ b/backend/windmill-api/src/scopes.rs @@ -484,21 +484,24 @@ pub fn check_route_access( ))) } -const SCRIPT_JOBS: [&'static str; 6] = [ +const SCRIPT_JOBS: [&'static str; 8] = [ "jobs/run/p", "jobs/run/h", "jobs/run_wait_result/p", "jobs/run_wait_result/h", "jobs/run/preview_bundle", "jobs/run/preview", + "jobs/run_and_stream/p", + "jobs/run_and_stream/h", ]; -const FLOW_JOBS: [&'static str; 5] = [ +const FLOW_JOBS: [&'static str; 6] = [ "jobs/run/f", "jobs/run_wait_result/f", "jobs/run/preview_flow", "jobs/restart/f", "jobs/flow/resume", + "jobs/run_and_stream/f", ]; lazy_static::lazy_static! { diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index d0ca14e9eb..aefd255068 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -32,6 +32,7 @@ use windmill_common::{ pub trait Listener: TriggerCrud + TriggerJobArgs { type Consumer: Send; type Extra: Send + Sync; + type ExtraState: Send + Sync; //to use in next PR to add job trigger kind to eow #[allow(unused)] @@ -53,6 +54,7 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { listening_trigger: &ListeningTrigger, err_message: Arc>>, killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, ); async fn fetch_enabled_unlistened_triggers( &self, @@ -159,10 +161,15 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { Ok(captures) } + async fn get_extra_state(&self) -> Option { + None + } + async fn cleanup( &self, _db: &DB, _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, ) -> Result<()> { Ok(()) } @@ -568,14 +575,14 @@ async fn listening( let killpill_rx_get_consumer = killpill_rx.resubscribe(); let loop_ping_status = Arc::new(RwLock::new(None)); - + let extra_state = listener.get_extra_state().await; tokio::select! { biased; _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; } _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; } consumer = { listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) @@ -583,18 +590,18 @@ async fn listening( tokio::select! { biased; _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } _ = async { match consumer { Ok(Some(consumer)) => { listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; - let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer).await; + let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; tracing::debug!("Stopping consumer for trigger"); } Err(error) => { @@ -604,7 +611,7 @@ async fn listening( _ => {} } } => { - let _ = listener.cleanup(&db, &listening_trigger).await; + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; return; } } diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-api/src/triggers/mqtt/listener.rs index 34af0b5eb1..af2d94b113 100644 --- a/backend/windmill-api/src/triggers/mqtt/listener.rs +++ b/backend/windmill-api/src/triggers/mqtt/listener.rs @@ -36,6 +36,7 @@ use crate::{ impl Listener for MqttTrigger { type Consumer = MqttClientResult; type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; async fn get_consumer( @@ -100,6 +101,7 @@ impl Listener for MqttTrigger { listening_trigger: &ListeningTrigger, _err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { tracing::info!( "Starting to listen for MQTT trigger {}", diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-api/src/triggers/postgres/listener.rs index 7bdfab0ed2..bfb527e3e7 100644 --- a/backend/windmill-api/src/triggers/postgres/listener.rs +++ b/backend/windmill-api/src/triggers/postgres/listener.rs @@ -133,6 +133,7 @@ impl PostgresSimpleClient { impl Listener for PostgresTrigger { type Consumer = (CopyBothDuplex, LogicalReplicationSettings); type Extra = (); + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; async fn get_consumer( @@ -209,6 +210,7 @@ impl Listener for PostgresTrigger { listening_trigger: &ListeningTrigger, err_message: Arc>>, _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { let (logical_replication_stream, logical_replication_settings) = consumer; pin_mut!(logical_replication_stream); @@ -403,6 +405,7 @@ impl Listener for PostgresTrigger { &self, db: &DB, listening_trigger: &ListeningTrigger, + _extra_state: Option<&Self::ExtraState>, ) -> Result<()> { let authed = listening_trigger .authed(db, &Self::TRIGGER_KIND.to_string()) diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs index 80eee431c4..7fb22f2b08 100644 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -164,6 +164,7 @@ impl Listener for WebsocketTrigger { Response>>, ); type Extra = ReturnMessageChannels; + type ExtraState = (); const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; async fn get_consumer( &self, @@ -214,6 +215,7 @@ impl Listener for WebsocketTrigger { listening_trigger: &ListeningTrigger, err_message: Arc>>, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, ) { let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 6ded25f923..949d563ac6 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -2131,7 +2131,6 @@ struct Runnable { workspace: String, endpoint_async: String, endpoint_sync: String, - endpoint_openai_sync: String, summary: String, description: String, schema: Option, @@ -2183,10 +2182,6 @@ async fn get_all_runnables( "/w/{}/jobs/run_wait_result/f/{}", &f.workspace, &f.path ), - endpoint_openai_sync: format!( - "/w/{}/jobs/openai_sync/f/{}", - &f.workspace, &f.path - ), summary: f.summary, description: f.description, schema: f.schema, @@ -2212,10 +2207,6 @@ async fn get_all_runnables( "/w/{}/jobs/run_wait_result/p/{}", &s.workspace, &s.path ), - endpoint_openai_sync: format!( - "/w/{}/jobs/openai_sync/p/{}", - &s.workspace, &s.path - ), summary: s.summary, description: s.description, schema: s.schema, diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index cca13a7c25..30cc9c0df3 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -210,7 +210,12 @@ async fn get_variable( } else if !value.is_empty() && decrypt_secret { let _ = tx.commit().await; let mc = build_crypt(&db, &w_id).await?; - Some(decrypt(&mc, value)?) + Some(decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })?) } else if q.include_encrypted.unwrap_or(false) { Some(value) } else { @@ -837,7 +842,12 @@ pub async fn get_value_internal<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres return Err(Error::internal_err("Require oauth2 feature".to_string())); } else if !value.is_empty() { let mc = build_crypt(&db, &w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } @@ -873,7 +883,12 @@ pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result, source_workspace_id: &str, target_workspace_id: &str, - db: &DB, ) -> Result<()> { // Clone workspace settings (merge with existing basic settings) update_workspace_settings(tx, source_workspace_id, target_workspace_id).await?; @@ -2363,7 +2362,7 @@ async fn clone_workspace_data( clone_resources(tx, source_workspace_id, target_workspace_id).await?; // Clone variables with re-encryption - clone_variables(tx, source_workspace_id, target_workspace_id, db).await?; + clone_variables(tx, source_workspace_id, target_workspace_id).await?; // Clone scripts with new hashes clone_scripts(tx, source_workspace_id, target_workspace_id).await?; @@ -2391,6 +2390,15 @@ async fn update_workspace_settings( source_workspace_id: &str, target_workspace_id: &str, ) -> Result<()> { + sqlx::query!( + "INSERT INTO workspace_key (workspace_id, kind, key) + SELECT $2, kind, key FROM workspace_key WHERE workspace_id = $1", + source_workspace_id, + target_workspace_id, + ) + .execute(&mut **tx) + .await?; + sqlx::query!( r#" UPDATE workspace_settings @@ -2544,81 +2552,18 @@ async fn clone_variables( tx: &mut Transaction<'_, Postgres>, source_workspace_id: &str, target_workspace_id: &str, - db: &DB, ) -> Result<()> { - // Get all variables from source workspace - let variables = sqlx::query_as!( - ExportableListableVariable, - "SELECT workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at - FROM variable + sqlx::query!( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) + SELECT $2, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at + FROM variable WHERE workspace_id = $1", - source_workspace_id + source_workspace_id, + target_workspace_id, ) - .fetch_all(&mut **tx) + .execute(&mut **tx) .await?; - if variables.is_empty() { - return Ok(()); - } - - // Get workspace keys from within the transaction - let source_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - source_workspace_id - ) - .fetch_one(db) - .await?; - - let target_key = sqlx::query_scalar!( - "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'", - target_workspace_id - ) - .fetch_one(&mut **tx) - .await?; - - // Build encryption keys manually - use windmill_common::variables::SECRET_SALT; - let source_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", source_key, salt) - } else { - source_key - }; - let target_crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { - format!("{}{}", target_key, salt) - } else { - target_key - }; - - let source_mc = magic_crypt::new_magic_crypt!(source_crypt_key, 256); - let target_mc = magic_crypt::new_magic_crypt!(target_crypt_key, 256); - - // Process each variable - for var in variables { - let final_value = if var.is_secret && var.value.is_some() { - // Decrypt with source key and re-encrypt with target key - let decrypted_value = decrypt(&source_mc, var.value.unwrap())?; - Some(encrypt(&target_mc, &decrypted_value)) - } else { - var.value - }; - - sqlx::query!( - "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account, is_oauth, expires_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", - target_workspace_id, - var.path, - final_value, - var.is_secret, - var.description, - var.extra_perms, - var.account, - var.is_oauth, - var.expires_at, - ) - .execute(&mut **tx) - .await?; - } - Ok(()) } @@ -2984,16 +2929,6 @@ async fn create_workspace_fork( ) .execute(&mut *tx) .await?; - let key = rd_string(64); - sqlx::query!( - "INSERT INTO workspace_key - (workspace_id, kind, key) - VALUES ($1, 'cloud', $2)", - forked_id, - &key - ) - .execute(&mut *tx) - .await?; sqlx::query!( "INSERT INTO usr @@ -3008,7 +2943,7 @@ async fn create_workspace_fork( .await?; // Clone all data from the parent workspace using Rust implementation - clone_workspace_data(&mut tx, &nw.parent_workspace_id, &forked_id, &db).await?; + clone_workspace_data(&mut tx, &nw.parent_workspace_id, &forked_id).await?; sqlx::query!( "INSERT INTO workspace_invite (workspace_id, email, is_admin, operator) diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index fb7afaccaa..e0a883ef5a 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -508,7 +508,12 @@ pub(crate) async fn tarball_workspace( && var.value.is_some() && var.is_secret { - var.value = Some(decrypt(&mc, var.value.unwrap())?); + var.value = Some(decrypt(&mc, var.value.unwrap()).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + var.path, e + )) + })?); } let var_str = &to_string_without_metadata(&var, false, None).unwrap(); archive diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 577f613f21..46d5d687c3 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -10,6 +10,9 @@ lazy_static::lazy_static! { static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); } +pub const AZURE_API_VERSION: &str = "2025-04-01-preview"; +pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "lowercase")] pub enum AIProvider { @@ -78,6 +81,39 @@ impl AIProvider { pub fn is_anthropic(&self) -> bool { matches!(self, AIProvider::Anthropic) } + + /// Check if this provider/URL combination represents Azure OpenAI + pub fn is_azure_openai(&self, base_url: &str) -> bool { + (matches!(self, AIProvider::OpenAI) && base_url != OPENAI_BASE_URL) + || matches!(self, AIProvider::AzureOpenAI) + } + + /// Build Azure OpenAI URL with deployment model path + pub fn build_azure_openai_url(base_url: &str, model: &str, path: &str) -> String { + let base_url = base_url.trim_end_matches('/'); + + if base_url.ends_with("/deployments") { + format!("{}/{}/{}", base_url, model, path) + } else if base_url.ends_with("/openai") { + format!("{}/deployments/{}/{}", base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } + + /// Extract model from request body (needed for Azure deployments) + pub fn extract_model_from_body(body: &[u8]) -> Result { + #[derive(serde::Deserialize)] + struct ModelRequest { + model: String, + } + + let model_request: ModelRequest = serde_json::from_slice(body).map_err(|e| { + Error::internal_err(format!("Failed to parse request body for model: {}", e)) + })?; + + Ok(model_request.model) + } } impl TryFrom<&str> for AIProvider { diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 1ae95843b7..8b16e6f427 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -43,6 +43,8 @@ pub struct FlowStatus { pub approval_conditions: Option, #[serde(skip_serializing_if = "Option::is_none")] pub restarted_from: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_job: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Default)] @@ -442,6 +444,7 @@ impl FlowStatus { retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, restarted_from: None, user_states: HashMap::new(), + stream_job: None, } } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 962ffd6c11..4f93812f6a 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -146,10 +146,10 @@ impl FlowValue { .preprocessor_module .as_deref() .with_context(|| format!("no preprocessor module")), - Step::Step(i) => self + Step::Step { idx, .. } => self .modules - .get(i) - .with_context(|| format!("no module found at index: {i}")), + .get(idx) + .with_context(|| format!("no module found at index: {idx}")), Step::FailureStep => self .failure_module .as_deref() @@ -162,7 +162,7 @@ impl FlowValue { #[derive(Debug, Copy, Clone)] pub enum Step { - Step(usize), + Step { idx: usize, len: usize }, PreprocessorStep, FailureStep, } @@ -172,7 +172,7 @@ impl Step { if step < 0 { Step::PreprocessorStep } else if (step as usize) < len { - Step::Step(step as usize) + Step::Step { idx: step as usize, len } } else { Step::FailureStep } @@ -180,13 +180,13 @@ impl Step { pub fn get_step_index(&self) -> Option { match self { - Step::Step(index) => Some(*index), + Step::Step { idx, .. } => Some(*idx), _ => None, } } pub fn is_index_step(&self) -> bool { - matches!(self, Step::Step(_)) + matches!(self, Step::Step { .. }) } pub fn is_preprocessor_step(&self) -> bool { @@ -196,6 +196,10 @@ impl Step { pub fn is_failure_step(&self) -> bool { matches!(self, Step::FailureStep) } + + pub fn is_last_step(&self) -> bool { + matches!(self, Step::Step { idx, len } if *idx == len - 1) + } } #[derive(Default, Deserialize, Serialize, Debug, Clone)] diff --git a/backend/windmill-common/src/result_stream.rs b/backend/windmill-common/src/result_stream.rs index 4daa0cbd40..d44933891d 100644 --- a/backend/windmill-common/src/result_stream.rs +++ b/backend/windmill-common/src/result_stream.rs @@ -1,5 +1,5 @@ -use uuid::Uuid; use crate::{error, DB}; +use uuid::Uuid; pub const STREAM_PREFIX: &str = "WM_STREAM: "; @@ -14,9 +14,12 @@ pub fn extract_stream_from_logs(line: &str) -> Option { None } - - -pub async fn append_result_stream_db(db: &DB, workspace_id: &str, job_id: &Uuid, nstream: &str) -> error::Result<()> { +pub async fn append_result_stream_db( + db: &DB, + workspace_id: &str, + job_id: &Uuid, + nstream: &str, +) -> error::Result<()> { if !nstream.is_empty() { sqlx::query!( r#" @@ -27,7 +30,9 @@ pub async fn append_result_stream_db(db: &DB, workspace_id: &str, job_id: &Uuid, workspace_id, job_id, nstream, - ).execute(db).await?; + ) + .execute(db) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index ae81512a0a..92edbcc348 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -164,7 +164,12 @@ pub async fn get_secret_value_as_admin( let value = variable.value; if !value.is_empty() { let mc = build_crypt(db, w_id).await?; - decrypt(&mc, value)? + decrypt(&mc, value).map_err(|e| { + crate::error::Error::internal_err(format!( + "Error decrypting variable {}: {}", + variable.path, e + )) + })? } else { "".to_string() } diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 70be40e28b..f1a0a35be3 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -268,7 +268,12 @@ async fn transform_json_unchecked( .await .map_err(to_anyhow)?; let mc = build_crypt(&db, &w_id).await?; - let variable = decrypt(&mc, variable)?; + let variable = decrypt(&mc, variable).map_err(|e| { + Error::internal_err(format!( + "Error decrypting variable {}: {}", + &s, e + )) + })?; serde_json::Value::String(variable) } s @ serde_json::Value::String(_) => s.clone(), diff --git a/backend/windmill-queue/src/flow_status.rs b/backend/windmill-queue/src/flow_status.rs index 50af234149..e6b64c659d 100644 --- a/backend/windmill-queue/src/flow_status.rs +++ b/backend/windmill-queue/src/flow_status.rs @@ -1,6 +1,9 @@ use uuid::Uuid; use windmill_common::{ - error::{self, Error}, flows::Step, utils::WarnAfterExt, DB + error::{self, Error}, + flows::Step, + utils::WarnAfterExt, + DB, }; pub async fn update_flow_status_in_progress( @@ -11,7 +14,7 @@ pub async fn update_flow_status_in_progress( ) -> error::Result { let step = get_step_of_flow_status(db, flow).await?; match step { - Step::Step(step) => { + Step::Step { idx: step, .. } => { sqlx::query!( "UPDATE v2_job_status SET flow_status = jsonb_set( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 487a7be632..7a7815dc6c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4098,6 +4098,7 @@ pub async fn push<'c, 'd>( }), user_states, preprocessor_module: None, + stream_job: None, } } _ => { @@ -4352,6 +4353,7 @@ pub async fn push<'c, 'd>( }), user_states, preprocessor_module: None, + stream_job: None, }; let value = flow_data.value(); let priority = value.priority; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index de7261a44e..19c8c0d31f 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -67,6 +67,7 @@ serde.workspace = true serde_json.workspace = true futures.workspace = true async-recursion.workspace = true +async-trait.workspace = true anyhow.workspace = true itertools.workspace = true regex.workspace = true diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs new file mode 100644 index 0000000000..c668ded17f --- /dev/null +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -0,0 +1,68 @@ +use base64::Engine; +use futures; +use ulid; +use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; +use windmill_queue::MiniPulledJob; + +/// Upload image to S3 and return S3Object +pub async fn upload_image_to_s3( + base64_image: &str, + job: &MiniPulledJob, + client: &AuthedClient, +) -> Result { + let image_bytes = base64::engine::general_purpose::STANDARD + .decode(base64_image) + .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; + + // Generate unique S3 key + let unique_id = ulid::Ulid::new().to_string(); + let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id); + + // Create byte stream + let byte_stream = futures::stream::once(async move { + Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes)) + }); + + // Upload to S3 + client + .upload_s3_file( + &job.workspace_id, + s3_key.clone(), + None, // storage - use default + byte_stream, + ) + .await + .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?; + + Ok(S3Object { + s3: s3_key, + storage: None, + filename: Some("generated_image.png".to_string()), + presigned: None, + }) +} + +/// Download an S3 image and convert it to a base64 data URL +pub async fn download_and_encode_s3_image( + image: &S3Object, + client: &AuthedClient, + workspace_id: &str, +) -> Result<(String, String), Error> { + // Download the image from S3 + let image_bytes = client + .download_s3_file(workspace_id, &image.s3, image.storage.clone()) + .await + .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; + + // Encode as base64 data URL + let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes); + + // Determine MIME type using mime_guess from file extension, with PNG as fallback + let mime_type = mime_guess::from_path(&image.s3).first(); + let mime_type = mime_type + .as_ref() + .map(|mime| mime.essence_str()) + .unwrap_or("image/png"); + + Ok((mime_type.to_string(), base64_data)) +} diff --git a/backend/windmill-worker/src/ai/mod.rs b/backend/windmill-worker/src/ai/mod.rs new file mode 100644 index 0000000000..b381e51693 --- /dev/null +++ b/backend/windmill-worker/src/ai/mod.rs @@ -0,0 +1,7 @@ +// AI executor module structure +// This module will contain all AI-related execution logic + +pub mod image_handler; +pub mod providers; +pub mod query_builder; +pub mod types; diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs new file mode 100644 index 0000000000..b5a27a5f99 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -0,0 +1,276 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// Google AI/Gemini-specific types +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct GeminiInlineData { + #[serde(rename = "mimeType")] + pub mime_type: String, + pub data: String, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum GeminiPart { + Text { text: String }, + InlineData { inline_data: GeminiInlineData }, + FunctionCall { function_call: GeminiFunctionCall }, + FunctionResponse { function_response: GeminiFunctionResponse }, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionCall { + pub name: String, + pub args: serde_json::Value, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct GeminiFunctionResponse { + pub name: String, + pub response: serde_json::Value, +} + +#[derive(Serialize)] +pub struct GeminiContent { + pub parts: Vec, +} + +#[derive(Serialize)] +pub struct GeminiImageRequest { + #[serde(skip_serializing_if = "Option::is_none")] + pub contents: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub instances: Option>, +} + +#[derive(Serialize)] +pub struct GeminiPredictContent { + pub prompt: String, +} + +#[derive(Deserialize)] +pub struct GeminiImageResponse { + #[serde(skip_serializing_if = "Option::is_none")] + pub candidates: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub predictions: Option>, +} + +#[derive(Deserialize)] +pub struct GeminiCandidate { + pub content: GeminiResponseContent, +} + +#[derive(Deserialize)] +pub struct GeminiPredictCandidate { + #[serde(rename = "bytesBase64Encoded")] + pub bytes_base64_encoded: String, // base64 encoded image +} + +#[derive(Deserialize)] +pub struct GeminiResponseContent { + pub parts: Vec, +} + +#[derive(Deserialize)] +pub struct GeminiResponsePart { + #[serde(skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub text: Option, + #[serde(rename = "inlineData", skip_serializing_if = "Option::is_none")] + pub inline_data: Option, + #[serde(rename = "functionCall", skip_serializing_if = "Option::is_none")] + #[allow(dead_code)] + pub function_call: Option, +} + +pub struct GoogleAIQueryBuilder; + +impl GoogleAIQueryBuilder { + pub fn new() -> Self { + Self + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + let is_imagen = args.model.contains("imagen"); + + let request = if is_imagen { + // For Imagen models, use simple prompt format + GeminiImageRequest { + instances: Some(vec![GeminiPredictContent { + prompt: args.user_message.trim().to_string(), + }]), + contents: None, + } + } else { + // For Gemini models with image generation, build parts + let mut parts = vec![GeminiPart::Text { text: args.user_message.trim().to_string() }]; + + if let Some(system_prompt) = args.system_prompt { + parts.insert( + 0, + GeminiPart::Text { text: format!("SYSTEM PROMPT: {}", system_prompt.trim()) }, + ); + } + + // Add input images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + parts.push(GeminiPart::InlineData { + inline_data: GeminiInlineData { + mime_type: mime_type, + data: image_bytes, + }, + }); + } + } + } + + GeminiImageRequest { instances: None, contents: Some(vec![GeminiContent { parts }]) } + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for GoogleAIQueryBuilder { + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool { + // Google AI supports tools only for text output + matches!(output_type, OutputType::Text) + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text output, use OpenAI-compatible format + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + openai_builder + .build_request(args, client, workspace_id) + .await + } + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let url = response.url().path(); + + // For chat completions (text), use OpenAI parser + if url.contains("/chat/completions") { + let openai_builder = super::openai::OpenAIQueryBuilder::new(AIProvider::GoogleAI); + return openai_builder.parse_response(response).await; + } + + // Check if this is an image generation response + if url.contains(":predict") || url.contains(":generateContent") { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse Gemini response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find image data in response + let image_data = gemini_response + .candidates + .as_ref() + .and_then(|candidates| { + candidates.iter().find_map(|candidate| { + candidate + .content + .parts + .iter() + .find_map(|part| part.inline_data.as_ref().map(|data| &data.data)) + }) + }) + .or_else(|| { + gemini_response + .predictions + .as_ref() + .and_then(|predictions| { + predictions + .iter() + .find_map(|prediction| Some(&prediction.bytes_base64_encoded)) + }) + }); + + if let Some(base64_image) = image_data { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No image data received from Gemini".to_string(), + )) + } + } else { + // This should not happen as we use OpenAI format for text + Err(Error::internal_err( + "Unexpected text response in Google AI parser".to_string(), + )) + } + } + + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + match output_type { + OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint + OutputType::Image => { + // For image generation, build the full URL with model name + let url_suffix = if model.contains("imagen") { + "predict" + } else { + "generateContent" + }; + format!( + "https://generativelanguage.googleapis.com/v1beta/models/{}:{}", + model, url_suffix + ) + } + } + } + + fn get_auth_headers( + &self, + api_key: &str, + _base_url: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + match output_type { + OutputType::Text => { + // For text output, use Bearer token (OpenAI-compatible) + vec![("Authorization", format!("Bearer {}", api_key))] + } + OutputType::Image => { + // For image generation, use Google API key header + vec![("x-goog-api-key", api_key.to_string())] + } + } + } +} diff --git a/backend/windmill-worker/src/ai/providers/mod.rs b/backend/windmill-worker/src/ai/providers/mod.rs new file mode 100644 index 0000000000..13cf766e28 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/mod.rs @@ -0,0 +1,3 @@ +pub mod google_ai; +pub mod openai; +pub mod openrouter; diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs new file mode 100644 index 0000000000..421c5761f6 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -0,0 +1,362 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + image_handler::download_and_encode_s3_image, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// OpenAI-specific types +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIFunction { + pub name: String, + pub arguments: String, +} + +#[derive(Deserialize, Serialize, Clone, Debug)] +pub struct OpenAIToolCall { + pub id: String, + pub function: OpenAIFunction, + pub r#type: String, +} + +#[derive(Deserialize)] +pub struct OpenAIChoice { + pub message: OpenAIMessage, +} + +#[derive(Deserialize)] +pub struct OpenAIResponse { + pub choices: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationTool { + pub r#type: String, + pub quality: Option, + pub background: Option, +} + +// Input content for image generation - supports both text and images +#[derive(Serialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ImageGenerationContent { + #[serde(rename = "input_text")] + InputText { text: String }, + #[serde(rename = "input_image")] + InputImage { image_url: String }, +} + +#[derive(Serialize)] +pub struct ImageGenerationMessage { + pub role: String, + pub content: Vec, +} + +#[derive(Serialize)] +pub struct ImageGenerationRequest<'a> { + pub model: &'a str, + pub input: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option<&'a str>, + pub tools: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageResponse { + pub output: Vec, +} + +#[derive(Deserialize)] +pub struct OpenAIImageOutput { + pub r#type: String, // Expected to be "image_generation_call" + pub status: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, // Base64 encoded image, None if not completed +} + +#[derive(Serialize)] +pub struct OpenAIRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, +} + +pub struct OpenAIQueryBuilder { + provider_kind: AIProvider, +} + +impl OpenAIQueryBuilder { + pub fn new(provider_kind: AIProvider) -> Self { + Self { provider_kind } + } + + pub async fn prepare_messages_for_api( + &self, + messages: &[OpenAIMessage], + client: &AuthedClient, + workspace_id: &str, + ) -> Result, Error> { + let mut prepared_messages = Vec::new(); + + for message in messages { + let mut prepared_message = message.clone(); + + if let Some(content) = &message.content { + match content { + OpenAIContent::Text(text) => { + prepared_message.content = Some(OpenAIContent::Text(text.clone())); + } + OpenAIContent::Parts(parts) => { + let mut prepared_content = Vec::new(); + + for part in parts { + match part { + ContentPart::S3Object { s3_object } => { + // Convert S3Object to base64 image URL + let (mime_type, image_bytes) = download_and_encode_s3_image( + s3_object, + client, + workspace_id, + ) + .await?; + prepared_content.push(ContentPart::ImageUrl { + image_url: ImageUrlData { + url: format!( + "data:{};base64,{}", + mime_type, image_bytes + ), + }, + }); + } + other => { + // Keep Text and ImageUrl as-is + prepared_content.push(other.clone()); + } + } + } + + prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); + } + } + } + + prepared_messages.push(prepared_message); + } + + Ok(prepared_messages) + } + + async fn build_text_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + let prepared_messages = self + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + let request = OpenAIRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + + async fn build_image_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + // Build content array with text and optional image + let mut content = + vec![ImageGenerationContent::InputText { text: args.user_message.to_string() }]; + + // Add images if provided + if let Some(images) = args.images { + for image in images.iter() { + if !image.s3.is_empty() { + let (mime_type, image_bytes) = + download_and_encode_s3_image(image, client, workspace_id).await?; + content.push(ImageGenerationContent::InputImage { + image_url: format!("data:{};base64,{}", mime_type, image_bytes), + }); + } + } + } + + // Build the request with tools if provided + let tools = vec![ImageGenerationTool { + r#type: "image_generation".to_string(), + quality: Some("low".to_string()), + background: None, + }]; + + // TODO: OpenAI's image generation API doesn't support custom tools in the same way as chat completions + // This would require a different approach, potentially using chat completions with image output + // For now, we'll use the standard image generation without custom tools + + let image_request = ImageGenerationRequest { + model: args.model, + input: vec![ImageGenerationMessage { role: "user".to_string(), content }], + instructions: args.system_prompt, + tools, + }; + + serde_json::to_string(&image_request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } +} + +#[async_trait] +impl QueryBuilder for OpenAIQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenAI supports tools for both text and image output + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => self.build_text_request(args, client, workspace_id).await, + OutputType::Image => self.build_image_request(args, client, workspace_id).await, + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + // Check if this is an image response + let url = response.url().path(); + if url.contains("/responses") { + // Parse image generation response + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + let image_response: OpenAIImageResponse = serde_json::from_str(&response_text) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse OpenAI image response: {}. Raw response: {}", + e, response_text + )) + })?; + + // Find the first completed image generation output + let image_generation_call = image_response + .output + .iter() + .find(|output| { + output.r#type == "image_generation_call" && output.status == "completed" + }) + .and_then(|output| output.result.as_ref()); + + if let Some(base64_image) = image_generation_call { + Ok(ParsedResponse::Image { base64_data: base64_image.clone() }) + } else { + Err(Error::internal_err( + "No completed image output received from OpenAI".to_string(), + )) + } + } else { + // Parse text/chat completion response + let openai_response: OpenAIResponse = response + .json() + .await + .map_err(|e| Error::internal_err(format!("Failed to parse response: {}", e)))?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => { + // Extract text from parts + parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" ") + } + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + }) + } + } + + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + let path = match output_type { + OutputType::Text => "chat/completions", + OutputType::Image => "responses", + }; + + if self.provider_kind.is_azure_openai(base_url) { + AIProvider::build_azure_openai_url(base_url, model, path) + } else { + format!("{}/{}", base_url, path) + } + } + + fn get_auth_headers( + &self, + api_key: &str, + base_url: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + if self.provider_kind.is_azure_openai(base_url) { + vec![("api-key", api_key.to_string())] + } else { + vec![("Authorization", format!("Bearer {}", api_key))] + } + } +} diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs new file mode 100644 index 0000000000..3d5dd0314a --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -0,0 +1,203 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use serde_json; +use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; + +use crate::ai::{ + providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, + query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder}, + types::*, +}; + +// OpenRouter-specific types +#[derive(Serialize)] +pub struct OpenRouterChatRequest<'a> { + pub model: &'a str, + pub messages: &'a [OpenAIMessage], + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option<&'a [ToolDef]>, + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_completion_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponse { + pub choices: Vec, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageChoice { + pub message: OpenRouterImageResponseMessage, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageResponseMessage { + #[serde(skip_serializing_if = "Option::is_none")] + pub images: Option>, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageData { + pub image_url: OpenRouterImageUrl, +} + +#[derive(Deserialize)] +pub struct OpenRouterImageUrl { + pub url: String, // data:image/png;base64,... format +} + +pub struct OpenRouterQueryBuilder { + // OpenRouter uses OpenAI-compatible API, so we delegate most work to OpenAI builder + openai_builder: OpenAIQueryBuilder, +} + +impl OpenRouterQueryBuilder { + pub fn new() -> Self { + Self { openai_builder: OpenAIQueryBuilder::new(AIProvider::OpenRouter) } + } +} + +#[async_trait] +impl QueryBuilder for OpenRouterQueryBuilder { + fn supports_tools_with_output_type(&self, _output_type: &OutputType) -> bool { + // OpenRouter supports tools for both text and image output (via OpenAI-compatible API) + true + } + + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result { + match args.output_type { + OutputType::Text => { + // For text, use standard OpenAI format without modalities + self.openai_builder + .build_request(args, client, workspace_id) + .await + } + OutputType::Image => { + // For image generation, we need to add modalities field + // First, prepare the messages using the OpenAI builder's logic + let openai_builder = &self.openai_builder; + let prepared_messages = openai_builder + .prepare_messages_for_api(args.messages, client, workspace_id) + .await?; + + // Check if we need to add response_format for structured output + let has_output_properties = args + .output_schema + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let response_format = if has_output_properties && args.output_schema.is_some() { + let schema = args.output_schema.unwrap(); + let strict_schema = schema.clone().make_strict(); + Some(ResponseFormat { + r#type: "json_schema".to_string(), + json_schema: JsonSchemaFormat { + name: "structured_output".to_string(), + schema: strict_schema, + strict: Some(true), + }, + }) + } else { + None + }; + + // Build OpenRouter-specific request with modalities + let request = OpenRouterChatRequest { + model: args.model, + messages: &prepared_messages, + tools: args.tools, + temperature: args.temperature, + max_completion_tokens: args.max_tokens, + response_format, + modalities: Some(vec!["image", "text"]), + }; + + serde_json::to_string(&request) + .map_err(|e| Error::internal_err(format!("Failed to serialize request: {}", e))) + } + } + } + + async fn parse_response(&self, response: reqwest::Response) -> Result { + let response_text = response + .text() + .await + .map_err(|e| Error::internal_err(format!("Failed to read response text: {}", e)))?; + + // First try to parse as OpenRouter image response + if let Ok(image_response) = serde_json::from_str::(&response_text) + { + // Extract base64 image from the first choice + let image_url = image_response + .choices + .get(0) + .and_then(|choice| choice.message.images.as_ref()) + .and_then(|images| images.get(0)) + .map(|image| &image.image_url.url); + + if let Some(data_url) = image_url { + // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... + if let Some(base64_start) = data_url.find("base64,") { + let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix + return Ok(ParsedResponse::Image { base64_data: base64_data.to_string() }); + } + } + } + + // If not an image response or parsing failed, try as regular OpenAI response + let openai_response: OpenAIResponse = + serde_json::from_str(&response_text).map_err(|e| { + Error::internal_err(format!( + "Failed to parse response: {}. Raw response: {}", + e, response_text + )) + })?; + + let first_choice = openai_response + .choices + .into_iter() + .next() + .ok_or_else(|| Error::internal_err("No response from API"))?; + + Ok(ParsedResponse::Text { + content: first_choice.message.content.map(|c| match c { + OpenAIContent::Text(text) => text, + OpenAIContent::Parts(parts) => parts + .into_iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text), + _ => None, + }) + .collect::>() + .join(" "), + }), + tool_calls: first_choice.message.tool_calls.unwrap_or_default(), + }) + } + + fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { + // OpenRouter uses the same endpoint for both text and image generation + format!("{}/chat/completions", base_url) + } + + fn get_auth_headers( + &self, + api_key: &str, + _base_url: &str, + _output_type: &OutputType, + ) -> Vec<(&'static str, String)> { + vec![("Authorization", format!("Bearer {}", api_key))] + } +} diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs new file mode 100644 index 0000000000..6b043726bc --- /dev/null +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -0,0 +1,71 @@ +use async_trait::async_trait; +use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; + +use crate::ai::{ + providers::{ + google_ai::GoogleAIQueryBuilder, + openai::{OpenAIQueryBuilder, OpenAIToolCall}, + openrouter::OpenRouterQueryBuilder, + }, + types::*, +}; + +/// Arguments for building an AI request +pub struct BuildRequestArgs<'a> { + pub messages: &'a [OpenAIMessage], + pub tools: Option<&'a [ToolDef]>, + pub model: &'a str, + pub temperature: Option, + pub max_tokens: Option, + pub output_schema: Option<&'a OpenAPISchema>, + pub output_type: &'a OutputType, + pub system_prompt: Option<&'a str>, + pub user_message: &'a str, + pub images: Option<&'a [S3Object]>, +} + +/// Response from AI provider +pub enum ParsedResponse { + Text { content: Option, tool_calls: Vec }, + Image { base64_data: String }, +} + +/// Trait for building provider-specific AI requests +#[async_trait] +pub trait QueryBuilder: Send + Sync { + /// Check if this provider supports tools with the given output type + fn supports_tools_with_output_type(&self, output_type: &OutputType) -> bool; + + /// Build the request body for the provider + async fn build_request( + &self, + args: &BuildRequestArgs<'_>, + client: &AuthedClient, + workspace_id: &str, + ) -> Result; + + /// Parse the response from the provider + async fn parse_response(&self, response: reqwest::Response) -> Result; + + /// Get the API endpoint for this provider + fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String; + + /// Get the authentication headers for this provider + fn get_auth_headers( + &self, + api_key: &str, + base_url: &str, + output_type: &OutputType, + ) -> Vec<(&'static str, String)>; +} + +/// Factory function to create the appropriate query builder for a provider +pub fn create_query_builder(provider: &ProviderWithResource) -> Box { + use windmill_common::ai_providers::AIProvider; + + match provider.kind { + AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new()), + AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()), + _ => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())), // Pass provider kind for Azure handling + } +} diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs new file mode 100644 index 0000000000..cc1417a92d --- /dev/null +++ b/backend/windmill-worker/src/ai/types.rs @@ -0,0 +1,368 @@ +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::collections::HashMap; +use windmill_common::{ + ai_providers::AIProvider, db::DB, error::Error, flow_status::AgentAction, flows::FlowModule, + s3_helpers::S3Object, +}; +use windmill_parser::Typ; + +use crate::ai::providers::openai::OpenAIToolCall; + +// Shared types used across multiple providers + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentPart { + Text { + text: String, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: ImageUrlData, + }, + #[serde(rename = "s3_object")] + S3Object { + s3_object: S3Object, + }, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ImageUrlData { + pub url: String, // data:image/png;base64,... or https://... +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum OpenAIContent { + Text(String), + Parts(Vec), +} + +#[derive(Deserialize, Serialize, Clone, Default, Debug)] +pub struct OpenAIMessage { + pub role: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_call_id: Option, + #[serde(skip_serializing)] + pub agent_action: Option, +} + +/// same as OpenAIMessage but with agent_action field included in the serialization +#[derive(Serialize)] +pub struct Message<'a> { + #[serde(flatten)] + pub message: &'a OpenAIMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub agent_action: Option<&'a AgentAction>, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ResponseFormat { + pub r#type: String, + pub json_schema: JsonSchemaFormat, +} + +#[derive(Serialize, Clone, Debug)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: OpenAPISchema, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDefFunction { + pub name: String, + pub description: Option, + pub parameters: Box, +} + +#[derive(Serialize, Clone, Debug)] +pub struct ToolDef { + pub r#type: String, + pub function: ToolDefFunction, +} + +pub struct Tool { + pub module: FlowModule, + pub def: ToolDef, +} + +#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum OutputType { + Text, + Image, +} + +impl Default for OutputType { + fn default() -> Self { + OutputType::Text + } +} + +#[derive(Deserialize, Debug)] +pub struct AIAgentArgs { + pub provider: ProviderWithResource, + pub system_prompt: Option, + pub user_message: String, + pub temperature: Option, + pub max_completion_tokens: Option, + pub output_schema: Option, + pub output_type: Option, + pub user_images: Option>, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderResource { + #[serde(alias = "apiKey")] + pub api_key: String, + #[serde(alias = "baseUrl")] + pub base_url: Option, +} + +#[derive(Deserialize, Debug)] +pub struct ProviderWithResource { + pub kind: AIProvider, + pub resource: ProviderResource, + pub model: String, +} + +impl ProviderWithResource { + pub fn get_api_key(&self) -> &str { + &self.resource.api_key + } + + pub fn get_model(&self) -> &str { + &self.model + } + + pub async fn get_base_url(&self, db: &DB) -> Result { + self.kind + .get_base_url(self.resource.base_url.clone(), db) + .await + } +} + +#[derive(Serialize)] +pub struct AIAgentResult<'a> { + pub output: Box, + pub messages: Vec>, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(untagged)] +pub enum SchemaType { + Single(String), + Multiple(Vec), +} + +impl Default for SchemaType { + fn default() -> Self { + SchemaType::Single("object".to_string()) + } +} + +#[derive(Serialize, Deserialize, Default, Clone, Debug)] +pub struct OpenAPISchema { + #[serde(skip_serializing_if = "Option::is_none")] + pub r#type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub items: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub properties: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub required: Option>, + #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] + pub one_of: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub r#enum: Option>, + #[serde( + skip_serializing_if = "Option::is_none", + rename = "additionalProperties" + )] + pub additional_properties: Option, +} + +impl OpenAPISchema { + pub fn from_str(typ: &str) -> Self { + OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } + } + + pub fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { + OpenAPISchema { + r#type: Some(SchemaType::Single(typ.to_string())), + r#enum: enu.clone(), + ..Default::default() + } + } + + pub fn datetime() -> Self { + Self { + r#type: Some(SchemaType::Single("string".to_string())), + format: Some("date-time".to_string()), + ..Default::default() + } + } + + pub fn from_typ(typ: &Typ) -> Self { + match typ { + Typ::Str(enu) => Self::from_str_with_enum("string", enu), + Typ::Int => Self::from_str("integer"), + Typ::Float => Self::from_str("number"), + Typ::Bool => Self::from_str("boolean"), + Typ::Bytes => Self::from_str("string"), + Typ::Datetime => Self::datetime(), + Typ::Resource(_) => Self::from_str("string"), + Typ::Email => Self::from_str("string"), + Typ::Sql => Self::from_str("string"), + Typ::DynSelect(_) => Self::from_str("string"), + Typ::DynMultiselect(_) => Self::from_str("string"), + Typ::List(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("array".to_string())), + items: Some(Box::new(Self::from_typ(typ))), + ..Default::default() + }, + Typ::Object(typ) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + items: None, + properties: typ.props.as_ref().map(|props| { + props + .iter() + .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) + .collect() + }), + required: typ + .props + .as_ref() + .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), + ..Default::default() + }, + Typ::OneOf(variants) => OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + one_of: Some( + variants + .iter() + .map(|variant| { + let schema = OpenAPISchema { + r#type: Some(SchemaType::Single("object".to_string())), + properties: Some( + variant + .properties + .iter() + .map(|prop| { + ( + prop.key.clone(), + Box::new( + if prop.key == "label" || prop.key == "kind" { + Self::from_str_with_enum( + "string", + &Some(vec![variant.label.clone()]), + ) + } else { + Self::from_typ(&prop.typ) + }, + ), + ) + }) + .collect(), + ), + required: Some( + variant + .properties + .iter() + .map(|prop| prop.key.clone()) + .collect(), + ), + ..Default::default() + }; + Box::new(schema) + }) + .collect(), + ), + ..Default::default() + }, + Typ::Unknown => Self::from_str("object"), + } + } + + /// Makes this schema compatible with OpenAI's strict mode by: + /// - Adding additionalProperties: false to all object types + /// - Making non-required properties nullable + /// - Ensuring all properties are in the required array + pub fn make_strict(mut self) -> Self { + // Handle this schema if it's an object type + if let Some(SchemaType::Single(ref type_str)) = self.r#type { + if type_str == "object" { + // Set additionalProperties to false + self.additional_properties = Some(false); + + if let Some(properties) = self.properties.as_mut() { + // Get original required fields + let original_required = self.required.as_ref(); + + if let Some(required) = original_required { + // Update properties to make non-required fields nullable + for (key, prop) in properties.iter_mut() { + let mut new_prop = (**prop).clone(); + // Make non-required fields nullable + if !required.contains(key) { + new_prop = new_prop.make_nullable(); + } + // Recursively make nested schemas strict + new_prop = new_prop.make_strict(); + *prop = Box::new(new_prop); + } + } + + // All properties must be in required array for strict mode + self.required = Some(properties.keys().cloned().collect()); + } + } + } + + // Recursively process nested schemas + if let Some(ref mut items) = self.items { + **items = items.as_ref().clone().make_strict(); + } + + if let Some(ref mut one_of) = self.one_of { + *one_of = one_of + .iter() + .map(|schema| Box::new(schema.as_ref().clone().make_strict())) + .collect(); + } + + self + } + + /// Makes this property nullable by converting its type to a union with null + pub fn make_nullable(mut self) -> Self { + match self.r#type.take() { + Some(SchemaType::Single(type_str)) => { + if type_str != "null" { + self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); + } else { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + Some(SchemaType::Multiple(mut types)) => { + if !types.iter().any(|t| t == "null") { + types.push("null".into()); + } + self.r#type = Some(SchemaType::Multiple(types)); + } + None => { + self.r#type = Some(SchemaType::Single("null".into())); + } + } + self + } +} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 7f5df68bc8..2eddb657b5 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -1,35 +1,35 @@ use async_recursion::async_recursion; -use base64::Engine; -use mime_guess; use regex::Regex; -use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::{collections::HashMap, sync::Arc}; -#[cfg(feature = "benchmark")] -use windmill_common::bench::BenchmarkIter; +use ulid; +use uuid::Uuid; use windmill_common::{ ai_providers::AIProvider, - auth::get_job_perms, + ai_providers::AZURE_API_VERSION, cache, client::AuthedClient, db::DB, error::{self, to_anyhow, Error}, flow_status::AgentAction, - flows::{FlowModule, FlowModuleValue, Step}, + flows::{FlowModuleValue, Step}, get_latest_hash_for_path, jobs::JobKind, - s3_helpers::S3Object, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, }; -use windmill_parser::Typ; use windmill_queue::{ flow_status::get_step_of_flow_status, get_mini_pulled_job, push, CanceledBy, JobCompleted, MiniPulledJob, PushArgs, PushIsolationLevel, }; use crate::{ + ai::{ + image_handler::upload_image_to_s3, + query_builder::{create_query_builder, BuildRequestArgs, ParsedResponse}, + types::*, + }, common::{build_args_map, error_to_value, OccupancyMetrics}, create_job_dir, handle_child::run_future_with_polling_update_job_poller, @@ -39,1093 +39,12 @@ use crate::{ JobCompletedSender, SendResult, SendResultPayload, }; -const MAX_AGENT_ITERATIONS: usize = 10; -const REQUEST_TIMEOUT: u64 = 120; - lazy_static::lazy_static! { static ref TOOL_NAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]+$").unwrap(); } -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIFunction { - name: String, - arguments: String, -} - -#[derive(Deserialize, Serialize, Clone, Debug)] -struct OpenAIToolCall { - id: String, - function: OpenAIFunction, - r#type: String, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ContentPart { - Text { - text: String, - }, - #[serde(rename = "image_url")] - ImageUrl { - image_url: ImageUrlData, - }, - #[serde(rename = "s3_object")] - S3Object { - s3_object: S3Object, - }, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -struct ImageUrlData { - url: String, // data:image/png;base64,... or https://... -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -enum OpenAIContent { - Text(String), - Parts(Vec), -} - -#[derive(Deserialize, Serialize, Clone, Default, Debug)] -struct OpenAIMessage { - role: String, - #[serde(skip_serializing_if = "Option::is_none")] - content: Option, - #[serde(skip_serializing_if = "Option::is_none")] - tool_calls: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - tool_call_id: Option, - #[serde(skip_serializing)] - agent_action: Option, -} - -/// same as OpenAIMessage but with agent_action field included in the serialization -#[derive(Serialize)] -struct Message<'a> { - #[serde(flatten)] - message: &'a OpenAIMessage, - #[serde(skip_serializing_if = "Option::is_none")] - agent_action: Option<&'a AgentAction>, -} - -#[derive(Deserialize)] -struct OpenAIChoice { - message: OpenAIMessage, -} - -#[derive(Deserialize)] -struct OpenAIResponse { - choices: Vec, -} - -#[derive(Serialize)] -struct ImageGenerationTool { - r#type: String, - quality: Option, - background: Option, -} - -// Input content for image generation - supports both text and images -#[derive(Serialize, Clone, Debug)] -#[serde(tag = "type", rename_all = "snake_case")] -enum ImageGenerationContent { - #[serde(rename = "input_text")] - InputText { text: String }, - #[serde(rename = "input_image")] - InputImage { image_url: String }, -} - -#[derive(Serialize)] -struct ImageGenerationMessage { - role: String, - content: Vec, -} - -#[derive(Serialize)] -struct ImageGenerationRequest<'a> { - model: &'a str, - input: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - instructions: Option<&'a str>, - tools: Vec, -} - -#[derive(Deserialize)] -struct OpenAIImageResponse { - output: Vec, -} - -#[derive(Deserialize)] -struct OpenAIImageOutput { - r#type: String, // Expected to be "image_generation_call" - #[serde(default)] - result: Option, // Base64 encoded image -} - -// Gemini API structures -#[derive(Serialize, Deserialize, Clone, Debug)] -struct GeminiInlineData { - #[serde(rename = "mimeType")] - mime_type: String, - data: String, -} - -#[derive(Serialize)] -#[serde(untagged)] -enum GeminiPart { - Text { text: String }, - InlineData { inline_data: GeminiInlineData }, -} - -#[derive(Serialize)] -struct GeminiContent { - parts: Vec, -} - -#[derive(Serialize)] -struct GeminiImageRequest { - #[serde(skip_serializing_if = "Option::is_none")] - contents: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - instances: Option>, -} - -#[derive(Serialize)] -struct GeminiPredictContent { - prompt: String, -} - -#[derive(Deserialize)] -struct GeminiImageResponse { - #[serde(skip_serializing_if = "Option::is_none")] - candidates: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - predictions: Option>, -} - -#[derive(Deserialize)] -struct GeminiCandidate { - content: GeminiResponseContent, -} - -#[derive(Deserialize)] -struct GeminiPredictCandidate { - #[serde(rename = "bytesBase64Encoded")] - bytes_base64_encoded: String, // base64 encoded image -} - -#[derive(Deserialize)] -struct GeminiResponseContent { - parts: Vec, -} - -#[derive(Deserialize)] -struct GeminiResponsePart { - #[serde(rename = "inlineData")] - inline_data: Option, -} - -// OpenRouter image generation structures -#[derive(Serialize)] -struct OpenRouterImageRequest<'a> { - model: &'a str, - messages: Vec, - modalities: Vec<&'a str>, -} - -#[derive(Serialize)] -struct OpenRouterImageMessage { - role: String, - content: String, -} - -#[derive(Deserialize)] -struct OpenRouterImageResponse { - choices: Vec, -} - -#[derive(Deserialize)] -struct OpenRouterImageChoice { - message: OpenRouterImageResponseMessage, -} - -#[derive(Deserialize)] -struct OpenRouterImageResponseMessage { - #[serde(skip_serializing_if = "Option::is_none")] - images: Option>, -} - -#[derive(Deserialize)] -struct OpenRouterImageData { - image_url: OpenRouterImageUrl, -} - -#[derive(Deserialize)] -struct OpenRouterImageUrl { - url: String, // data:image/png;base64,... format -} - -#[derive(Serialize)] -struct OpenAIRequest<'a> { - model: &'a str, - messages: &'a Vec, - #[serde(skip_serializing_if = "Option::is_none")] - tools: Option<&'a Vec>, - #[serde(skip_serializing_if = "Option::is_none")] - temperature: Option, - #[serde(skip_serializing_if = "Option::is_none")] - max_completion_tokens: Option, - #[serde(skip_serializing_if = "Option::is_none")] - response_format: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ResponseFormat { - r#type: String, - json_schema: JsonSchemaFormat, -} - -#[derive(Serialize, Clone, Debug)] -struct JsonSchemaFormat { - name: String, - schema: OpenAPISchema, - #[serde(skip_serializing_if = "Option::is_none")] - strict: Option, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDefFunction { - name: String, - description: Option, - parameters: Box, -} - -#[derive(Serialize, Clone, Debug)] -struct ToolDef { - r#type: String, - function: ToolDefFunction, -} - -struct Tool { - module: FlowModule, - def: ToolDef, -} - -#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] -#[serde(rename_all = "lowercase")] -enum OutputType { - Text, - Image, -} - -impl Default for OutputType { - fn default() -> Self { - OutputType::Text - } -} - -#[derive(Deserialize, Debug)] -struct AIAgentArgs { - provider: ProviderWithResource, - system_prompt: Option, - user_message: String, - temperature: Option, - max_completion_tokens: Option, - output_schema: Option, - output_type: Option, - image: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderResource { - #[serde(alias = "apiKey")] - api_key: String, - #[serde(alias = "baseUrl")] - base_url: Option, -} - -#[derive(Deserialize, Debug)] -struct ProviderWithResource { - kind: AIProvider, - resource: ProviderResource, - model: String, -} - -impl ProviderWithResource { - fn get_api_key(&self) -> &str { - &self.resource.api_key - } - - fn get_model(&self) -> &str { - &self.model - } - - async fn get_base_url(&self, db: &DB) -> Result { - self.kind - .get_base_url(self.resource.base_url.clone(), db) - .await - } -} - -#[derive(Serialize)] -struct AIAgentResult<'a> { - output: Box, - messages: Vec>, -} - -#[derive(Serialize, Deserialize, Clone, Debug)] -#[serde(untagged)] -enum SchemaType { - Single(String), - Multiple(Vec), -} - -impl Default for SchemaType { - fn default() -> Self { - SchemaType::Single("object".to_string()) - } -} - -#[derive(Serialize, Deserialize, Default, Clone, Debug)] -struct OpenAPISchema { - #[serde(skip_serializing_if = "Option::is_none")] - r#type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - items: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - properties: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - required: Option>, - #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] - one_of: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - format: Option, - #[serde(skip_serializing_if = "Option::is_none")] - r#enum: Option>, - #[serde( - skip_serializing_if = "Option::is_none", - rename = "additionalProperties" - )] - additional_properties: Option, -} - -impl OpenAPISchema { - fn from_str(typ: &str) -> Self { - OpenAPISchema { r#type: Some(SchemaType::Single(typ.to_string())), ..Default::default() } - } - - fn from_str_with_enum(typ: &str, enu: &Option>) -> Self { - OpenAPISchema { - r#type: Some(SchemaType::Single(typ.to_string())), - r#enum: enu.clone(), - ..Default::default() - } - } - - fn datetime() -> Self { - Self { - r#type: Some(SchemaType::Single("string".to_string())), - format: Some("date-time".to_string()), - ..Default::default() - } - } - - fn from_typ(typ: &Typ) -> Self { - match typ { - Typ::Str(enu) => Self::from_str_with_enum("string", enu), - Typ::Int => Self::from_str("integer"), - Typ::Float => Self::from_str("number"), - Typ::Bool => Self::from_str("boolean"), - Typ::Bytes => Self::from_str("string"), - Typ::Datetime => Self::datetime(), - Typ::Resource(_) => Self::from_str("string"), - Typ::Email => Self::from_str("string"), - Typ::Sql => Self::from_str("string"), - Typ::DynSelect(_) => Self::from_str("string"), - Typ::DynMultiselect(_) => Self::from_str("string"), - Typ::List(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("array".to_string())), - items: Some(Box::new(Self::from_typ(typ))), - ..Default::default() - }, - Typ::Object(typ) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - items: None, - properties: typ.props.as_ref().map(|props| { - props - .iter() - .map(|prop| (prop.key.clone(), Box::new(Self::from_typ(&prop.typ)))) - .collect() - }), - required: typ - .props - .as_ref() - .map(|props| props.iter().map(|prop| prop.key.clone()).collect()), - ..Default::default() - }, - Typ::OneOf(variants) => OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - one_of: Some( - variants - .iter() - .map(|variant| { - let schema = OpenAPISchema { - r#type: Some(SchemaType::Single("object".to_string())), - properties: Some( - variant - .properties - .iter() - .map(|prop| { - ( - prop.key.clone(), - Box::new( - if prop.key == "label" || prop.key == "kind" { - Self::from_str_with_enum( - "string", - &Some(vec![variant.label.clone()]), - ) - } else { - Self::from_typ(&prop.typ) - }, - ), - ) - }) - .collect(), - ), - required: Some( - variant - .properties - .iter() - .map(|prop| prop.key.clone()) - .collect(), - ), - ..Default::default() - }; - Box::new(schema) - }) - .collect(), - ), - ..Default::default() - }, - Typ::Unknown => Self::from_str("object"), - } - } - - /// Makes this schema compatible with OpenAI's strict mode by: - /// - Adding additionalProperties: false to all object types - /// - Making non-required properties nullable - /// - Ensuring all properties are in the required array - fn make_strict(mut self) -> Self { - // Handle this schema if it's an object type - if let Some(SchemaType::Single(ref type_str)) = self.r#type { - if type_str == "object" { - // Set additionalProperties to false - self.additional_properties = Some(false); - - if let Some(properties) = self.properties.as_mut() { - // Get original required fields - let original_required = self.required.as_ref(); - - if let Some(required) = original_required { - // Update properties to make non-required fields nullable - for (key, prop) in properties.iter_mut() { - let mut new_prop = (**prop).clone(); - // Make non-required fields nullable - if !required.contains(key) { - new_prop = new_prop.make_nullable(); - } - // Recursively make nested schemas strict - new_prop = new_prop.make_strict(); - *prop = Box::new(new_prop); - } - } - - // All properties must be in required array for strict mode - self.required = Some(properties.keys().cloned().collect()); - } - } - } - - // Recursively process nested schemas - if let Some(ref mut items) = self.items { - **items = items.as_ref().clone().make_strict(); - } - - if let Some(ref mut one_of) = self.one_of { - *one_of = one_of - .iter() - .map(|schema| Box::new(schema.as_ref().clone().make_strict())) - .collect(); - } - - self - } - - /// Makes this property nullable by converting its type to a union with null - fn make_nullable(mut self) -> Self { - match self.r#type.take() { - Some(SchemaType::Single(type_str)) => { - if type_str != "null" { - self.r#type = Some(SchemaType::Multiple(vec![type_str, "null".into()])); - } else { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - Some(SchemaType::Multiple(mut types)) => { - if !types.iter().any(|t| t == "null") { - types.push("null".into()); - } - self.r#type = Some(SchemaType::Multiple(types)); - } - None => { - self.r#type = Some(SchemaType::Single("null".into())); - } - } - self - } -} - -/// Find a unique tool name to avoid collisions with user-provided tools -fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { - let Some(tools) = existing_tools else { - return base_name.to_string(); - }; - - if !tools.iter().any(|t| t.function.name == base_name) { - return base_name.to_string(); - } - - for i in 1..100 { - let candidate = format!("{}_{}", base_name, i); - if !tools.iter().any(|t| t.function.name == candidate) { - return candidate; - } - } - - // Fallback with process id if somehow we can't find a unique name - format!("{}_{}_fallback", base_name, std::process::id()) -} - -/// Helper function to download an S3 image and convert it to a base64 data URL -async fn download_and_encode_s3_image( - image: &S3Object, - client: &AuthedClient, - workspace_id: &str, -) -> error::Result<(String, String)> { - // Download the image from S3 - let image_bytes = client - .download_s3_file(workspace_id, &image.s3, image.storage.clone()) - .await - .map_err(|e| Error::internal_err(format!("Failed to download S3 image: {}", e)))?; - - // Encode as base64 data URL - let base64_data = base64::engine::general_purpose::STANDARD.encode(&image_bytes); - - // Determine MIME type using mime_guess from file extension, with PNG as fallback - let mime_type = mime_guess::from_path(&image.s3).first(); - let mime_type = mime_type - .as_ref() - .map(|mime| mime.essence_str()) - .unwrap_or("image/png"); - - Ok((mime_type.to_string(), base64_data)) -} - -/// Convert messages with S3Objects to messages with base64 image URLs for API calls -async fn prepare_messages_for_api( - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, -) -> error::Result> { - let mut prepared_messages = Vec::new(); - - for message in messages { - let mut prepared_message = message.clone(); - - if let Some(content) = &message.content { - match content { - OpenAIContent::Text(text) => { - prepared_message.content = Some(OpenAIContent::Text(text.clone())); - } - OpenAIContent::Parts(parts) => { - let mut prepared_content = Vec::new(); - - for part in parts { - match part { - ContentPart::S3Object { s3_object } => { - // Convert S3Object to base64 image URL - let (mime_type, image_data_url) = - download_and_encode_s3_image(s3_object, client, workspace_id) - .await?; - prepared_content.push(ContentPart::ImageUrl { - image_url: ImageUrlData { - url: format!( - "data:{};base64,{}", - mime_type, image_data_url - ), - }, - }); - } - other => { - // Keep Text and ImageUrl as-is - prepared_content.push(other.clone()); - } - } - } - - prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); - } - } - } - - prepared_messages.push(prepared_message); - } - - Ok(prepared_messages) -} - -/// Generate image from provider and extract base64 data -async fn generate_image_from_provider( - provider: &ProviderWithResource, - user_message: &str, - system_prompt: Option<&str>, - base_url: &str, - api_key: &str, - image: Option<&S3Object>, - client: &AuthedClient, - workspace_id: &str, -) -> error::Result { - match provider.kind { - AIProvider::OpenAI => { - // Build content array with text and optional image - let mut content = - vec![ImageGenerationContent::InputText { text: user_message.to_string() }]; - - // Add image if provided - if let Some(image) = image { - if !image.s3.is_empty() { - // Download and encode S3 image to base64 - let (mime_type, bytes64) = - download_and_encode_s3_image(image, client, workspace_id).await?; - content.push(ImageGenerationContent::InputImage { - image_url: format!("data:{};base64,{}", mime_type, bytes64), - }); - } - } - - let image_request = ImageGenerationRequest { - model: provider.get_model(), - input: vec![ImageGenerationMessage { role: "user".to_string(), content }], - instructions: system_prompt, - tools: vec![ImageGenerationTool { - r#type: "image_generation".to_string(), - quality: Some("low".to_string()), - background: None, - }], - }; - - let resp = HTTP_CLIENT - .post(format!("{}/responses", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&image_request) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call OpenAI API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => { - let image_response = resp.json::().await.map_err(|e| { - Error::internal_err(format!("Failed to parse OpenAI response: {}", e)) - })?; - - // Find the first image generation output - let image_generation_call = image_response - .output - .iter() - .find(|output| output.r#type == "image_generation_call") - .and_then(|output| output.result.as_ref()); - - if let Some(base64_image) = image_generation_call { - Ok(base64_image.to_string()) - } else { - Err(Error::internal_err( - "No image output received from OpenAI".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "OpenAI API error: {} - {}", - e, text - ))) - } - } - } - AIProvider::GoogleAI => { - let is_imagen = provider.get_model().contains("imagen"); - - let gemini_request = if is_imagen { - // For Imagen models, we keep the simple prompt format (no image support) - GeminiImageRequest { - instances: Some(vec![GeminiPredictContent { - prompt: user_message.trim().to_string(), - }]), - contents: None, - } - } else { - // For Gemini models, build parts array with text and optional image - let mut parts = vec![GeminiPart::Text { text: user_message.trim().to_string() }]; - - if let Some(system_prompt) = system_prompt { - parts.insert( - 0, - GeminiPart::Text { - text: format!("SYSTEM PROMPT: {}", system_prompt.trim().to_string()), - }, - ); - } - - // Add image if provided - if let Some(image) = image { - if !image.s3.is_empty() { - // Download and encode S3 image to base64 - let (mime_type, bytes64) = - download_and_encode_s3_image(image, client, workspace_id).await?; - - parts.push(GeminiPart::InlineData { - inline_data: GeminiInlineData { mime_type, data: bytes64 }, - }); - } - } - - GeminiImageRequest { - instances: None, - contents: Some(vec![GeminiContent { parts }]), - } - }; - - let url_suffix = if is_imagen { - "predict" - } else { - "generateContent" - }; - let gemini_url = format!( - "https://generativelanguage.googleapis.com/v1beta/models/{}:{}", - provider.get_model(), - url_suffix - ); - - let resp = HTTP_CLIENT - .post(&gemini_url) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .header("x-goog-api-key", api_key) - .header("Content-Type", "application/json") - .json(&gemini_request) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call Gemini API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => { - let response_text = resp.text().await.map_err(|e| { - Error::internal_err(format!("Failed to read response text: {}", e)) - })?; - - let gemini_response: GeminiImageResponse = serde_json::from_str(&response_text) - .map_err(|e| { - Error::internal_err(format!( - "Failed to parse Gemini response: {}. Raw response: {}", - e, response_text - )) - })?; - - // Find the first candidate with inline image data - let mut image_data = - gemini_response.candidates.as_ref().and_then(|candidates| { - candidates.iter().find_map(|candidate| { - candidate.content.parts.iter().find_map(|part| { - part.inline_data.as_ref().map(|data| &data.data) - }) - }) - }); - - if image_data.is_none() { - image_data = gemini_response - .predictions - .as_ref() - .and_then(|predictions| { - predictions - .iter() - .find_map(|prediction| Some(&prediction.bytes_base64_encoded)) - }); - } - - if let Some(base64_image) = image_data { - Ok(base64_image.clone()) - } else { - Err(Error::internal_err( - "No image data received from Gemini".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "Gemini API error: {} - {}", - e, text - ))) - } - } - } - AIProvider::OpenRouter => { - let mut messages = Vec::new(); - - // Add system message if provided - if let Some(system_prompt) = system_prompt { - messages.push(OpenRouterImageMessage { - role: "system".to_string(), - content: system_prompt.to_string(), - }); - } - - // Add user message - messages.push(OpenRouterImageMessage { - role: "user".to_string(), - content: user_message.to_string(), - }); - - let openrouter_request = OpenRouterImageRequest { - model: provider.get_model(), - messages, - modalities: vec!["image", "text"], - }; - - let resp = HTTP_CLIENT - .post(format!("{}/chat/completions", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&openrouter_request) - .send() - .await - .map_err(|e| { - Error::internal_err(format!("Failed to call OpenRouter API: {}", e)) - })?; - - match resp.error_for_status_ref() { - Ok(_) => { - let openrouter_response = - resp.json::().await.map_err(|e| { - Error::internal_err(format!( - "Failed to parse OpenRouter response: {}", - e - )) - })?; - - // Extract base64 image from the first choice - let image_url = openrouter_response - .choices - .get(0) - .and_then(|choice| choice.message.images.as_ref()) - .and_then(|images| images.get(0)) - .map(|image| &image.image_url.url); - - if let Some(data_url) = image_url { - // Extract base64 data from data URL format: data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAA... - if let Some(base64_start) = data_url.find("base64,") { - let base64_data = &data_url[base64_start + 7..]; // Skip "base64," prefix - Ok(base64_data.to_string()) - } else { - Err(Error::internal_err( - "Invalid data URL format received from OpenRouter".to_string(), - )) - } - } else { - Err(Error::internal_err( - "No image data received from OpenRouter".to_string(), - )) - } - } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - Err(Error::internal_err(format!( - "OpenRouter API error: {} - {}", - e, text - ))) - } - } - } - _ => Err(Error::BadRequest(format!( - "Image generation is not supported for provider: {:?}", - provider.kind - ))), - } -} - -/// Upload image to S3 and return S3Object -async fn upload_image_to_s3( - base64_image: &str, - job: &MiniPulledJob, - client: &AuthedClient, -) -> error::Result { - let image_bytes = base64::engine::general_purpose::STANDARD - .decode(base64_image) - .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; - - // Generate unique S3 key - let unique_id = ulid::Ulid::new().to_string(); - let s3_key = format!("ai_images/{}/{}.png", job.id, unique_id); - - // Create byte stream - let byte_stream = futures::stream::once(async move { - Ok::<_, std::convert::Infallible>(bytes::Bytes::from(image_bytes)) - }); - - // Upload to S3 - client - .upload_s3_file( - &job.workspace_id, - s3_key.clone(), - None, // storage - use default - byte_stream, - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to upload image to S3: {}", e)))?; - - Ok(S3Object { - s3: s3_key, - storage: None, - filename: Some("generated_image.png".to_string()), - presigned: None, - }) -} - -/// Handle image output generation and return S3 object and messages -async fn handle_image_output( - args: &AIAgentArgs, - job: &MiniPulledJob, - client: &AuthedClient, - db: &DB, -) -> error::Result<(Option, Vec)> { - let base_url = args.provider.get_base_url(db).await?; - let api_key = args.provider.get_api_key(); - - let mut messages = - if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { - vec![OpenAIMessage { - role: "system".to_string(), - content: Some(OpenAIContent::Text(system_prompt)), - ..Default::default() - }] - } else { - vec![] - }; - - // Generate image from provider - let base64_image = generate_image_from_provider( - &args.provider, - &args.user_message, - args.system_prompt.as_deref(), - &base_url, - api_key, - args.image.as_ref(), - client, - &job.workspace_id, - ) - .await?; - - // Add assistant success message - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(OpenAIContent::Text( - "Image created successfully".to_string(), - )), - ..Default::default() - }); - - // Upload to S3 - let s3_object = upload_image_to_s3(&base64_image, job, client).await?; - - Ok((Some(s3_object), messages)) -} - -async fn update_flow_status_module_with_actions( - db: &DB, - parent_job: &uuid::Uuid, - actions: &[AgentAction], -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $3::TEXT, 'agent_actions'], - $2 - ) - WHERE id = $1 - "#, - parent_job, - sqlx::types::Json(actions) as _, - step as i32 - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} - -async fn update_flow_status_module_with_actions_success( - db: &DB, - parent_job: &uuid::Uuid, - action_success: bool, -) -> Result<(), Error> { - let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; - match step { - Step::Step(step) => { - // Append the new bool to the existing array, or create a new array if it doesn't exist - sqlx::query!( - r#" - UPDATE v2_job_status SET - flow_status = jsonb_set( - flow_status, - array['modules', $2::TEXT, 'agent_actions_success'], - COALESCE( - flow_status->'modules'->$2->'agent_actions_success', - to_jsonb(ARRAY[]::bool[]) - ) || to_jsonb(ARRAY[$3::bool]) - ) - WHERE id = $1 - "#, - parent_job, - step as i32, - action_success - ) - .execute(db) - .await?; - } - _ => {} - } - Ok(()) -} +const MAX_AGENT_ITERATIONS: usize = 10; +const REQUEST_TIMEOUT_SECONDS: u64 = 120; fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, Error> { let main_arg_signature = parse_sig_of_lang(content, Some(&language), None)?.unwrap(); // safe to unwrap as langauge is some @@ -1156,546 +75,6 @@ fn parse_raw_script_schema(content: &str, language: &ScriptLang) -> Result, -) -> error::Result> { - let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); - - match *output_type { - OutputType::Image => { - let (s3_result, messages) = handle_image_output(&args, job, client, db).await?; - - let final_messages: Vec = messages - .iter() - .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) - .collect(); - - if let Some(s3_output) = s3_result { - Ok(to_raw_value(&s3_output)) - } else { - Ok(to_raw_value(&AIAgentResult { - output: to_raw_value(&None::), - messages: final_messages, - })) - } - } - OutputType::Text => { - let base_url = args.provider.get_base_url(db).await?; - let api_key = args.provider.get_api_key(); - - let mut messages = - if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { - vec![OpenAIMessage { - role: "system".to_string(), - content: Some(OpenAIContent::Text(system_prompt)), - ..Default::default() - }] - } else { - vec![] - }; - - // Create user message with optional image - let user_content = if let Some(image) = &args.image { - if !image.s3.is_empty() { - OpenAIContent::Parts(vec![ - ContentPart::Text { text: args.user_message.clone() }, - ContentPart::S3Object { s3_object: image.clone() }, - ]) - } else { - OpenAIContent::Text(args.user_message.clone()) - } - } else { - OpenAIContent::Text(args.user_message.clone()) - }; - - messages.push(OpenAIMessage { - role: "user".to_string(), - content: Some(user_content), - ..Default::default() - }); - - let mut actions = vec![]; - let mut content = None; - - let mut tool_defs: Option> = if tools.is_empty() { - None - } else { - Some(tools.iter().map(|t| t.def.clone()).collect()) - }; - - let has_output_properties = args - .output_schema - .as_ref() - .and_then(|schema| schema.properties.as_ref()) - .map(|props| !props.is_empty()) - .unwrap_or(false); - let provider_is_anthropic = args.provider.kind.is_anthropic(); - let is_openrouter_anthropic = args.provider.kind == AIProvider::OpenRouter - && args.provider.model.starts_with("anthropic/"); - let is_anthropic = provider_is_anthropic || is_openrouter_anthropic; - let mut response_format: Option = None; - let mut used_structured_output_tool = false; - let mut structured_output_tool_name: Option = None; - - if has_output_properties { - let schema = args.output_schema.as_ref().unwrap(); // we know it's some because of the check above - if is_anthropic { - // if output schema is provided, and provider is anthropic, add a structured_output tool in the list of tools - let unique_tool_name = - find_unique_tool_name("structured_output", tool_defs.as_deref()); - structured_output_tool_name = Some(unique_tool_name.clone()); - - let output_tool = ToolDef { - r#type: "function".to_string(), - function: ToolDefFunction { - name: unique_tool_name, - description: Some( - "This tool MUST be used last to return a structured JSON object as the final output." - .to_string(), - ), - parameters: to_raw_value(&schema), - }, - }; - if let Some(ref mut existing_tools) = tool_defs { - existing_tools.push(output_tool); - } else { - tool_defs = Some(vec![output_tool]); - } - } else { - // if output schema is provided, and provider is openai, add a response_format with json_schema - let strict_schema = schema.clone().make_strict(); - response_format = Some(ResponseFormat { - r#type: "json_schema".to_string(), - json_schema: JsonSchemaFormat { - name: "structured_output".to_string(), - schema: strict_schema, - strict: Some(true), - }, - }); - } - } - - for i in 0..MAX_AGENT_ITERATIONS { - if used_structured_output_tool { - break; - } - - let response = { - // Convert messages with S3Objects to base64 image URLs for API request - let prepared_messages = - prepare_messages_for_api(&messages, client, &job.workspace_id).await?; - - let resp = HTTP_CLIENT - .post(format!("{}/chat/completions", base_url)) - .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT)) - .bearer_auth(api_key) - .json(&OpenAIRequest { - model: args.provider.get_model(), - messages: &prepared_messages, - tools: tool_defs.as_ref(), - temperature: args.temperature, - max_completion_tokens: args.max_completion_tokens, - response_format: if has_output_properties && !is_anthropic { - response_format.clone() - } else { - None - }, - }) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => resp, - Err(e) => { - let status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - tracing::error!( - "Non 200 response from API: status: {}, body: {}", - status, - text - ); - return Err(Error::internal_err(format!( - "Non 200 response from API: {} - {}", - e, text - ))); - } - } - }; - - let mut response = response.json::().await.map_err(|e| { - Error::internal_err(format!("Failed to parse API response: {}", e)) - })?; - - let first_choice = response - .choices - .pop() - .ok_or_else(|| Error::internal_err("No response from API"))?; - - content = first_choice.message.content; - let tool_calls = first_choice.message.tool_calls.unwrap_or_default(); - - if let Some(ref response_content) = content { - actions.push(AgentAction::Message {}); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(response_content.clone()), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - update_flow_status_module_with_actions_success(db, parent_job, true).await?; - } - - if tool_calls.is_empty() { - break; - } else if i == MAX_AGENT_ITERATIONS - 1 { - return Err(Error::internal_err( - "AI agent reached max iterations, but there are still tool calls" - .to_string(), - )); - } - - messages.push(OpenAIMessage { - role: "assistant".to_string(), - tool_calls: Some(tool_calls.clone()), - ..Default::default() - }); - - for tool_call in tool_calls.iter() { - // Structured output tool is used, we stop here as this will be the final output - if structured_output_tool_name - .as_ref() - .map_or(false, |name| tool_call.function.name == *name) - { - used_structured_output_tool = true; - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text( - "Successfully ran structured_output tool".to_string(), - )), - tool_call_id: Some(tool_call.id.clone()), - ..Default::default() - }); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(OpenAIContent::Text( - tool_call.function.arguments.clone(), - )), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - content = Some(OpenAIContent::Text(tool_call.function.arguments.clone())); - break; - } - - let tool = tools - .iter() - .find(|t| t.def.function.name == tool_call.function.name); - if let Some(tool) = tool { - let job_id = ulid::Ulid::new().into(); - actions.push(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }); - - update_flow_status_module_with_actions(db, parent_job, &actions).await?; - - let tool_call_args = serde_json::from_str::>>( - &tool_call.function.arguments, - )?; - - let job_payload = match tool.module.get_value()? { - FlowModuleValue::Script { - path: script_path, - hash: script_hash, - tag_override, - .. - } => { - let payload = script_to_payload( - script_hash, - script_path, - db, - job, - &tool.module, - tag_override, - tool.module.apply_preprocessor, - ) - .await?; - payload - } - FlowModuleValue::RawScript { - path, - content, - language, - lock, - tag, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - .. - } => { - let path = path.unwrap_or_else(|| { - format!("{}/tools/{}", job.runnable_path(), tool.module.id) - }); - - let payload = raw_script_to_payload( - path, - content, - language, - lock, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - &tool.module, - tag, - tool.module.delete_after_use.unwrap_or(false), - ); - payload - } - _ => { - return Err(Error::internal_err(format!( - "Unsupported tool: {}", - tool_call.function.name - ))); - } - }; - - let mut tx = db.begin().await?; - - let job_perms = get_job_perms(&mut *tx, &job.id, &job.workspace_id) - .await? - .map(|x| x.into()); - - let (email, permissioned_as) = - if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { - (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) - } else { - (&job.permissioned_as_email, job.permissioned_as.to_owned()) - }; - - let job_priority = tool.module.priority.or(job.priority); - - let tx = PushIsolationLevel::Transaction(tx); - let (uuid, tx) = push( - db, - tx, - &job.workspace_id, - job_payload.payload, - PushArgs { args: &tool_call_args, extra: None }, - &job.created_by, - email, - permissioned_as, - Some(&format!("job-span-{}", job.id)), - None, - job.schedule_path(), - Some(job.id), - None, - None, - Some(job_id), - false, - false, - None, - job.visible_to_owner, - Some(job.tag.clone()), // we reuse the same tag as the agent job because it's run on the same worker - job_payload.timeout, - None, - job_priority, - job_perms.as_ref(), - true, - ) - .await?; - - tx.commit().await?; - - let tool_job = get_mini_pulled_job(db, &uuid).await?; - - let Some(tool_job) = tool_job else { - return Err(Error::internal_err("Tool job not found".to_string())); - }; - - let tool_job = Arc::new(tool_job); - - let job_dir = create_job_dir(&worker_dir, job.id).await; - - let (inner_job_completed_tx, inner_job_completed_rx) = - JobCompletedSender::new(&conn, 1); - - let inner_job_completed_rx = inner_job_completed_rx.expect( - "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", - ); - - #[cfg(feature = "benchmark")] - let mut bench = BenchmarkIter::new(); - - match handle_queued_job( - tool_job.clone(), - None, - None, - None, - None, - conn, - client, - hostname, - worker_name, - worker_dir, - &job_dir, - None, - base_internal_url, - inner_job_completed_tx, - occupancy_metrics, - killpill_rx, - None, - #[cfg(feature = "benchmark")] - &mut bench, - ) - .await - { - Err(err) => { - let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = error_to_value(&err); - let _ = handle_non_flow_job_error( - db, - &tool_job, - 0, - None, - err_string.clone(), - err_json, - worker_name, - ) - .await; - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(format!( - "Error running tool: {}", - err_string - ))), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success( - db, parent_job, false, - ) - .await?; - } - Ok(success) => { - let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); - - let result = if let Some(SendResult { - result: - SendResultPayload::JobCompleted(JobCompleted { result, .. }), - .. - }) = send_result.as_ref() - { - job_completed_tx - .send(send_result.as_ref().unwrap().result.clone(), true) - .await - .map_err(to_anyhow)?; - result - } else { - if let Some(send_result) = send_result { - job_completed_tx - .send(send_result.result, true) - .await - .map_err(to_anyhow)?; - } - return Err(Error::internal_err( - "Tool job completed but no result".to_string(), - )); - }; - - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(result.get().to_string())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool.module.id.clone(), - }), - ..Default::default() - }); - update_flow_status_module_with_actions_success( - db, parent_job, success, - ) - .await?; - } - } - } else { - return Err(Error::internal_err(format!( - "Tool not found: {}", - tool_call.function.name - ))); - } - } - } - - let final_messages: Vec = messages - .iter() - .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) - .collect(); - - // Parse content as JSON, fallback to string if it fails - let output_value = match content { - Some(content_str) => match has_output_properties { - true => match content_str { - OpenAIContent::Text(text) => serde_json::from_str::>(&text) - .map_err(|_e| { - Error::internal_err(format!( - "Failed to parse structured output: {}", - text - )) - }), - // No need to handle this, it will always be a text string - OpenAIContent::Parts(_parts) => Err(Error::internal_err( - "Failed to parse structured output".to_string(), - )), - }, - false => Ok(match content_str { - OpenAIContent::Text(text) => to_raw_value(&text), - OpenAIContent::Parts(parts) => to_raw_value(&parts), - }), - }?, - None => to_raw_value(&""), - }; - - Ok(to_raw_value(&AIAgentResult { - output: output_value, - messages: final_messages, - })) - } - } -} - pub struct FlowJobRunnableIdAndRawFlow { pub runnable_id: Option, pub raw_flow: Option>>, @@ -1918,3 +297,641 @@ pub async fn handle_ai_agent_job( Ok(result) } + +/// Find a unique tool name to avoid collisions with user-provided tools +fn find_unique_tool_name(base_name: &str, existing_tools: Option<&[ToolDef]>) -> String { + let Some(tools) = existing_tools else { + return base_name.to_string(); + }; + + if !tools.iter().any(|t| t.function.name == base_name) { + return base_name.to_string(); + } + + for i in 1..100 { + let candidate = format!("{}_{}", base_name, i); + if !tools.iter().any(|t| t.function.name == candidate) { + return candidate; + } + } + + // Fallback with process id if somehow we can't find a unique name + format!("{}_{}_fallback", base_name, std::process::id()) +} + +async fn update_flow_status_module_with_actions( + db: &DB, + parent_job: &Uuid, + actions: &[AgentAction], +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step { idx: step, .. } => { + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $3::TEXT, 'agent_actions'], + $2 + ) + WHERE id = $1 + "#, + parent_job, + sqlx::types::Json(actions) as _, + step as i32 + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +async fn update_flow_status_module_with_actions_success( + db: &DB, + parent_job: &Uuid, + action_success: bool, +) -> Result<(), Error> { + let step = get_step_of_flow_status(db, parent_job.to_owned()).await?; + match step { + Step::Step { idx: step, .. } => { + // Append the new bool to the existing array, or create a new array if it doesn't exist + sqlx::query!( + r#" + UPDATE v2_job_status SET + flow_status = jsonb_set( + flow_status, + array['modules', $2::TEXT, 'agent_actions_success'], + COALESCE( + flow_status->'modules'->$2->'agent_actions_success', + to_jsonb(ARRAY[]::bool[]) + ) || to_jsonb(ARRAY[$3::bool]) + ) + WHERE id = $1 + "#, + parent_job, + step as i32, + action_success + ) + .execute(db) + .await?; + } + _ => {} + } + Ok(()) +} + +/// Check if the provider is Anthropic (either direct or through OpenRouter) +fn is_anthropic_provider(provider: &ProviderWithResource) -> bool { + let provider_is_anthropic = provider.kind.is_anthropic(); + let is_openrouter_anthropic = + provider.kind == AIProvider::OpenRouter && provider.model.starts_with("anthropic/"); + provider_is_anthropic || is_openrouter_anthropic +} + +#[async_recursion] +pub async fn run_agent( + // connection + db: &DB, + conn: &Connection, + + // agent job and flow data + job: &MiniPulledJob, + parent_job: &Uuid, + args: &AIAgentArgs, + tools: &[Tool], + + // job execution context + client: &AuthedClient, + occupancy_metrics: &mut OccupancyMetrics, + job_completed_tx: &JobCompletedSender, + worker_dir: &str, + base_internal_url: &str, + worker_name: &str, + hostname: &str, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, +) -> error::Result> { + let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); + let base_url = args.provider.get_base_url(db).await?; + let api_key = args.provider.get_api_key(); + + // Create the query builder for the provider + let query_builder = create_query_builder(&args.provider); + + // Initialize messages + let mut messages = + if let Some(system_prompt) = args.system_prompt.clone().filter(|s| !s.is_empty()) { + vec![OpenAIMessage { + role: "system".to_string(), + content: Some(OpenAIContent::Text(system_prompt)), + ..Default::default() + }] + } else { + vec![] + }; + + // Create user message with optional images + let mut parts = vec![ContentPart::Text { text: args.user_message.clone() }]; + if let Some(images) = &args.user_images { + for image in images.iter() { + if !image.s3.is_empty() { + parts.push(ContentPart::S3Object { s3_object: image.clone() }); + } + } + } + let user_content = OpenAIContent::Parts(parts); + + messages.push(OpenAIMessage { + role: "user".to_string(), + content: Some(user_content), + ..Default::default() + }); + + let mut actions = vec![]; + let mut content = None; + + // Check if this provider supports tools with the current output type + let supports_tools = query_builder.supports_tools_with_output_type(output_type); + + let mut tool_defs: Option> = if tools.is_empty() || !supports_tools { + None + } else { + Some(tools.iter().map(|t| t.def.clone()).collect()) + }; + + // Handle structured output schema + let has_output_properties = args + .output_schema + .as_ref() + .and_then(|schema| schema.properties.as_ref()) + .map(|props| !props.is_empty()) + .unwrap_or(false); + + let is_anthropic = is_anthropic_provider(&args.provider); + let mut used_structured_output_tool = false; + let mut structured_output_tool_name: Option = None; + + // For text output with schema, handle structured output + if has_output_properties && output_type == &OutputType::Text { + let schema = args.output_schema.as_ref().unwrap(); + if is_anthropic { + // Anthropic uses a tool for structured output + let unique_tool_name = find_unique_tool_name("structured_output", tool_defs.as_deref()); + structured_output_tool_name = Some(unique_tool_name.clone()); + + let output_tool = ToolDef { + r#type: "function".to_string(), + function: ToolDefFunction { + name: unique_tool_name, + description: Some( + "This tool MUST be used last to return a structured JSON object as the final output." + .to_string(), + ), + parameters: to_raw_value(&schema), + }, + }; + if let Some(ref mut existing_tools) = tool_defs { + existing_tools.push(output_tool); + } else { + tool_defs = Some(vec![output_tool]); + } + } + // For non-Anthropic providers, response_format is handled by the query builder + } + + // Main agent loop + for i in 0..MAX_AGENT_ITERATIONS { + if used_structured_output_tool { + break; + } + + // For text output or image output with tools + let build_args = BuildRequestArgs { + messages: &messages, + tools: tool_defs.as_deref(), + model: args.provider.get_model(), + temperature: args.temperature, + max_tokens: args.max_completion_tokens, + output_schema: args.output_schema.as_ref(), + output_type, + system_prompt: args.system_prompt.as_deref(), + user_message: &args.user_message, + images: args.user_images.as_deref(), + }; + + let request_body = query_builder + .build_request(&build_args, client, &job.workspace_id) + .await?; + + let endpoint = + query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); + let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); + + let mut request = HTTP_CLIENT + .post(&endpoint) + .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECONDS)) + .header("Content-Type", "application/json"); + + // Apply authentication headers + for (header_name, header_value) in auth_headers { + request = request.header(header_name, header_value); + } + + if args.provider.kind.is_azure_openai(&base_url) { + request = request.query(&[("api-version", AZURE_API_VERSION)]) + } + + let resp = request + .body(request_body) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; + + match resp.error_for_status_ref() { + Ok(_) => { + let parsed = query_builder.parse_response(resp).await?; + + match parsed { + ParsedResponse::Text { content: response_content, tool_calls } => { + if let Some(ref response_content) = response_content { + actions.push(AgentAction::Message {}); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text(response_content.clone())), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + + update_flow_status_module_with_actions(db, parent_job, &actions) + .await?; + update_flow_status_module_with_actions_success(db, parent_job, true) + .await?; + + content = Some(OpenAIContent::Text(response_content.clone())); + } + + if tool_calls.is_empty() { + break; + } else if i == MAX_AGENT_ITERATIONS - 1 { + return Err(Error::internal_err( + "AI agent reached max iterations, but there are still tool calls" + .to_string(), + )); + } + + messages.push(OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(tool_calls.clone()), + ..Default::default() + }); + + // Handle tool calls (keeping existing tool execution logic) + for tool_call in tool_calls.iter() { + // Check if this is the structured output tool + if structured_output_tool_name + .as_ref() + .map_or(false, |name| tool_call.function.name == *name) + { + used_structured_output_tool = true; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text( + "Successfully ran structured_output tool".to_string(), + )), + tool_call_id: Some(tool_call.id.clone()), + ..Default::default() + }); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text( + tool_call.function.arguments.clone(), + )), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + content = + Some(OpenAIContent::Text(tool_call.function.arguments.clone())); + break; + } + + // Execute regular tool + let tool = tools + .iter() + .find(|t| t.def.function.name == tool_call.function.name); + if let Some(tool) = tool { + let job_id = ulid::Ulid::new().into(); + actions.push(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }); + + update_flow_status_module_with_actions(db, parent_job, &actions) + .await?; + + let tool_call_args = + serde_json::from_str::>>( + &tool_call.function.arguments, + )?; + + let job_payload = match tool.module.get_value()? { + FlowModuleValue::Script { + path: script_path, + hash: script_hash, + tag_override, + .. + } => { + let payload = script_to_payload( + script_hash, + script_path, + db, + job, + &tool.module, + tag_override, + tool.module.apply_preprocessor, + ) + .await?; + payload + } + FlowModuleValue::RawScript { + path, + content, + language, + lock, + tag, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + .. + } => { + let path = path.unwrap_or_else(|| { + format!( + "{}/tools/{}", + job.runnable_path(), + tool.module.id + ) + }); + + let payload = raw_script_to_payload( + path, + content, + language, + lock, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + &tool.module, + tag, + tool.module.delete_after_use.unwrap_or(false), + ); + payload + } + _ => { + return Err(Error::internal_err(format!( + "Unsupported tool: {}", + tool_call.function.name + ))); + } + }; + + let mut tx = db.begin().await?; + + let job_perms = windmill_common::auth::get_job_perms( + &mut *tx, + &job.id, + &job.workspace_id, + ) + .await? + .map(|x| x.into()); + + let (email, permissioned_as) = + if let Some(on_behalf_of) = job_payload.on_behalf_of.as_ref() { + (&on_behalf_of.email, on_behalf_of.permissioned_as.clone()) + } else { + (&job.permissioned_as_email, job.permissioned_as.to_owned()) + }; + + let job_priority = tool.module.priority.or(job.priority); + + let tx = PushIsolationLevel::Transaction(tx); + let (uuid, tx) = push( + db, + tx, + &job.workspace_id, + job_payload.payload, + PushArgs { args: &tool_call_args, extra: None }, + &job.created_by, + email, + permissioned_as, + Some(&format!("job-span-{}", job.id)), + None, + job.schedule_path(), + Some(job.id), + None, + None, + Some(job_id), + false, + false, + None, + job.visible_to_owner, + Some(job.tag.clone()), + job_payload.timeout, + None, + job_priority, + job_perms.as_ref(), + true, + ) + .await?; + + tx.commit().await?; + + let tool_job = get_mini_pulled_job(db, &uuid).await?; + + let Some(tool_job) = tool_job else { + return Err(Error::internal_err( + "Tool job not found".to_string(), + )); + }; + + let tool_job = Arc::new(tool_job); + + let job_dir = create_job_dir(&worker_dir, job.id).await; + + let (inner_job_completed_tx, inner_job_completed_rx) = + JobCompletedSender::new(&conn, 1); + + let inner_job_completed_rx = inner_job_completed_rx.expect( + "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", + ); + + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + + match handle_queued_job( + tool_job.clone(), + None, + None, + None, + None, + conn, + client, + hostname, + worker_name, + worker_dir, + &job_dir, + None, + base_internal_url, + inner_job_completed_tx, + occupancy_metrics, + killpill_rx, + None, + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await + { + Err(err) => { + let err_string = + format!("{}: {}", err.name(), err.to_string()); + let err_json = error_to_value(&err); + let _ = handle_non_flow_job_error( + db, + &tool_job, + 0, + None, + err_string.clone(), + err_json, + worker_name, + ) + .await; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(format!( + "Error running tool: {}", + err_string + ))), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }), + ..Default::default() + }); + update_flow_status_module_with_actions_success( + db, parent_job, false, + ) + .await?; + } + Ok(success) => { + let send_result = + inner_job_completed_rx.bounded_rx.try_recv().ok(); + + let result = if let Some(SendResult { + result: + SendResultPayload::JobCompleted(JobCompleted { + result, + .. + }), + .. + }) = send_result.as_ref() + { + job_completed_tx + .send( + send_result.as_ref().unwrap().result.clone(), + true, + ) + .await + .map_err(to_anyhow)?; + result + } else { + if let Some(send_result) = send_result { + job_completed_tx + .send(send_result.result, true) + .await + .map_err(to_anyhow)?; + } + return Err(Error::internal_err( + "Tool job completed but no result".to_string(), + )); + }; + + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text( + result.get().to_string(), + )), + tool_call_id: Some(tool_call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: tool_call.function.name.clone(), + module_id: tool.module.id.clone(), + }), + ..Default::default() + }); + update_flow_status_module_with_actions_success( + db, parent_job, success, + ) + .await?; + } + } + } else { + return Err(Error::internal_err(format!( + "Tool not found: {}", + tool_call.function.name + ))); + } + } + } + ParsedResponse::Image { base64_data } => { + // For image output with tools, we got an image response + let s3_object = upload_image_to_s3(&base64_data, job, client).await?; + return Ok(to_raw_value(&s3_object)); + } + } + } + Err(e) => { + let _status = resp.status(); + let text = resp + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::internal_err(format!("API error: {} - {}", e, text))); + } + } + } + + // Return the final result + let final_messages: Vec = messages + .iter() + .map(|m| Message { message: m, agent_action: m.agent_action.as_ref() }) + .collect(); + + // Parse content as JSON for structured output, fallback to string if it fails + let output_value = match content { + Some(content_str) => match has_output_properties { + true => match content_str { + OpenAIContent::Text(text) => { + serde_json::from_str::>(&text).map_err(|_e| { + Error::internal_err(format!("Failed to parse structured output: {}", text)) + }) + } + OpenAIContent::Parts(_parts) => Err(Error::internal_err( + "Failed to parse structured output".to_string(), + )), + }, + false => Ok(match content_str { + OpenAIContent::Text(text) => to_raw_value(&text), + OpenAIContent::Parts(parts) => to_raw_value(&parts), + }), + }?, + None => to_raw_value(&""), + }; + + Ok(to_raw_value(&AIAgentResult { + output: output_value, + messages: final_messages, + })) +} diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 72640eb9da..04567b6217 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -93,6 +93,7 @@ async fn clone_repo( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -112,7 +113,8 @@ async fn clone_repo( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str(), false).await?; + let checkout_cmd_child = + start_child_process(checkout_cmd, GIT_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -127,6 +129,7 @@ async fn clone_repo( false, &mut Some(occupancy_metrics), None, + None, ) .await?; } @@ -232,6 +235,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -249,7 +253,8 @@ async fn clone_repo_without_history( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let add_remote_cmd_child = start_child_process(add_remote_cmd, GIT_PATH.as_str(), false).await?; + let add_remote_cmd_child = + start_child_process(add_remote_cmd, GIT_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -264,6 +269,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -296,6 +302,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -328,6 +335,7 @@ async fn clone_repo_without_history( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -462,6 +470,7 @@ pub async fn install_galaxy_collections( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -484,7 +493,8 @@ pub async fn install_galaxy_collections( .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let child = start_child_process(galaxy_collections_cmd, ANSIBLE_GALAXY_PATH.as_str(), false).await?; + let child = + start_child_process(galaxy_collections_cmd, ANSIBLE_GALAXY_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -499,6 +509,7 @@ pub async fn install_galaxy_collections( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -1133,6 +1144,7 @@ fi false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_and_check_result(job_dir).await diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index b224957958..e55ee7eeaa 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -231,6 +231,7 @@ exit $exit_status true, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -647,6 +648,7 @@ pub async fn handle_powershell_job( false, &mut Some(occupancy_metrics), None, + None, ) .await?; } @@ -871,6 +873,7 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", false, &mut Some(occupancy_metrics), None, + None, ) .await?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index dce084bdf4..4016010b63 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -18,6 +18,7 @@ use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, + StreamNotifier, }, handle_child::handle_child, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_NO_CACHE, BUN_PATH, @@ -168,6 +169,7 @@ pub async fn gen_bun_lockfile( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -374,6 +376,7 @@ pub async fn install_bun_lockfile( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -546,6 +549,7 @@ pub async fn generate_wrapper_mjs( false, occupancy_metrics, None, + None, ) .await?; fs::rename( @@ -597,6 +601,7 @@ pub async fn generate_bun_bundle( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -1308,6 +1313,8 @@ try {{ append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), conn).await; + let stream_notifier = StreamNotifier::new(conn, job); + let result = crate::js_eval::eval_fetch_timeout( env_code, inner_content.clone(), @@ -1323,6 +1330,7 @@ try {{ &job.workspace_id, false, occupancy_metrics, + stream_notifier, ) .await?; tracing::info!( @@ -1465,6 +1473,8 @@ try {{ .await? }; + let stream_notifier = StreamNotifier::new(conn, job); + let handle_result = handle_child( &job.id, conn, @@ -1479,6 +1489,7 @@ try {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 723d99e062..a90d14ca46 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -14,6 +14,7 @@ use sqlx::{Pool, Postgres}; use tokio::process::Command; use tokio::{fs::File, io::AsyncReadExt}; +use windmill_common::flows::Step; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::{ get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object, @@ -32,8 +33,10 @@ use windmill_common::{ use anyhow::{anyhow, Result}; use windmill_parser_sql::{s3_mode_extension, S3ModeArgs, S3ModeFormat}; +use windmill_queue::flow_status::get_step_of_flow_status; use windmill_queue::MiniPulledJob; +use std::collections::HashSet; use std::path::Path; use std::{collections::HashMap, sync::Arc, time::Duration}; @@ -261,7 +264,11 @@ pub async fn transform_json_value( ) .await?; decrypt(&mc, encrypted.to_string()).and_then(|x| { - serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string())) + serde_json::from_str(&x).map_err(|e| { + Error::internal_err(format!( + "Failed to decrypt '$encrypted:' value: {e}" + )) + }) }) } Connection::Http(_) => { @@ -1061,6 +1068,149 @@ pub fn get_root_job_id(job: &MiniPulledJob) -> uuid::Uuid { .unwrap_or(job.id) } +#[derive(Clone)] +pub struct StreamNotifier { + db: DB, + job_id: uuid::Uuid, + parent_job: uuid::Uuid, + root_job: uuid::Uuid, +} + +#[async_recursion] +async fn check_if_nested_step_is_last( + db: &DB, + parent_job: Uuid, + parent_of_parent_job: Option, + root_job: Uuid, + visited: Option>, +) -> error::Result { + // Initialize or use the provided visited set for cycle detection + let mut visited = visited.unwrap_or_else(HashSet::new); + + // Check for cycles - if we've already visited this job, return false to break the recursion + if !visited.insert(parent_job) { + return Ok(false); + } + + // get parent of parent job to get step of parent job + let parent_of_parent_job = parent_of_parent_job.or(sqlx::query_scalar!( + "SELECT parent_job FROM v2_job WHERE id = $1", + parent_job + ) + .fetch_one(db) + .await?); + if let Some(parent_of_parent_job) = parent_of_parent_job { + // Check for cycles again with the parent_of_parent_job + if !visited.insert(parent_of_parent_job) { + return Ok(false); + } + + let r = sqlx::query!( + r#"SELECT + (flow_status->'step')::integer as step, + jsonb_array_length(flow_status->'modules') as len, + flow_status->'modules'->-1->>'branch_chosen' IS NOT NULL as is_branch_one, + parent_job as ppp_job + FROM v2_job + LEFT JOIN v2_job_status USING (id) + WHERE v2_job.id = $1"#, + parent_of_parent_job + ) + .fetch_one(db) + .await + .map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?; + + if let Some(step) = r.step { + let step = Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize); + + // if parent job is last and a branch one and + // - root_job is equal to parent of parent job, return true + // - root job is not equal to parent of parent job, recursively check if the parent of parent job is a branch one and last + if step.is_last_step() && r.is_branch_one.unwrap_or(false) { + if parent_of_parent_job == root_job { + return Ok(true); + } else { + return check_if_nested_step_is_last( + db, + parent_of_parent_job, + r.ppp_job, + root_job, + Some(visited), + ) + .await; + } + } + } + } + + Ok(false) +} + +impl StreamNotifier { + pub fn new(conn: &Connection, job: &MiniPulledJob) -> Option { + let root_job = get_root_job_id(job); + if job.is_flow_step() && job.parent_job.is_some() { + match conn { + Connection::Sql(db) => Some(Self { + db: db.clone(), + parent_job: job.parent_job.unwrap(), + job_id: job.id, + root_job, + }), + Connection::Http(_) => { + tracing::warn!( + "Flow job streaming is only supported for workers connected to a database" + ); + None + } + } + } else { + None + } + } + + async fn update_flow_status_with_stream_job_inner( + db: DB, + parent_job: Uuid, + job_id: Uuid, + root_job: Uuid, + ) -> Result<(), Error> { + let step = get_step_of_flow_status(&db, parent_job).await?; + + if step.is_last_step() + && (parent_job == root_job + || check_if_nested_step_is_last(&db, parent_job, None, root_job, None).await?) + { + sqlx::query!(r#" + UPDATE v2_job_status + SET flow_status = jsonb_set(flow_status, array['stream_job'], to_jsonb($1::UUID::TEXT)) + WHERE id = $2"#, + job_id, + root_job + ) + .execute(&db) + .await?; + } + + Ok(()) + } + + pub fn update_flow_status_with_stream_job(&self) -> () { + let db = self.db.clone(); + let parent_job = self.parent_job; + let job_id = self.job_id; + let root_job = self.root_job; + tokio::spawn(async move { + if let Err(err) = + Self::update_flow_status_with_stream_job_inner(db, parent_job, job_id, root_job) + .await + { + tracing::error!("Could not notify about stream job {}: {err:#?}", parent_job); + } + }); + } +} + #[derive(Clone)] pub struct S3ModeWorkerData { pub client: AuthedClient, @@ -1120,4 +1270,3 @@ pub fn s3_mode_args_to_worker_data( workspace_id: job.workspace_id.clone(), } } - diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 84cc3184a3..51bdf2f8e8 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -129,6 +129,7 @@ pub async fn generate_nuget_lockfile( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -386,6 +387,7 @@ async fn build_cs_proj( false, &mut Some(occupancy_metrics), None, + None, ) .await?; append_logs(job_id, w_id, "\n\n", conn).await; @@ -643,6 +645,7 @@ pub async fn handle_csharp_job( false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 431a457d74..0e15e5ba12 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -8,7 +8,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, - start_child_process, OccupancyMetrics, + start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, @@ -161,6 +161,7 @@ pub async fn generate_deno_lock( false, occupancy_metrics, None, + None, ) .await?; } else { @@ -417,6 +418,9 @@ try {{ .stderr(Stdio::piped()); start_child_process(deno_cmd, DENO_PATH.as_str(), false).await? }; + + let stream_notifier = StreamNotifier::new(conn, job); + // logs.push_str(format!("prepare: {:?}\n", start.elapsed().as_micros()).as_str()); // start = Instant::now(); let handle_result = handle_child( @@ -433,6 +437,7 @@ try {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; // logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str()); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 85adbf7fea..7a758051ab 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -274,6 +274,7 @@ func Run(req Req) (interface{{}}, error){{ false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -422,6 +423,7 @@ func Run(req Req) (interface{{}}, error){{ false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -507,6 +509,7 @@ pub async fn install_go_dependencies( false, &mut Some(occupation_metrics), None, + None, ) .await?; @@ -622,6 +625,7 @@ pub async fn install_go_dependencies( false, &mut Some(occupation_metrics), None, + None, ) .await?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 11078c7892..6d36b8ca0d 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -29,7 +29,7 @@ use windmill_queue::{append_logs, CanceledBy}; use std::os::unix::process::ExitStatusExt; use std::process::ExitStatus; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; use std::{io, panic, time::Duration}; @@ -52,7 +52,7 @@ use futures::{ stream, StreamExt, }; -use crate::common::{resolve_job_timeout, OccupancyMetrics}; +use crate::common::{resolve_job_timeout, OccupancyMetrics, StreamNotifier}; use crate::job_logger::{append_job_logs, append_result_stream, append_with_limit}; use crate::job_logger_oss::process_streaming_log_lines; use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; @@ -114,6 +114,7 @@ pub async fn handle_child( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, // Do not print logs to output, but instead save to string. pipe_stdout: Option<&mut String>, + stream_notifier: Option, ) -> error::Result { let start = Instant::now(); @@ -315,6 +316,7 @@ pub async fn handle_child( &mut rx2, child_name, &mut stream_result, + stream_notifier, ) .instrument(trace_span!("child_lines")); @@ -354,6 +356,7 @@ pub async fn write_lines( rx2: &mut broadcast::Receiver<()>, child_name: &str, stream_result: &mut Vec, + stream_notifier: Option, ) { let max_log_size = if *CLOUD_HOSTED { MAX_RESULT_SIZE @@ -384,6 +387,7 @@ pub async fn write_lines( let mut pipe_stdout = pipe_stdout; + let is_stream = Arc::new(AtomicBool::new(false)); while let Some(line) = output.by_ref().next().await { let do_write_ = do_write.shared(); @@ -410,6 +414,7 @@ pub async fn write_lines( let job_id = job_id.clone(); let mut nstream = String::new(); + while let Some(line) = read_lines.next().await { match line { Ok(line) => { @@ -479,8 +484,17 @@ pub async fn write_lines( let w_id = w_id.to_string(); let job_id = job_id.clone(); let pg_log_total_size = pg_log_total_size.clone(); + let stream_notifier = stream_notifier.clone(); + let is_stream = is_stream.clone(); (do_write, write_result) = tokio::spawn(async move { if !nstream.is_empty() { + if let Some(stream_notifier) = stream_notifier { + if !is_stream.load(Ordering::SeqCst) { + is_stream.store(true, Ordering::SeqCst); + stream_notifier.update_flow_status_with_stream_job(); + } + } + if let Err(err) = append_result_stream(&conn, &w_id, &job_id, &nstream).await { tracing::error!( "Unable to send result stream for job {job_id}. Error was: {:?}", diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 6aa12d5024..d133308489 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -534,6 +534,7 @@ async fn compile<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -718,6 +719,7 @@ async fn run<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index 7f6fa8146c..cbf19b5946 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -48,7 +48,7 @@ use windmill_common::worker::{write_file, TMP_DIR}; use windmill_common::flow_status::JobResult; use windmill_queue::CanceledBy; -use crate::common::OccupancyMetrics; +use crate::common::{OccupancyMetrics, StreamNotifier}; use windmill_common::client::AuthedClient; #[cfg(feature = "deno_core")] @@ -769,6 +769,7 @@ pub async fn eval_fetch_timeout( _w_id: &str, _load_client: bool, _occupation_metrics: &mut OccupancyMetrics, + _stream_notifier: Option, ) -> anyhow::Result> { use serde_json::value::to_raw_value; Ok(to_raw_value("require deno_core").unwrap()) @@ -790,6 +791,7 @@ pub async fn eval_fetch_timeout( w_id: &str, load_client: bool, occupation_metrics: &mut OccupancyMetrics, + stream_notifier: Option, ) -> windmill_common::error::Result> { let (sender, mut receiver) = oneshot::channel::(); let (append_logs_sender, mut append_logs_receiver) = mpsc::unbounded_channel::(); @@ -940,10 +942,18 @@ pub async fn eval_fetch_timeout( } let handle = tokio::spawn(async move { let mut result_stream = String::new(); + let mut is_stream = false; while let Some(log) = log_receiver.recv().await { use windmill_common::result_stream::extract_stream_from_logs; if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { + if let Some(sn) = stream_notifier.as_ref() { + if !is_stream { + is_stream = true; + sn.update_flow_status_with_stream_job(); + } + } + result_stream.push_str(&stream); if let Err(e) = result_stream_sender.send(stream) { tracing::error!("failed to send result stream: {e}"); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index ace218c207..87f8282e9e 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -16,6 +16,7 @@ mod java_executor; #[cfg(feature = "ruby")] mod ruby_executor; +mod ai; mod ai_executor; mod bun_executor; pub mod common; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index a52270e745..2497757d9d 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -340,6 +340,7 @@ async fn run<'a>( false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index cdfd4cfe6f..b272d631da 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -106,6 +106,7 @@ pub async fn composer_install( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -343,6 +344,7 @@ try {{ false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index c41089652b..57936969b8 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -121,7 +121,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, read_file, read_result, - start_child_process, OccupancyMetrics, + start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, worker_utils::ping_job_status, @@ -386,6 +386,7 @@ pub async fn uv_pip_compile( false, occupancy_metrics, None, + None, ) .await .map_err(|e| { @@ -864,6 +865,8 @@ mount {{ start_child_process(python_cmd, &python_path, false).await? }; + let stream_notifier = StreamNotifier::new(conn, job); + let handle_result = handle_child( &job.id, conn, @@ -878,6 +881,7 @@ mount {{ false, &mut Some(occupancy_metrics), None, + stream_notifier, ) .await?; diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index 3d71e9f9ec..2c5b828313 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -655,6 +655,7 @@ impl PyV { false, occupancy_metrics, None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index fb087a9d24..b635a6bade 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -420,6 +420,7 @@ Your Gemfile syntax will continue to work as-is." &mut None, // Some(&mut stdout), None, + None, ) .await?; @@ -871,6 +872,7 @@ mount {{ false, &mut Some(occupancy_metrics), None, + None, ) .await?; Ok(()) diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index f47399c54a..6a05de459c 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -175,7 +175,8 @@ pub async fn generate_cargo_lockfile( std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); } - let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str(), false).await?; + let gen_lockfile_process = + start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str(), false).await?; handle_child( job_id, conn, @@ -190,6 +191,7 @@ pub async fn generate_cargo_lockfile( false, &mut Some(occupancy_metrics), None, + None, ) .await?; @@ -298,6 +300,7 @@ async fn get_build_dir( false, &mut None, None, + None, ) .await } @@ -410,6 +413,7 @@ pub async fn build_rust_crate( false, &mut Some(occupancy_metrics), None, + None, ) .await?; append_logs(&job.id, &job.workspace_id, "\n\n", conn).await; @@ -599,6 +603,7 @@ pub async fn handle_rust_job( false, &mut Some(occupancy_metrics), None, + None, ) .await?; read_result(job_dir, None).await diff --git a/backend/windmill-worker/src/universal_pkg_installer.rs b/backend/windmill-worker/src/universal_pkg_installer.rs index dc4b5dab2a..3f3198339a 100644 --- a/backend/windmill-worker/src/universal_pkg_installer.rs +++ b/backend/windmill-worker/src/universal_pkg_installer.rs @@ -159,6 +159,7 @@ pub async fn par_install_language_dependencies_all_at_once< false, &mut None, pipe_stdout, + None, ) .await { @@ -587,6 +588,7 @@ async fn try_install_one_detached<'a, T: Clone + std::marker::Send + Sync + 'a + false, &mut None, None, + None, ) .await { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 6532c1bb22..2ae1a42f74 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -101,6 +101,7 @@ use tokio::{ use rand::Rng; use crate::ai_executor::handle_ai_agent_job; +use crate::common::StreamNotifier; use crate::{ agent_workers::{queue_init_job, queue_periodic_job}, bash_executor::{handle_bash_job, handle_powershell_job}, @@ -2229,6 +2230,8 @@ async fn do_nativets( job.args.as_ref() }; + let stream_notifier = StreamNotifier::new(conn, job); + Ok(eval_fetch_timeout( env_code, code.clone(), @@ -2244,6 +2247,7 @@ async fn do_nativets( &job.workspace_id, true, occupancy_metrics, + stream_notifier, ) .await?) } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 0ccebcf477..201924e825 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -328,7 +328,7 @@ pub async fn update_flow_status_after_job_completion_internal( let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); let current_module = match module_step { - Step::Step(i) => flow_value.modules.get(i), + Step::Step { idx: i, .. } => flow_value.modules.get(i), _ => None, }; @@ -342,7 +342,7 @@ pub async fn update_flow_status_after_job_completion_internal( .as_ref() .ok_or_else(|| Error::internal_err(format!("preprocessor module not found")))?, Step::FailureStep => &old_status.failure_module.module_status, - Step::Step(i) => old_status + Step::Step { idx: i, .. } => old_status .modules .get(i as usize) .ok_or_else(|| Error::internal_err(format!("module {i} not found")))?, @@ -1984,7 +1984,7 @@ async fn push_next_flow_job( tracing::info!(id = %flow_job.id, root_id = %job_root, step = ?step, "pushing next flow job"); let mut status_module = match step { - Step::Step(i) => status + Step::Step { idx: i, .. } => status .modules .get(i) .cloned() @@ -2035,7 +2035,7 @@ async fn push_next_flow_job( }))); } - if matches!(step, Step::Step(0)) { + if matches!(step, Step::Step { idx: 0, .. }) { if !flow_job.is_flow_step() && flow_job.schedule_path().is_some() { let schedule_path = flow_job.schedule_path(); let no_flow_overlap = sqlx::query_scalar!( @@ -2125,7 +2125,7 @@ async fn push_next_flow_job( let arc_last_job_result = if status_module.is_failure() { // if job is being retried, pass the result of its previous failure last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}")))) - } else if matches!(step, Step::Step(0)) || step.is_preprocessor_step() { + } else if matches!(step, Step::Step { idx: 0, .. }) || step.is_preprocessor_step() { // if it's the first job executed in the flow, pass the flow args Arc::new(to_raw_value(&flow_job.args)) } else { @@ -2444,7 +2444,9 @@ async fn push_next_flow_job( let current_id = &module.id; let mut previous_id = match step { - Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(), + Step::Step { idx: i, .. } if i >= 1 => { + flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() + } _ => String::new(), }; @@ -2458,7 +2460,7 @@ async fn push_next_flow_job( ) { None } else { - let sleep_input_transform = if let Step::Step(i) = step { + let sleep_input_transform = if let Step::Step { idx: i, .. } = step { i.checked_sub(1) .and_then(|i| flow.modules.get(i)) .and_then(|m| m.sleep.clone()) @@ -3272,7 +3274,7 @@ async fn push_next_flow_job( .warn_after_seconds(3) .await?; } - Step::Step(i) => { + Step::Step { idx: i, .. } => { sqlx::query!( "UPDATE v2_job_status SET flow_status = JSONB_SET( diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 3c0cdbe3dd..6fc84ad172 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.543.0"; +export const VERSION = "v1.544.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index c4bbac2f9b..fc3e7f57e4 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.543.0"; +export const VERSION = "1.544.1"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4b40a2bbff..564fc8780f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.543.0", + "version": "1.544.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.543.0", + "version": "1.544.1", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index c026d14c6c..d0ac4dde1a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.543.0", + "version": "1.544.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/frontend/src/lib/components/DropdownV2.svelte b/frontend/src/lib/components/DropdownV2.svelte index 9415635bab..34404779ad 100644 --- a/frontend/src/lib/components/DropdownV2.svelte +++ b/frontend/src/lib/components/DropdownV2.svelte @@ -18,6 +18,7 @@ import { twMerge } from 'tailwind-merge' import { triggerableByAI } from '$lib/actions/triggerableByAI.svelte' import { untrack } from 'svelte' + import { fly } from 'svelte/transition' interface Props { aiId?: string | undefined @@ -33,6 +34,7 @@ customWidth?: number | undefined customMenu?: boolean class?: string | undefined + enableFlyTransition?: boolean buttonReplacement?: import('svelte').Snippet menu?: import('svelte').Snippet } @@ -51,6 +53,7 @@ customWidth = undefined, customMenu = false, class: classNames = undefined, + enableFlyTransition = false, buttonReplacement, menu }: Props = $props() @@ -148,7 +151,12 @@ {#if open && !hidePopup} -
+
{#if customMenu} {@render menu?.()} {:else} diff --git a/frontend/src/lib/components/EditableSchemaForm.svelte b/frontend/src/lib/components/EditableSchemaForm.svelte index 297ee73ad0..9c91888960 100644 --- a/frontend/src/lib/components/EditableSchemaForm.svelte +++ b/frontend/src/lib/components/EditableSchemaForm.svelte @@ -186,7 +186,7 @@ let index = 0 let hasChanged = false for (let k of properties) { - if (schema.properties[k].type === 'object' && schema.properties[k].properties) { + if (schema.properties[k]?.type === 'object' && schema.properties[k].properties) { hasChanged = hasChanged || alignOrderWithProperties(schema.properties[k]) } if (!norder.includes(k)) { @@ -220,7 +220,7 @@ function computeSelected(property: any) { if (!opened) return '' - if (property.type !== 'object') return property.type + if (property?.type !== 'object') return property?.type if (property.format === 'resource-s3_object') return 'S3' if (property.format?.startsWith('dynselect-')) return 'dynselect' if (property.format?.startsWith('dynmultiselect-')) return 'dynmultiselect' @@ -344,7 +344,7 @@ Object.entries(schema?.properties ?? {}) .filter(([_, property]) => { const props = property as any - return props.type === 'object' && DynamicInput.isDynInputFormat(props.format) + return props?.type === 'object' && DynamicInput.isDynInputFormat(props.format) }) .map(([fieldName, _]) => fieldName.replace(/\s+/g, '_')) ) @@ -652,7 +652,7 @@ {#if typeof args == 'object' && schema?.properties[argName]} Empty flow
{/if}
-
- { - loadPreviousIters(key, amount) - }} - /> -
- {#if selected == 'assets' && render} + {#if selected == 'logs' && render} +
+ { + loadPreviousIters(key, amount) + }} + /> +
+ {:else if selected == 'assets' && render}
{ + flowDone + untrack(() => { + timelineCompute?.setFlowDone(flowDone) + }) + })
{min ? displayDate(new Date(min), true) : ''}
{#if max && min}{msToSec(max - min, 1)}s
{/if}
{max ? displayDate(new Date(max), true) : ''}{#if !max && min}{#if now} diff --git a/frontend/src/lib/components/InputTransformForm.svelte b/frontend/src/lib/components/InputTransformForm.svelte index 53e2eb704e..b7b00ac5f8 100644 --- a/frontend/src/lib/components/InputTransformForm.svelte +++ b/frontend/src/lib/components/InputTransformForm.svelte @@ -11,6 +11,7 @@ import type { Schema } from '$lib/common' import type { InputCat } from '$lib/utils' import { createEventDispatcher, getContext, untrack } from 'svelte' + import { computeShow } from '$lib/utils' import ArgInput from './ArgInput.svelte' import FieldHeader from './FieldHeader.svelte' @@ -36,6 +37,7 @@ import { twMerge } from 'tailwind-merge' import FlowPlugConnect from './FlowPlugConnect.svelte' import { deepEqual } from 'fast-equals' + import S3ArrayHelperButton from './S3ArrayHelperButton.svelte' interface Props { schema: Schema | { properties?: Record; required?: string[] } @@ -57,6 +59,7 @@ hideHelpButton?: boolean class?: string editor?: SimpleEditor | undefined + otherArgs?: Record } let { @@ -78,7 +81,8 @@ enableAi = false, hideHelpButton = false, class: className = '', - editor = $bindable(undefined) + editor = $bindable(undefined), + otherArgs = {} }: Props = $props() let monaco: SimpleEditor | undefined = $state(undefined) @@ -86,6 +90,8 @@ let argInput: ArgInput | undefined = $state(undefined) let focusedPrev = false + let hidden = $state(false) + const variableMatch = (value: string): RegExpMatchArray | null => value.match(/^variable\('([^']+)'\)$/) const resourceMatch = (value: string): RegExpMatchArray | null => @@ -235,6 +241,51 @@ return inputCat === 'string' || inputCat === 'sql' || inputCat == 'yaml' } + function appendPathToArrayExpr(currentExpr: string | undefined, path: string) { + const trimmedExpr = currentExpr?.trim() || '' + + let newExpr = trimmedExpr + if (trimmedExpr.startsWith('[') && trimmedExpr.endsWith(']')) { + // Parse existing array and append new item + const innerContent = trimmedExpr.slice(1, -1).trim() + if (innerContent) { + newExpr = `[${innerContent}, ${path}]` + } else { + newExpr = `[${path}]` + } + } else { + // Create new array with single item + newExpr = `[${path}]` + } + arg.expr = newExpr + arg.type = 'javascript' + + // Update Monaco editor after setting the expression + tick().then(() => { + monaco?.setCode(newExpr) + }) + + // Dispatch change + dispatch('change', { argName, arg }) + } + + async function switchToJsAndConnect(onPath: (path: string) => void) { + // Switch to JavaScript mode + propertyType = 'javascript' + arg.type = 'javascript' + arg.expr = arg.expr || '[]' + arg.value = undefined + + // Wait for the component to re-render and Monaco to be available + await tick() + + // Activate connect mode + focusProp?.(argName, 'connect', (path) => { + onPath(path) + return true + }) + } + function connectProperty(rawValue: string) { // Extract path from variable('x') or resource('x') format const varMatch = variableMatch(rawValue) @@ -258,6 +309,47 @@ } } + function handleFieldVisibility( + schema: Schema | any, + arg: InputTransform | any, + otherArgs: Record + ) { + const schemaProperty = schema?.properties?.[argName] + if (schemaProperty?.showExpr) { + // Build args object with current field value and other context + const currentValue = propertyType === 'static' ? arg?.value : arg?.expr + + // Convert otherArgs from InputTransform objects to their actual values + const contextArgs = { + [argName]: currentValue + } + + // Extract values from InputTransform objects in otherArgs + Object.keys(otherArgs ?? {}).forEach((key) => { + const otherArg = otherArgs[key] + const otherArgValue = otherArg.type === 'static' ? otherArg.value : otherArg.expr + contextArgs[key] = otherArgValue + }) + + const shouldShow = computeShow(argName, schemaProperty.showExpr, contextArgs) + if (shouldShow) { + hidden = false + } else if (!hidden) { + hidden = true + // Clear the arg value when hidden (following SchemaForm pattern) + if (arg) { + arg.value = undefined + arg.expr = undefined + } + // Make sure validation passes when hidden + inputCheck = true + } + } else { + // No showExpr, always show + hidden = false + } + } + function onFocus() { focused = true if (isStaticTemplate(inputCat)) { @@ -358,9 +450,21 @@ $effect(() => { schema?.properties?.[argName]?.default && untrack(() => setDefaultCode()) }) + $effect.pre(() => { + // Monitor changes that affect field visibility + JSON.stringify(schema) + JSON.stringify(arg) + JSON.stringify(otherArgs) + + untrack(() => handleFieldVisibility(schema, arg, otherArgs)) + }) let connecting = $derived( $propPickerConfig?.propName == argName && $propPickerConfig?.insertionMode == 'connect' ) + let shouldShowS3ArrayHelper = $derived( + inputCat === 'list' && + ['s3object', 's3_object'].includes(schema?.properties?.[argName]?.items?.resourceType) + ) {#if arg != undefined} @@ -463,6 +567,7 @@ on:selected={(e) => { if (e.detail == propertyType) return const staticTemplate = isStaticTemplate(inputCat) + if (e.detail === 'javascript') { if (arg.expr == undefined) { arg.expr = getDefaultExpr( @@ -635,6 +740,14 @@ bind:title={schema.properties[argName].title} bind:placeholder={schema.properties[argName].placeholder} /> + + {#if shouldShowS3ArrayHelper} + + switchToJsAndConnect((path) => appendPathToArrayExpr(arg.expr, path))} + /> + {/if} {:else if arg.expr != undefined}
{/if} + + {#if shouldShowS3ArrayHelper} + + focusProp?.(argName, 'connect', (path) => { + appendPathToArrayExpr(arg.expr, path) + return true + })} + /> + {/if} +
{:else} Not recognized input type {argName} ({arg.expr}, {propertyType}) diff --git a/frontend/src/lib/components/InputTransformSchemaForm.svelte b/frontend/src/lib/components/InputTransformSchemaForm.svelte index a478b0123c..7025c1baa7 100644 --- a/frontend/src/lib/components/InputTransformSchemaForm.svelte +++ b/frontend/src/lib/components/InputTransformSchemaForm.svelte @@ -116,6 +116,9 @@ {noDynamicToggle} {pickableProperties} {enableAi} + otherArgs={Object.fromEntries( + Object.entries(args ?? {}).filter(([key]) => key !== argName) + )} /> {/if} diff --git a/frontend/src/lib/components/JobLoader.svelte b/frontend/src/lib/components/JobLoader.svelte index 89281c7619..14e675328e 100644 --- a/frontend/src/lib/components/JobLoader.svelte +++ b/frontend/src/lib/components/JobLoader.svelte @@ -669,6 +669,12 @@ if (resultStreamOffset) { params.set('stream_offset', resultStreamOffset.toString()) } + if (job) { + params.set( + 'is_flow', + (job.job_kind === 'flow' || job.job_kind === 'flowpreview').toString() + ) + } const sseUrl = `/api/w/${workspace}/jobs_u/getupdate_sse/${id}?${params.toString()}` diff --git a/frontend/src/lib/components/S3ArrayHelperButton.svelte b/frontend/src/lib/components/S3ArrayHelperButton.svelte new file mode 100644 index 0000000000..2ea6450602 --- /dev/null +++ b/frontend/src/lib/components/S3ArrayHelperButton.svelte @@ -0,0 +1,20 @@ + + +{#if !connecting} +
+ +
+{/if} diff --git a/frontend/src/lib/components/common/popup/PopupV2.svelte b/frontend/src/lib/components/common/popup/PopupV2.svelte index 5523b3fc73..bbd68fe47b 100644 --- a/frontend/src/lib/components/common/popup/PopupV2.svelte +++ b/frontend/src/lib/components/common/popup/PopupV2.svelte @@ -2,6 +2,7 @@ import Portal from '$lib/components/Portal.svelte' import { clickOutside } from '$lib/utils' import { createFloatingActions, type ComputeConfig } from 'svelte-floating-ui' + import { fly } from 'svelte/transition' interface Props { floatingConfig?: ComputeConfig @@ -58,6 +59,7 @@ class="border rounded-lg shadow-lg bg-surface z5000" style="position:absolute" use:floatingContent + transition:fly={{ duration: 100, y: -16 }} >
{ + window.open( + scriptToHubUrl( + script.content, + script.summary, + script.description ?? '', + script.kind, + script.language, + script.schema, + script.lock ?? '', + $hubBaseUrlStore + ).toString(), + '_blank' + ) + } + }, { displayName: script.archived ? 'Unarchive' : 'Archive', icon: Archive, diff --git a/frontend/src/lib/components/copilot/FlowCopilotInputsModal.svelte b/frontend/src/lib/components/copilot/FlowCopilotInputsModal.svelte index 35553c70f4..f80d6cf537 100644 --- a/frontend/src/lib/components/copilot/FlowCopilotInputsModal.svelte +++ b/frontend/src/lib/components/copilot/FlowCopilotInputsModal.svelte @@ -2,6 +2,7 @@ import { createEventDispatcher } from 'svelte' import { Button, Badge } from '../common' import Modal from '../common/modal/Modal.svelte' + import Portal from '../Portal.svelte' export let open = false @@ -10,30 +11,32 @@ const dispatch = createEventDispatcher() - { - open = false - dispatch('confirmed') - }} - on:canceled - title="Windmill AI wants to add the following inputs to the flow:" -> -
    - {#each inputs as input} -
  • {input}
    • - {/each} -
- - - +
    + {#each inputs as input} +
  • {input}
    • + {/each} +
+ + + + diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 75489ecfaa..b3f624a9d2 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -84,7 +84,9 @@ } }) - const isLastMessageTool = $derived(messages.length > 0 && messages[messages.length - 1].role === 'tool') + const isLastMessageTool = $derived( + messages.length > 0 && messages[messages.length - 1].role === 'tool' + )
diff --git a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte index 6c7068a80c..ffd2b7075a 100644 --- a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte +++ b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte @@ -26,7 +26,7 @@
{providerModel.model} {#if multipleModels} -
+
{/if} diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index addcda69e2..0ccc7ab975 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1,5 +1,6 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' import { + copilotInfo, getCurrentModel, workspaceStore, type DBSchema, @@ -157,7 +158,7 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -function getModelMaxTokens(provider: AIProvider, model: string) { +export function getModelMaxTokens(provider: AIProvider, model: string) { if (model.startsWith('gpt-5')) { return 128000 } else if ((provider === 'azure_openai' || provider === 'openai') && model.startsWith('o')) { @@ -196,7 +197,10 @@ function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[] ) { - const maxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const modelKey = `${modelProvider.provider}:${modelProvider.model}` + const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel + const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) diff --git a/frontend/src/lib/components/flows/flowInfers.ts b/frontend/src/lib/components/flows/flowInfers.ts index 8f419fce5c..7c24278cc2 100644 --- a/frontend/src/lib/components/flows/flowInfers.ts +++ b/frontend/src/lib/components/flows/flowInfers.ts @@ -62,50 +62,58 @@ export async function loadSchemaFromModule(module: FlowModule): Promise<{ type: 'object', format: 'ai-provider' }, + output_type: { + type: 'string', + description: + 'The type of output the AI agent will generate (text or image). Image output requires a configured workspace S3 storage, will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.', + enum: ['text', 'image'], + default: 'text' + }, user_message: { - type: 'string' + type: 'string', + description: 'The message to give as input to the AI agent.' }, system_prompt: { - type: 'string' + type: 'string', + description: 'The system prompt to give as input to the AI agent.' }, - image: { - type: 'object', - description: 'Image to send to the AI agent (optional)', - format: 'resource-s3_object' + user_images: { + type: 'array', + description: + 'Array of images to give as input to the AI agent. Requires a configured workspace S3 storage.', + items: { + type: 'object' as const, + resourceType: 's3object' + } }, max_completion_tokens: { - type: 'number' + type: 'number', + description: 'The maximum number of output tokens.' }, temperature: { type: 'number', description: - 'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).' - }, - output_type: { - type: 'string', - description: - 'The type of output the AI agent will generate (text or image). Image output will ignore tools, and only works with OpenAI, Google AI and OpenRouter gemini-image-preview model.', - enum: ['text', 'image'], - default: 'text' + 'Controls randomness in text generation. Range: 0.0 (deterministic) to 2.0 (random).', + showExpr: "fields.output_type === 'text'" }, output_schema: { type: 'object', description: 'JSON schema that the AI agent will follow for its response format (only used if output_type is text)', - format: 'json-schema' + format: 'json-schema', + showExpr: "fields.output_type === 'text'" } }, - required: ['provider', 'model', 'user_message'], + required: ['provider', 'user_message', 'output_type'], type: 'object', order: [ 'provider', - 'model', + 'output_type', 'user_message', 'system_prompt', - 'image', + 'user_images', 'max_completion_tokens', 'temperature', - 'output_type', 'output_schema' ] } diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index c2e887063a..7c531a8d87 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -6,8 +6,6 @@ import ToggleHubWorkspaceQuick from '$lib/components/ToggleHubWorkspaceQuick.svelte' import TopLevelNode from '../pickers/TopLevelNode.svelte' - // import type { Writable } from 'svelte/store' - const dispatch = createEventDispatcher() interface Props { stop?: boolean @@ -32,26 +30,14 @@ $state(kind) let preFilter: 'all' | 'workspace' | 'hub' = $state('all') let loading = $state(false) - let small = $state(false) + let small = $derived(kind === 'preprocessor' || kind === 'failure') let width = $state(0) let height = $state(0) let displayPath = $derived(width > 650 || height > 400) - - $effect(() => { - small = kind === 'preprocessor' || kind === 'failure' - }) - - - -
+ let listHubIntegrationsCached = createCache( + (params: { kind: HubScriptKind & string }) => IntegrationService.listHubIntegrations(params), + { initial: { kind: 'script' } } + ) + let listHubScriptsCached = createCache( + async ({ + filter, + kind, + appFilter + }: { + filter: string + kind: HubScriptKind & string + appFilter: string | undefined + }) => + filter.length > 0 + ? await ScriptService.queryHubScripts({ text: filter, limit: 40, kind }) + : ((await ScriptService.getTopHubScripts({ limit: 40, kind, app: appFilter })).asks ?? []), + { initial: { filter: '', kind: 'script', appFilter: undefined } } + ) + + + + +{#if Object.keys(aiProviders).length > 0} +
+
+

Model Output Limits

+

+ Configure maximum token limits for each model. These limits apply to all AI chat + interactions in the workspace. +

+
+ +
+ {#each Object.entries(modelsByProvider).filter(([provider, models]) => models.length > 0) as [provider, models]} + {@const isExpanded = !collapsedProviders[provider]} + {@const hasCustom = hasCustomSettings(provider, models)} +
+ + + {#if isExpanded} +
+
+ {#each models as { model }} + {@const currentTokens = getCurrentTokensForModel(provider as AIProvider, model)} + {@const defaultTokens = getDefaultTokensForModel(provider as AIProvider, model)} + {@const isAtDefault = isModelAtDefault(provider as AIProvider, model)} +
+
+
+ {model} +
+
+ { + const value = parseInt(e.currentTarget.value) + if (!isNaN(value)) { + updateTokensForModel(provider as AIProvider, model, value) + } + }} + class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" + /> + tokens +
+
+ {#if !isAtDefault} +
+ Default: {defaultTokens} tokens + +
+ {#if errors[getModelKey(provider as AIProvider, model)]} +
{errors[getModelKey(provider as AIProvider, model)]}
+ {/if} + {/if} +
+ {/each} +
+
+ {/if} +
+ {/each} +
+
+{/if} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 61f22fc478..5c33b2cca6 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -102,12 +102,14 @@ export const copilotInfo = writable<{ defaultModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record + maxTokensPerModel?: Record }>({ enabled: false, codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) export async function loadCopilot(workspace: string) { @@ -143,7 +145,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: aiConfig.code_completion_model, defaultModel: aiConfig.default_model, aiModels: aiModels, - customPrompts: aiConfig.custom_prompts ?? {} + customPrompts: aiConfig.custom_prompts ?? {}, + maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} }) } else { copilotSessionModel.set(undefined) @@ -153,7 +156,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) } } @@ -197,6 +201,7 @@ export const copilotSessionModel = writable( } : undefined ) + export const usedTriggerKinds = writable([]) type SQLBaseSchema = { diff --git a/frontend/src/lib/timelineCompute.svelte.ts b/frontend/src/lib/timelineCompute.svelte.ts index 1708968bf2..ba01b8e7f3 100644 --- a/frontend/src/lib/timelineCompute.svelte.ts +++ b/frontend/src/lib/timelineCompute.svelte.ts @@ -54,6 +54,7 @@ export class TimelineCompute { reset() { this.min = undefined this.max = undefined + this.#flowDone = false this.items = this.computeItems(this.#durationStatuses) } @@ -67,6 +68,10 @@ export class TimelineCompute { this.#flowDone = flowDone } + setFlowDone(flowDone: boolean) { + this.#flowDone = flowDone + } + destroy() { if (this.#interval) { clearInterval(this.#interval) @@ -129,6 +134,7 @@ export class TimelineCompute { }) this.items = nitems this.min = nmin + this.max = isStillRunning || (cnt < this.#flowModules.length && !this.#flowDone) ? undefined : nmax if (this.max && this.min) { diff --git a/frontend/src/lib/utils.ts b/frontend/src/lib/utils.ts index fd14922a13..fc60db5042 100644 --- a/frontend/src/lib/utils.ts +++ b/frontend/src/lib/utils.ts @@ -1582,3 +1582,32 @@ export function assert(msg: string, condition: boolean, value?: any) { console.error(m) } } + +export function createCache, T, InitialKeys extends Keys = Keys>( + compute: (keys: Keys) => T, + params?: { maxSize?: number; initial?: InitialKeys } +): (keys: Keys) => T { + let cache = new Map() + const maxSize = params?.maxSize ?? 15 + + if (params?.initial) { + let key = JSON.stringify(params.initial, Object.keys(params.initial).sort()) + let value = compute(params.initial) + cache.set(key, value) + } + + return (keys: Keys) => { + let key = JSON.stringify(keys, Object.keys(keys).sort()) + if (!cache.get(key)) { + let value = compute(keys) + cache.set(key, value) + + if (cache.size > maxSize) { + // remove the oldest entry (first inserted) + const oldestKey = cache.keys().next().value! + cache.delete(oldestKey) + } + } + return cache.get(key)! + } +} diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index fd2c69cd45..4f53e5a3bf 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -134,7 +134,7 @@ } async function loadUsage() { - if (isCloudHosted()) { + if (isCloudHosted() && $workspaceStore) { $usageStore = await UserService.getUsage() $workspaceUsageStore = await WorkspaceService.getWorkspaceUsage({ workspace: $workspaceStore! diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index f7251ec844..47fd6e33b7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -81,6 +81,7 @@ let codeCompletionModel: string | undefined = $state(undefined) let defaultModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) + let maxTokensPerModel: Record = $state({}) let s3ResourceSettings: S3ResourceSettings = $state({ resourceType: 's3', @@ -246,6 +247,7 @@ defaultModel = settings.ai_config?.default_model?.model codeCompletionModel = settings.ai_config?.code_completion_model?.model customPrompts = settings.ai_config?.custom_prompts ?? {} + maxTokensPerModel = settings.ai_config?.max_tokens_per_model ?? {} for (const mode of Object.values(AIMode)) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -813,6 +815,7 @@ bind:codeCompletionModel bind:defaultModel bind:customPrompts + bind:maxTokensPerModel bind:usingOpenaiClientCredentialsOauth /> {:else if tab == 'windmill_lfs'} diff --git a/lsp/Pipfile b/lsp/Pipfile index 6ec7817949..b40e89781e 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.543.0" -wmill_pg = ">=1.543.0" +wmill = ">=1.544.1" +wmill_pg = ">=1.544.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 243b3971c8..b202f0bba6 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.543.0 + version: 1.544.1 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 35536c940c..acaa735084 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.543.0' + ModuleVersion = '1.544.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 1b395306c4..137afcb36b 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.543.0" +version = "1.544.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index d4efada8ee..e705ab949d 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.543.0" +version = "1.544.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index b9011cb312..9c3a5c3f6c 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.543.0", + "version": "1.544.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index e06df5883f..a1bc3203c7 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.543.0", + "version": "1.544.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index c1c2dacfba..405633a255 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.543.0 +1.544.1