diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 5fa4a289f2..43e8954c33 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -67,13 +67,23 @@ jobs: - name: Substitute EE code (EE logic is behind feature flag) run: | ./substitute_ee_code.sh --copy --dir ./windmill-ee-private + - name: Cache DuckDB FFI module build + uses: actions/cache@v3 + with: + path: ./backend/windmill-duckdb-ffi-internal/target + key: ${{ runner.os }}-duckdb-ffi-${{ hashFiles('./backend/windmill-duckdb-ffi-internal/src/**/*.rs', './backend/windmill-duckdb-ffi-internal/Cargo.toml', './backend/windmill-duckdb-ffi-internal/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-duckdb-ffi- - name: cargo test timeout-minutes: 16 - run: deno --version && bun -v && go version && python3 --version && - SQLX_OFFLINE=true - DATABASE_URL=postgres://postgres:changeme@localhost:5432/windmill - DISABLE_EMBEDDING=true RUST_LOG=info RUST_LOG_STYLE=never - DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) - UV_PATH=$(which uv) cargo test --features - enterprise,deno_core,license,python,rust,scoped_cache,private --all -- - --nocapture + env: + SQLX_OFFLINE: true + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + DISABLE_EMBEDDING: true + RUST_LOG: info + RUST_LOG_STYLE: never + CARGO_NET_GIT_FETCH_WITH_CLI: true + run: | + deno --version && bun -v && go version && python3 --version + cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. + DENO_PATH=$(which deno) BUN_PATH=$(which bun) GO_PATH=$(which go) UV_PATH=$(which uv) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,private --all -- --nocapture diff --git a/.github/workflows/build-publish-rh8-image.yml b/.github/workflows/build-publish-rh8-image.yml new file mode 100644 index 0000000000..aabb17a592 --- /dev/null +++ b/.github/workflows/build-publish-rh8-image.yml @@ -0,0 +1,140 @@ +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +name: Build and publish windmill for RHEL8 +on: workflow_dispatch + +permissions: write-all + +jobs: + build_ee: + runs-on: ubicloud + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Read EE repo commit hash + run: | + echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV" + + - uses: actions/checkout@v4 + with: + repository: windmill-labs/windmill-ee-private + path: ./windmill-ee-private + ref: ${{ env.ee_repo_ref }} + token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} + fetch-depth: 0 + + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v2 + - uses: depot/setup-action@v1 + + - name: Docker meta + id: meta-ee-public + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-rhel8 + flavor: | + latest=false + tags: | + type=sha + + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Substitute EE code + run: | + ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private + + - name: Copy RHEL8 Dockerfile + run: | + cp ./docker/RHEL8/Dockerfile ./Dockerfile + + - name: Build and push publicly ee amd64 + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/amd64 + push: true + build-args: | + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private + secrets: | + rh_username=${{ secrets.RH_USERNAME }} + rh_password=${{ secrets.RH_PASSWORD }} + tags: | + ${{ steps.meta-ee-public.outputs.tags }}-amd64 + labels: | + ${{ steps.meta-ee-public.outputs.labels }}-amd64 + org.opencontainers.image.licenses=Windmill-Enterprise-License + + - name: Build and push publicly ee arm64 + uses: depot/build-push-action@v1 + with: + context: . + platforms: linux/arm64 + push: true + build-args: | + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,license,otel,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,postgres_trigger,gcp_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages,deno_core,mcp,private + secrets: | + rh_username=${{ secrets.RH_USERNAME }} + rh_password=${{ secrets.RH_PASSWORD }} + tags: | + ${{ steps.meta-ee-public.outputs.tags }}-arm64 + labels: | + ${{ steps.meta-ee-public.outputs.labels }}-arm64 + org.opencontainers.image.licenses=Windmill-Enterprise-License + + - uses: shrink/actions-docker-extract@v3 + id: extract-ee-amd64 + with: + image: ${{ steps.meta-ee-public.outputs.tags}}-amd64 + path: "/windmill/target/release/windmill" + + - uses: shrink/actions-docker-extract@v3 + id: extract-duckdb-ffi-internal + with: + image: ${{ steps.meta-ee-public.outputs.tags}}-amd64 + path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so" + + # - uses: shrink/actions-docker-extract@v3 + # id: extract-ee-arm64 + # with: + # image: ${{ steps.meta-ee-public.outputs.tags}}-arm64 + # path: "/windmill/target/release/windmill" + + - name: Rename binary with corresponding architecture + run: | + mv "${{ steps.extract-ee-amd64.outputs.destination }}/windmill" "${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel8" + # mv "${{ steps.extract-ee-arm64.outputs.destination }}/windmill" "${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel8" + + - uses: actions/upload-artifact@v4 + with: + name: RHEL8-amd64 build + path: ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel8 + + - uses: actions/upload-artifact@v4 + with: + name: RHEL8-amd64 dynamic libraries build + path: ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/libwindmill_duckdb_ffi_internal.so + + # - uses: actions/upload-artifact@v4 + # with: + # name: RHEL8-arm64 build + # path: + # ${{ steps.extract-ee-arm64.outputs.destination + # }}/windmill-ee-arm64-rhel8 + + # - name: Attach binary to release + # uses: softprops/action-gh-release@v2 + # if: startsWith(github.ref, 'refs/tags/') + # with: + # files: | + # ${{ steps.extract-ee-arm64.outputs.destination }}/windmill-ee-arm64-rhel8 + # ${{ steps.extract-ee-amd64.outputs.destination }}/windmill-ee-amd64-rhel8 diff --git a/CHANGELOG.md b/CHANGELOG.md index efd125ea43..c42c850475 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,149 @@ # Changelog +## [1.582.2](https://github.com/windmill-labs/windmill/compare/v1.582.1...v1.582.2) (2025-11-21) + + +### Bug Fixes + +* fix aws oidc refresh ([98bdb68](https://github.com/windmill-labs/windmill/commit/98bdb6825a1b85c973ddec3e6b933e4a3d6d6972)) + +## [1.582.1](https://github.com/windmill-labs/windmill/compare/v1.582.0...v1.582.1) (2025-11-21) + + +### Bug Fixes + +* fix aws oidc refresh ([a3b4cfc](https://github.com/windmill-labs/windmill/commit/a3b4cfcb8f11db326b0ebf1777ad7e6479425125)) + +## [1.582.0](https://github.com/windmill-labs/windmill/compare/v1.581.1...v1.582.0) (2025-11-20) + + +### Features + +* **aichat:** handle duckdb scripts ([#7187](https://github.com/windmill-labs/windmill/issues/7187)) ([ce5a318](https://github.com/windmill-labs/windmill/commit/ce5a31865cf6965ec28c449c2a832b93572a8eb6)) +* **ee:** support iamrds ([e9691c9](https://github.com/windmill-labs/windmill/commit/e9691c9eb080236849850a1ea6f3237ae39a2c4c)) + + +### Bug Fixes + +* **aichat:** fallback to completion if responses fails ([#7190](https://github.com/windmill-labs/windmill/issues/7190)) ([b56e611](https://github.com/windmill-labs/windmill/commit/b56e611700f06844dda4f30d02a1119e714d73a4)) +* **frontend:** show code/lock in flow steps on runs page ([#7191](https://github.com/windmill-labs/windmill/issues/7191)) ([338fd8a](https://github.com/windmill-labs/windmill/commit/338fd8a38cb035de298006ed1b96b6513eab9769)) + +## [1.581.1](https://github.com/windmill-labs/windmill/compare/v1.581.0...v1.581.1) (2025-11-20) + + +### Bug Fixes + +* **frontend:** missing node Result id migration ([#7182](https://github.com/windmill-labs/windmill/issues/7182)) ([054aeb3](https://github.com/windmill-labs/windmill/commit/054aeb33271288dc9458b012881164c3c4597280)) + +## [1.581.0](https://github.com/windmill-labs/windmill/compare/v1.580.0...v1.581.0) (2025-11-19) + + +### Features + +* **frontend:** add notes to flow ([#6628](https://github.com/windmill-labs/windmill/issues/6628)) ([cfeb294](https://github.com/windmill-labs/windmill/commit/cfeb294308ba85763025f3628cbb85144d7f0778)) + +## [1.580.0](https://github.com/windmill-labs/windmill/compare/v1.579.2...v1.580.0) (2025-11-18) + + +### Features + +* **aichat:** use responses api for openai models ([#7163](https://github.com/windmill-labs/windmill/issues/7163)) ([5c79a35](https://github.com/windmill-labs/windmill/commit/5c79a35306855143428d0725519578aea0a746fd)) +* disabling/enabling email triggers ([#7171](https://github.com/windmill-labs/windmill/issues/7171)) ([8ae266b](https://github.com/windmill-labs/windmill/commit/8ae266b6a9ced16e1b7416cfc8bea5fe7a7af042)) +* **security:** unshare pid of worker job process ([#7106](https://github.com/windmill-labs/windmill/issues/7106)) ([5aa251a](https://github.com/windmill-labs/windmill/commit/5aa251a2d276cc9d27bf104f8e4f724ea6a28231)) +* support secondary promotion repos in git sync settings ([#7173](https://github.com/windmill-labs/windmill/issues/7173)) ([5548221](https://github.com/windmill-labs/windmill/commit/55482210921fe2eb0fd158abd4f7369495f2dfd7)) + + +### Bug Fixes + +* change uv tool dir from /root to /usr/local/uv ([c3e59fe](https://github.com/windmill-labs/windmill/commit/c3e59fe064fc3b9d4c05958eea54601ff3410899)) +* improve delete to handle ai chat ([f371fbe](https://github.com/windmill-labs/windmill/commit/f371fbeb9bb0946bd29a6413ee7ede75dedda5d9)) +* support IRSA for duckdb s3 proxy ([2058f27](https://github.com/windmill-labs/windmill/commit/2058f27e03468d45813f340b8563f935ca2142f4)) + +## [1.579.2](https://github.com/windmill-labs/windmill/compare/v1.579.1...v1.579.2) (2025-11-18) + + +### Bug Fixes + +* ducklake manager table explorer issue ([d08c091](https://github.com/windmill-labs/windmill/commit/d08c0916f72a67f01e0c4475f03f9d1d33c10905)) + +## [1.579.1](https://github.com/windmill-labs/windmill/compare/v1.579.0...v1.579.1) (2025-11-18) + + +### Bug Fixes + +* fix s3 object download frontend freezes ([09a6e1f](https://github.com/windmill-labs/windmill/commit/09a6e1feaa79ce3f8548f8090fddbf46abb08b18)) + +## [1.579.0](https://github.com/windmill-labs/windmill/compare/v1.578.0...v1.579.0) (2025-11-17) + + +### Features + +* **ai:** handle aws bedrock as provider ([#7155](https://github.com/windmill-labs/windmill/issues/7155)) ([79ac631](https://github.com/windmill-labs/windmill/commit/79ac6312e87afa3646bddc0f7e66fc4367dbff7c)) +* **mcp:** granular token scopes for scripts, flows, and endpoints ([#7130](https://github.com/windmill-labs/windmill/issues/7130)) ([88d04b9](https://github.com/windmill-labs/windmill/commit/88d04b9cbeee98f3256b78e9d34beb930cd729ec)) +* rhel8 + fix rhel9 ([#7165](https://github.com/windmill-labs/windmill/issues/7165)) ([499d7d4](https://github.com/windmill-labs/windmill/commit/499d7d4098758726a8cb2bf3e4837927b8fd70a4)) + + +### Bug Fixes + +* **backend:** worker count in latest worker usage ([#7160](https://github.com/windmill-labs/windmill/issues/7160)) ([b87d2cc](https://github.com/windmill-labs/windmill/commit/b87d2cc64cb54b602ee599fcde7f0fd3c8931550)) +* fix custom email triggers enabled ([#7164](https://github.com/windmill-labs/windmill/issues/7164)) ([90b5569](https://github.com/windmill-labs/windmill/commit/90b5569c911f9025b0e6b5318f57705efbd9bd17)) + +## [1.578.0](https://github.com/windmill-labs/windmill/compare/v1.577.0...v1.578.0) (2025-11-17) + + +### Features + +* support to run windows binary as service ([#7153](https://github.com/windmill-labs/windmill/issues/7153)) ([ceeff5f](https://github.com/windmill-labs/windmill/commit/ceeff5f76c69d98319bb3fb7f7779b6046478d6b)) + +## [1.577.0](https://github.com/windmill-labs/windmill/compare/v1.576.3...v1.577.0) (2025-11-17) + + +### Features + +* add support for validateset in pwsh ([#7158](https://github.com/windmill-labs/windmill/issues/7158)) ([b66e038](https://github.com/windmill-labs/windmill/commit/b66e038a0f8b6bffe157a83671c8e692c1441f23)) +* allow http trigger to be disabled ([#6976](https://github.com/windmill-labs/windmill/issues/6976)) ([09082de](https://github.com/windmill-labs/windmill/commit/09082de53971d0d2f2a6308bc8ee573458a3b913)) + + +### Bug Fixes + +* create app_themes/groups/components only when needed ([cf5d58e](https://github.com/windmill-labs/windmill/commit/cf5d58ea43cef6add3da2aa1e24efc83be6df3b9)) +* fix parse_postgres_uri not decoding password ([#7157](https://github.com/windmill-labs/windmill/issues/7157)) ([2cae72c](https://github.com/windmill-labs/windmill/commit/2cae72c9db6bd08689e1672be6dda32f6af831fb)) + +## [1.576.3](https://github.com/windmill-labs/windmill/compare/v1.576.2...v1.576.3) (2025-11-15) + + +### Bug Fixes + +* handle better alias types in duckdb ([2c04e04](https://github.com/windmill-labs/windmill/commit/2c04e04bf0e3272c89f321392158888d02a1191b)) + +## [1.576.2](https://github.com/windmill-labs/windmill/compare/v1.576.1...v1.576.2) (2025-11-15) + + +### Bug Fixes + +* temporary fix for duckdb type_aliases causing issues ([#7148](https://github.com/windmill-labs/windmill/issues/7148)) ([6426ebf](https://github.com/windmill-labs/windmill/commit/6426ebf8cb713443904065064b6a07eb1db0761a)) + +## [1.576.1](https://github.com/windmill-labs/windmill/compare/v1.576.0...v1.576.1) (2025-11-14) + + +### Bug Fixes + +* DuckDB FFI crash fix ([#7145](https://github.com/windmill-labs/windmill/issues/7145)) ([d3fc459](https://github.com/windmill-labs/windmill/commit/d3fc459b407682bf588236236916363d94f3e1ff)) + +## [1.576.0](https://github.com/windmill-labs/windmill/compare/v1.575.4...v1.576.0) (2025-11-14) + + +### Features + +* add support for switch and attributes in pwsh params ([#7143](https://github.com/windmill-labs/windmill/issues/7143)) ([c16bef8](https://github.com/windmill-labs/windmill/commit/c16bef8f296645ff873f9d8d28e3dcb50a65e304)) +* **ai:** handle aws bedrock as provider ([#7131](https://github.com/windmill-labs/windmill/issues/7131)) ([30eb9aa](https://github.com/windmill-labs/windmill/commit/30eb9aae25eeb563ad119ef93f3ff1ab17c66d75)) +* webhook by flow version ([#7062](https://github.com/windmill-labs/windmill/issues/7062)) ([09cdfb4](https://github.com/windmill-labs/windmill/commit/09cdfb4556748903dc5bbf53ef3356ac97c57d90)) + + +### Bug Fixes + +* use proper TLS connector for DuckLake instance catalog setup ([#7138](https://github.com/windmill-labs/windmill/issues/7138)) ([cf36fe3](https://github.com/windmill-labs/windmill/commit/cf36fe3bb1beec80fa84dc342a8a38cc7369bc4d)) + ## [1.575.4](https://github.com/windmill-labs/windmill/compare/v1.575.3...v1.575.4) (2025-11-13) diff --git a/Dockerfile b/Dockerfile index b3f18b4e17..7aeb2b218a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,10 @@ ARG WITH_GIT=true ARG LATEST_STABLE_PY=3.11.10 ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime ENV UV_PYTHON_PREFERENCE=only-managed + +RUN mkdir -p /usr/local/uv ENV UV_TOOL_BIN_DIR=/usr/local/bin +ENV UV_TOOL_DIR=/usr/local/uv ENV PATH /usr/local/bin:/root/.local/bin:$PATH diff --git a/backend/.cargo/config.toml b/backend/.cargo/config.toml index 5babef3f8a..234ac9a50f 100644 --- a/backend/.cargo/config.toml +++ b/backend/.cargo/config.toml @@ -13,4 +13,7 @@ rustflags = [ "-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup", "-C", "link-args=-Wl,-rpath,$ORIGIN/" -] \ No newline at end of file +] + +[net] +git-fetch-with-cli = true \ No newline at end of file diff --git a/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json b/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json deleted file mode 100644 index 9affa78033..0000000000 --- a/backend/.sqlx/query-045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "045b26db0cefe6eaac0e572661d984ff5ce7086ac511e8647e2024d9dbe0af56" -} diff --git a/backend/.sqlx/query-08574e8e5dc165041750880fb02e7ffea83ae94a670b598b6dada0b3d0914629.json b/backend/.sqlx/query-08574e8e5dc165041750880fb02e7ffea83ae94a670b598b6dada0b3d0914629.json new file mode 100644 index 0000000000..12d5a410f8 --- /dev/null +++ b/backend/.sqlx/query-08574e8e5dc165041750880fb02e7ffea83ae94a670b598b6dada0b3d0914629.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM flow_conversation WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "08574e8e5dc165041750880fb02e7ffea83ae94a670b598b6dada0b3d0914629" +} diff --git a/backend/.sqlx/query-207a0721b6f0b8b6ddd4120343eba524a2bc1e9047bdde5f568af4d993dbb74c.json b/backend/.sqlx/query-207a0721b6f0b8b6ddd4120343eba524a2bc1e9047bdde5f568af4d993dbb74c.json new file mode 100644 index 0000000000..b0cf45284a --- /dev/null +++ b/backend/.sqlx/query-207a0721b6f0b8b6ddd4120343eba524a2bc1e9047bdde5f568af4d993dbb74c.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE\n flow\n SET\n path = $1,\n summary = $2,\n description = $3,\n dependency_job = NULL,\n lock_error_logs = '',\n draft_only = NULL,\n tag = $4,\n dedicated_worker = $5,\n visible_to_runner_only = $6,\n on_behalf_of_email = $7,\n value = $8,\n schema = $9::text::json,\n edited_by = $10,\n edited_at = now()\n WHERE\n path = $11 AND workspace_id = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Varchar", + "Bool", + "Bool", + "Text", + "Jsonb", + "Text", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "207a0721b6f0b8b6ddd4120343eba524a2bc1e9047bdde5f568af4d993dbb74c" +} diff --git a/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json b/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json deleted file mode 100644 index de5b99791f..0000000000 --- a/backend/.sqlx/query-42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "42baae3c69367bbb21771a9711c311afe67d339ca022ba61c2767c004b038ef0" -} diff --git a/backend/.sqlx/query-544a00afb5c72d2aa24ab4ccfe68d1e383aeda9a5a71183b7fbaa41deca4e333.json b/backend/.sqlx/query-544a00afb5c72d2aa24ab4ccfe68d1e383aeda9a5a71183b7fbaa41deca4e333.json deleted file mode 100644 index bf1418bb0d..0000000000 --- a/backend/.sqlx/query-544a00afb5c72d2aa24ab4ccfe68d1e383aeda9a5a71183b7fbaa41deca4e333.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO flow \n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) \n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at\n FROM flow\n WHERE path = $2 AND workspace_id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "544a00afb5c72d2aa24ab4ccfe68d1e383aeda9a5a71183b7fbaa41deca4e333" -} diff --git a/backend/.sqlx/query-56d3dccce81c652d6ab3d4ba69de89c4a5cb1fbc9a8b73e6b8ba4d6e74088352.json b/backend/.sqlx/query-56d3dccce81c652d6ab3d4ba69de89c4a5cb1fbc9a8b73e6b8ba4d6e74088352.json new file mode 100644 index 0000000000..7be13f4746 --- /dev/null +++ b/backend/.sqlx/query-56d3dccce81c652d6ab3d4ba69de89c4a5cb1fbc9a8b73e6b8ba4d6e74088352.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM ai_agent_memory WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "56d3dccce81c652d6ab3d4ba69de89c4a5cb1fbc9a8b73e6b8ba4d6e74088352" +} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json b/backend/.sqlx/query-628d303956d55d22e5ac64496b66010d4476d5c222184f09e5a189975a37b941.json similarity index 90% rename from backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json rename to backend/.sqlx/query-628d303956d55d22e5ac64496b66010d4476d5c222184f09e5a189975a37b941.json index 7520bc8879..3f5e733cb1 100644 --- a/backend/.sqlx/query-668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4.json +++ b/backend/.sqlx/query-628d303956d55d22e5ac64496b66010d4476d5c222184f09e5a189975a37b941.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE local_part = $1 \n AND workspaced_local_part = FALSE\n ", + "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE local_part = $1 \n AND workspaced_local_part = FALSE\n AND enabled IS TRUE\n ", "describe": { "columns": [ { @@ -66,5 +66,5 @@ true ] }, - "hash": "668edc2f84eccf5db6b8daa3ed97e25e46a37c50a9d1a6eca5a277fb58a2e7b4" + "hash": "628d303956d55d22e5ac64496b66010d4476d5c222184f09e5a189975a37b941" } diff --git a/backend/.sqlx/query-6525e65ffe66643fed4db83aa81b525fc3ec54fdbca5d12647c1da88c9983b10.json b/backend/.sqlx/query-6525e65ffe66643fed4db83aa81b525fc3ec54fdbca5d12647c1da88c9983b10.json new file mode 100644 index 0000000000..1059fac898 --- /dev/null +++ b/backend/.sqlx/query-6525e65ffe66643fed4db83aa81b525fc3ec54fdbca5d12647c1da88c9983b10.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "6525e65ffe66643fed4db83aa81b525fc3ec54fdbca5d12647c1da88c9983b10" +} diff --git a/backend/.sqlx/query-676c758d9c4492dada50edd3ad06dc628e2ba56eab5e1a50c99481da9793759e.json b/backend/.sqlx/query-676c758d9c4492dada50edd3ad06dc628e2ba56eab5e1a50c99481da9793759e.json new file mode 100644 index 0000000000..11339abea1 --- /dev/null +++ b/backend/.sqlx/query-676c758d9c4492dada50edd3ad06dc628e2ba56eab5e1a50c99481da9793759e.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow\n (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at)\n SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at\n FROM flow\n WHERE path = $2 AND workspace_id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "676c758d9c4492dada50edd3ad06dc628e2ba56eab5e1a50c99481da9793759e" +} diff --git a/backend/.sqlx/query-d328d00e5393b0e8d2c4b3674221fabbae580fc404e0d4481fd8d7fb51272c94.json b/backend/.sqlx/query-6c568509908c1833d9e6f58f739b8a186fc91f94d4fcc40eeb9914084e4ef60c.json similarity index 70% rename from backend/.sqlx/query-d328d00e5393b0e8d2c4b3674221fabbae580fc404e0d4481fd8d7fb51272c94.json rename to backend/.sqlx/query-6c568509908c1833d9e6f58f739b8a186fc91f94d4fcc40eeb9914084e4ef60c.json index 474ce7b688..b6e4f96c3c 100644 --- a/backend/.sqlx/query-d328d00e5393b0e8d2c4b3674221fabbae580fc404e0d4481fd8d7fb51272c94.json +++ b/backend/.sqlx/query-6c568509908c1833d9e6f58f739b8a186fc91f94d4fcc40eeb9914084e4ef60c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO email_trigger (\n workspace_id,\n path,\n script_path,\n is_flow,\n local_part,\n workspaced_local_part,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, now(), $9, $10, $11\n )\n ", + "query": "\n INSERT INTO email_trigger (\n workspace_id,\n path,\n script_path,\n is_flow,\n local_part,\n workspaced_local_part,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n enabled\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, now(), $9, $10, $11, $12\n )\n ", "describe": { "columns": [], "parameters": { @@ -15,10 +15,11 @@ "Varchar", "Varchar", "Jsonb", - "Jsonb" + "Jsonb", + "Bool" ] }, "nullable": [] }, - "hash": "d328d00e5393b0e8d2c4b3674221fabbae580fc404e0d4481fd8d7fb51272c94" + "hash": "6c568509908c1833d9e6f58f739b8a186fc91f94d4fcc40eeb9914084e4ef60c" } diff --git a/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json b/backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json similarity index 69% rename from backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json rename to backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json index 50d4a9595d..f0c32e5dd0 100644 --- a/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json +++ b/backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9, job_isolation = $10 WHERE worker = $5", "describe": { "columns": [], "parameters": { @@ -13,10 +13,11 @@ "Float4", "Float4", "Float4", - "Float4" + "Float4", + "Text" ] }, "nullable": [] }, - "hash": "506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773" + "hash": "76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64" } diff --git a/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json b/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json similarity index 80% rename from backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json rename to backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json index 9df7dced7b..644cb25123 100644 --- a/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json +++ b/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -97,6 +97,11 @@ "ordinal": 18, "name": "wm_memory_usage", "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "job_isolation", + "type_info": "Text" } ], "parameters": { @@ -126,8 +131,9 @@ true, true, true, + true, true ] }, - "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab" + "hash": "771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30" } diff --git a/backend/.sqlx/query-778ab8ceb2a84978919ceb07f399468e01c4bee4cd755322eb2a83353a279a2b.json b/backend/.sqlx/query-778ab8ceb2a84978919ceb07f399468e01c4bee4cd755322eb2a83353a279a2b.json new file mode 100644 index 0000000000..a35e6560e3 --- /dev/null +++ b/backend/.sqlx/query-778ab8ceb2a84978919ceb07f399468e01c4bee4cd755322eb2a83353a279a2b.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO unique_ext_jwt_token (jwt_hash, last_used_at)\n VALUES ($1, NOW())\n ON CONFLICT (jwt_hash)\n DO UPDATE SET last_used_at = NOW()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "778ab8ceb2a84978919ceb07f399468e01c4bee4cd755322eb2a83353a279a2b" +} diff --git a/backend/.sqlx/query-88d529fd26d6ecba7640c74daccb5cb0c7146c11adae8c287c0c9e71ac68f7a7.json b/backend/.sqlx/query-88d529fd26d6ecba7640c74daccb5cb0c7146c11adae8c287c0c9e71ac68f7a7.json new file mode 100644 index 0000000000..b664b2a91a --- /dev/null +++ b/backend/.sqlx/query-88d529fd26d6ecba7640c74daccb5cb0c7146c11adae8c287c0c9e71ac68f7a7.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "88d529fd26d6ecba7640c74daccb5cb0c7146c11adae8c287c0c9e71ac68f7a7" +} diff --git a/backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json b/backend/.sqlx/query-94e2e899fc0a5134c29217feda1f0530d03548cc5c884c83d7b85d068e738b8e.json similarity index 71% rename from backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json rename to backend/.sqlx/query-94e2e899fc0a5134c29217feda1f0530d03548cc5c884c83d7b85d068e738b8e.json index f69ec4f2a6..ca3310858b 100644 --- a/backend/.sqlx/query-1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117.json +++ b/backend/.sqlx/query-94e2e899fc0a5134c29217feda1f0530d03548cc5c884c83d7b85d068e738b8e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path,\n summary,\n description,\n is_flow,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n email,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22\n )\n ", + "query": "\n INSERT INTO http_trigger (\n workspace_id,\n path,\n route_path,\n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path,\n summary,\n description,\n is_flow,\n enabled,\n request_type,\n authentication_method,\n http_method,\n static_asset_config,\n edited_by,\n email,\n edited_at,\n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23\n )\n ", "describe": { "columns": [], "parameters": { @@ -17,6 +17,7 @@ "Varchar", "Text", "Bool", + "Bool", { "Custom": { "name": "request_type", @@ -69,5 +70,5 @@ }, "nullable": [] }, - "hash": "1f6b773ce34fe51d03d6f9a2345481629c62453eebbb08f82dd2da23389bc117" + "hash": "94e2e899fc0a5134c29217feda1f0530d03548cc5c884c83d7b85d068e738b8e" } diff --git a/backend/.sqlx/query-acb094aef60bba9083087264d65034fce38417099f15e8312be72a386f10bc1f.json b/backend/.sqlx/query-a704283ff62ac1cd6db489ca3f84d6994d75c6b9e5f61daf54d298a2fdcfd9af.json similarity index 73% rename from backend/.sqlx/query-acb094aef60bba9083087264d65034fce38417099f15e8312be72a386f10bc1f.json rename to backend/.sqlx/query-a704283ff62ac1cd6db489ca3f84d6994d75c6b9e5f61daf54d298a2fdcfd9af.json index 805e5f0e79..b59a7f294a 100644 --- a/backend/.sqlx/query-acb094aef60bba9083087264d65034fce38417099f15e8312be72a386f10bc1f.json +++ b/backend/.sqlx/query-a704283ff62ac1cd6db489ca3f84d6994d75c6b9e5f61daf54d298a2fdcfd9af.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8\n WHERE \n workspace_id = $9 AND path = $10\n ", + "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n edited_by = $4,\n email = $5,\n edited_at = now(),\n error_handler_path = $6,\n error_handler_args = $7,\n retry = $8,\n enabled = $9\n WHERE \n workspace_id = $10 AND path = $11\n ", "describe": { "columns": [], "parameters": { @@ -13,11 +13,12 @@ "Varchar", "Jsonb", "Jsonb", + "Bool", "Text", "Text" ] }, "nullable": [] }, - "hash": "acb094aef60bba9083087264d65034fce38417099f15e8312be72a386f10bc1f" + "hash": "a704283ff62ac1cd6db489ca3f84d6994d75c6b9e5f61daf54d298a2fdcfd9af" } diff --git a/backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json b/backend/.sqlx/query-ada1a14b4f25e41742df153b07e8bdf5b5feac9c6fa9de38bd24104bac8539d2.json similarity index 67% rename from backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json rename to backend/.sqlx/query-ada1a14b4f25e41742df153b07e8bdf5b5feac9c6fa9de38bd24104bac8539d2.json index dfe905113e..1a33383b67 100644 --- a/backend/.sqlx/query-4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90.json +++ b/backend/.sqlx/query-ada1a14b4f25e41742df153b07e8bdf5b5feac9c6fa9de38bd24104bac8539d2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n authentication_resource_path = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n http_method = $7,\n static_asset_config = $8,\n edited_by = $9,\n email = $10,\n request_type = $11,\n authentication_method = $12,\n summary = $13,\n description = $14,\n edited_at = now(),\n is_static_website = $15,\n error_handler_path = $16,\n error_handler_args = $17,\n retry = $18\n WHERE\n workspace_id = $19 AND\n path = $20\n ", + "query": "\n UPDATE\n http_trigger\n SET\n wrap_body = $1,\n raw_string = $2,\n authentication_resource_path = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n enabled = $7,\n http_method = $8,\n static_asset_config = $9,\n edited_by = $10,\n email = $11,\n request_type = $12,\n authentication_method = $13,\n summary = $14,\n description = $15,\n edited_at = now(),\n is_static_website = $16,\n error_handler_path = $17,\n error_handler_args = $18,\n retry = $19\n WHERE\n workspace_id = $20 AND\n path = $21\n ", "describe": { "columns": [], "parameters": { @@ -11,6 +11,7 @@ "Varchar", "Varchar", "Bool", + "Bool", { "Custom": { "name": "http_method", @@ -67,5 +68,5 @@ }, "nullable": [] }, - "hash": "4c64bd0e364f536597db83161b5a27ff58b5ec7148bc94807423593bcfa27b90" + "hash": "ada1a14b4f25e41742df153b07e8bdf5b5feac9c6fa9de38bd24104bac8539d2" } diff --git a/backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json b/backend/.sqlx/query-b8bcf1fef244395dad802174dfc9bff45ff6f9cd1298eae5de40218cbab2eb96.json similarity index 70% rename from backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json rename to backend/.sqlx/query-b8bcf1fef244395dad802174dfc9bff45ff6f9cd1298eae5de40218cbab2eb96.json index 7869fe789e..5a0f76fc07 100644 --- a/backend/.sqlx/query-465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61.json +++ b/backend/.sqlx/query-b8bcf1fef244395dad802174dfc9bff45ff6f9cd1298eae5de40218cbab2eb96.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7,\n path = $8,\n is_flow = $9,\n http_method = $10,\n static_asset_config = $11,\n edited_by = $12,\n email = $13,\n request_type = $14,\n authentication_method = $15,\n summary = $16,\n description = $17,\n edited_at = now(),\n is_static_website = $18,\n error_handler_path = $19,\n error_handler_args = $20,\n retry = $21\n WHERE\n workspace_id = $22 AND\n path = $23\n ", + "query": "\n UPDATE\n http_trigger\n SET\n route_path = $1,\n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7,\n path = $8,\n is_flow = $9,\n enabled = $10,\n http_method = $11,\n static_asset_config = $12,\n edited_by = $13,\n email = $14,\n request_type = $15,\n authentication_method = $16,\n summary = $17,\n description = $18,\n edited_at = now(),\n is_static_website = $19,\n error_handler_path = $20,\n error_handler_args = $21,\n retry = $22\n WHERE\n workspace_id = $23 AND\n path = $24\n ", "describe": { "columns": [], "parameters": { @@ -14,6 +14,7 @@ "Varchar", "Varchar", "Bool", + "Bool", { "Custom": { "name": "http_method", @@ -70,5 +71,5 @@ }, "nullable": [] }, - "hash": "465144ea7e2930203618d9814a3e20c77b4363cf9e7c655d395f3fe40c247f61" + "hash": "b8bcf1fef244395dad802174dfc9bff45ff6f9cd1298eae5de40218cbab2eb96" } diff --git a/backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json b/backend/.sqlx/query-bbc96ae911d4ca0330582340e65c10e008e507610f095ff3936865101c9ba346.json similarity index 95% rename from backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json rename to backend/.sqlx/query-bbc96ae911d4ca0330582340e65c10e008e507610f095ff3936865101c9ba346.json index 2999ae4bf2..5b9959ce13 100644 --- a/backend/.sqlx/query-1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50.json +++ b/backend/.sqlx/query-bbc96ae911d4ca0330582340e65c10e008e507610f095ff3936865101c9ba346.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n email,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1\n ", + "query": "\n SELECT\n path,\n script_path,\n is_flow,\n route_path,\n authentication_resource_path,\n workspace_id,\n request_type AS \"request_type: _\",\n authentication_method AS \"authentication_method: _\",\n edited_by,\n email,\n static_asset_config AS \"static_asset_config: _\",\n wrap_body,\n raw_string,\n workspaced_route,\n is_static_website,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n http_trigger\n WHERE\n http_method = $1 AND\n enabled is TRUE\n ", "describe": { "columns": [ { @@ -158,5 +158,5 @@ true ] }, - "hash": "1301f873a829db137573b8b39449f6160f2adf44f864f26a99b8eab5818fbd50" + "hash": "bbc96ae911d4ca0330582340e65c10e008e507610f095ff3936865101c9ba346" } diff --git a/backend/.sqlx/query-cabc79429825b5f4861e4ba2001a5f235f39558dee88cbb486d7ac988d0a44d1.json b/backend/.sqlx/query-cabc79429825b5f4861e4ba2001a5f235f39558dee88cbb486d7ac988d0a44d1.json new file mode 100644 index 0000000000..9e4ee46248 --- /dev/null +++ b/backend/.sqlx/query-cabc79429825b5f4861e4ba2001a5f235f39558dee88cbb486d7ac988d0a44d1.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO usage (id, is_workspace, month_, usage)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Bool", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "cabc79429825b5f4861e4ba2001a5f235f39558dee88cbb486d7ac988d0a44d1" +} diff --git a/backend/.sqlx/query-cccf9e216d84dfbb801b92d697496f44d93b344190a26c8393867b3a268b9aba.json b/backend/.sqlx/query-cccf9e216d84dfbb801b92d697496f44d93b344190a26c8393867b3a268b9aba.json deleted file mode 100644 index 4a34830b0e..0000000000 --- a/backend/.sqlx/query-cccf9e216d84dfbb801b92d697496f44d93b344190a26c8393867b3a268b9aba.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme', $2, now()) ON CONFLICT DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "cccf9e216d84dfbb801b92d697496f44d93b344190a26c8393867b3a268b9aba" -} diff --git a/backend/.sqlx/query-d2732640f09ec029025ebdd4de502c8adee995eac05b5051e480f0a20fa6b7bb.json b/backend/.sqlx/query-d2732640f09ec029025ebdd4de502c8adee995eac05b5051e480f0a20fa6b7bb.json new file mode 100644 index 0000000000..fda5b760b3 --- /dev/null +++ b/backend/.sqlx/query-d2732640f09ec029025ebdd4de502c8adee995eac05b5051e480f0a20fa6b7bb.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM unique_ext_jwt_token WHERE last_used_at > NOW() - INTERVAL '30 days'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "d2732640f09ec029025ebdd4de502c8adee995eac05b5051e480f0a20fa6b7bb" +} diff --git a/backend/.sqlx/query-d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772.json b/backend/.sqlx/query-d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772.json deleted file mode 100644 index a5329c6a9b..0000000000 --- a/backend/.sqlx/query-d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n flow \n SET\n path = $1,\n summary = $2,\n description = $3,\n dependency_job = NULL,\n lock_error_logs = '',\n draft_only = NULL,\n tag = $4,\n dedicated_worker = $5,\n visible_to_runner_only = $6,\n on_behalf_of_email = $7,\n value = $8,\n schema = $9::text::json,\n edited_by = $10,\n edited_at = now()\n WHERE \n path = $11 AND workspace_id = $12", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Varchar", - "Bool", - "Bool", - "Text", - "Jsonb", - "Text", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d2acc380c45df31f741bbd420c2f4a5b962f7803def9ecd6e33d2aec6abce772" -} diff --git a/backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json b/backend/.sqlx/query-df6972fb16a2364f10379fe37e125ae228697602e3769c479582470dc0b9488b.json similarity index 91% rename from backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json rename to backend/.sqlx/query-df6972fb16a2364f10379fe37e125ae228697602e3769c479582470dc0b9488b.json index ab4e5e5d36..1e5c26b2e7 100644 --- a/backend/.sqlx/query-bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356.json +++ b/backend/.sqlx/query-df6972fb16a2364f10379fe37e125ae228697602e3769c479582470dc0b9488b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE workspace_id = $1 \n AND local_part = $2 \n AND (workspaced_local_part = TRUE OR $3 IS TRUE)\n ", + "query": "\n SELECT \n script_path, \n is_flow, \n workspace_id, \n edited_by, \n email, \n path, \n error_handler_path as \"error_handler_path: _\", \n error_handler_args as \"error_handler_args: _\", \n retry as \"retry: _\" \n FROM email_trigger \n WHERE workspace_id = $1 \n AND local_part = $2 \n AND (workspaced_local_part = TRUE OR $3 IS TRUE)\n AND enabled IS TRUE\n ", "describe": { "columns": [ { @@ -68,5 +68,5 @@ true ] }, - "hash": "bbd51f4f0a8bb2db5d6b634f2d32f5f7f7f57390a0e4b6e7cad45c9147fd6356" + "hash": "df6972fb16a2364f10379fe37e125ae228697602e3769c479582470dc0b9488b" } diff --git a/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json b/backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json similarity index 56% rename from backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json rename to backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json index 3867319977..15e83648ec 100644 --- a/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json +++ b/backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) \n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", "describe": { "columns": [], "parameters": { @@ -13,10 +13,11 @@ "Varchar", "Varchar", "Int8", - "Int8" + "Int8", + "Text" ] }, "nullable": [] }, - "hash": "e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2" + "hash": "ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c" } diff --git a/backend/.sqlx/query-6fafc23924eded970689040bd4a94d4d23ebee4f2b7d37bb54c47edb7720be00.json b/backend/.sqlx/query-fe12006498b9e7aece6b104fc5d9a80eeac12c6ca07a0e8950941ba796b0ba79.json similarity index 73% rename from backend/.sqlx/query-6fafc23924eded970689040bd4a94d4d23ebee4f2b7d37bb54c47edb7720be00.json rename to backend/.sqlx/query-fe12006498b9e7aece6b104fc5d9a80eeac12c6ca07a0e8950941ba796b0ba79.json index 706bcacdb2..150d2d9504 100644 --- a/backend/.sqlx/query-6fafc23924eded970689040bd4a94d4d23ebee4f2b7d37bb54c47edb7720be00.json +++ b/backend/.sqlx/query-fe12006498b9e7aece6b104fc5d9a80eeac12c6ca07a0e8950941ba796b0ba79.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10\n WHERE \n workspace_id = $11 AND path = $12\n ", + "query": "\n UPDATE email_trigger \n SET \n script_path = $1,\n path = $2,\n is_flow = $3,\n local_part = $4,\n workspaced_local_part = $5,\n edited_by = $6,\n email = $7,\n edited_at = now(),\n error_handler_path = $8,\n error_handler_args = $9,\n retry = $10,\n enabled = $11\n WHERE \n workspace_id = $12 AND path = $13\n ", "describe": { "columns": [], "parameters": { @@ -15,11 +15,12 @@ "Varchar", "Jsonb", "Jsonb", + "Bool", "Text", "Text" ] }, "nullable": [] }, - "hash": "6fafc23924eded970689040bd4a94d4d23ebee4f2b7d37bb54c47edb7720be00" + "hash": "fe12006498b9e7aece6b104fc5d9a80eeac12c6ca07a0e8950941ba796b0ba79" } diff --git a/backend/CLAUDE.md b/backend/CLAUDE.md index 1fe509a0d6..78faa0bf77 100644 --- a/backend/CLAUDE.md +++ b/backend/CLAUDE.md @@ -10,3 +10,16 @@ 1. Update database schema with migration if necessary 2. Update backend/windmill-api/openapi.yaml after modifying API endpoints + +## Querying the Database + +To query the database directly, use psql with the following connection string: + +```bash +psql postgres://postgres:changeme@localhost:5432/windmill +``` + +This can be helpful for: +- Inspecting database state during development +- Testing queries before implementing them in Rust +- Debugging data-related issues diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6452d027a9..7c9a4f373c 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -199,22 +199,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86590e57ea40121d47d3f2e131bfd873dea15d78dc2f4604f4734537ad9e56c4" +checksum = "b01c9521fa01558f750d183c8c68c81b0155b9d193a4ba7f84c36bd1b6d04a06" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -823,13 +823,14 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.10" +version = "1.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c034a1bc1d70e16e7f4e4caf7e9f7693e4c9c24cd91cf17c2a0b21abaebc7c8b" +checksum = "7ce527fb7e53ba9626fc47824f25e256250556c40d8f81d27dd92aa38239d632" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -845,6 +846,31 @@ dependencies = [ "uuid", ] +[[package]] +name = "aws-sdk-bedrockruntime" +version = "1.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5d2b8f081b9e8ff455b8dd7387b6b02263c3dac73172d188d2b523ff1e775e9" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "hyper 0.14.32", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-config" version = "1.68.0" @@ -868,6 +894,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-rds" +version = "1.117.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56b116d0229c41d5b72050dda07aa2cc7d1c571de8f7870c8b87fa297a724982" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sqs" version = "1.77.0" @@ -964,6 +1015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c35452ec3f001e1f2f6db107b6373f1f48f05ec63ba2c5c9fa91f07dad32af11" dependencies = [ "aws-credential-types", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -990,12 +1042,24 @@ dependencies = [ "tokio", ] +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e29a304f8319781a39808847efb39561351b1bb76e933da7aa90232673638658" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.62.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "445d5d720c99eed0b4aa674ed00d835d9b1427dd73e04adaf2f94c6b2d6f9fca" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", @@ -1013,9 +1077,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.6" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f108f1ca850f3feef3009bdcc977be201bca9a91058864d9de0684e64514bee0" +checksum = "623254723e8dfd535f566ee7b2381645f8981da086b5c4aa26c0c41582bb1d2c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1026,16 +1090,17 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-native-certs 0.8.2", "rustls-pki-types", "tokio", + "tokio-rustls 0.26.4", "tower 0.5.2", "tracing", ] @@ -1070,9 +1135,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.6" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e107ce0783019dbff59b3a244aa0c114e4a8c9d93498af9162608cd5474e796" +checksum = "0bbe9d018d646b96c7be063dd07987849862b0e6d07c778aad7d93d1be6c1ef0" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1182,7 +1247,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "itoa", "matchit", @@ -1569,7 +1634,7 @@ dependencies = [ "hex", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1799,9 +1864,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -1945,9 +2010,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.45" +version = "1.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36" dependencies = [ "find-msvc-tools", "jobserver", @@ -2054,9 +2119,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", "clap_derive", @@ -2064,9 +2129,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.51" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstream", "anstyle", @@ -3625,7 +3690,7 @@ dependencies = [ "hickory-resolver", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "ipnet", @@ -3713,7 +3778,7 @@ dependencies = [ "http 1.3.1", "httparse", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "itertools 0.10.5", "memmem", @@ -3906,7 +3971,7 @@ dependencies = [ "hkdf", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "idna", "indexmap 2.11.1", @@ -4177,7 +4242,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "libc", "log", @@ -4231,7 +4296,7 @@ dependencies = [ "deno_error", "deno_tls", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "log", @@ -4266,7 +4331,7 @@ dependencies = [ "deno_core", "deno_error", "deno_native_certs", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-pemfile 2.2.0", "rustls-tokio-stream", "rustls-webpki 0.102.8", @@ -4361,7 +4426,7 @@ dependencies = [ "h2 0.4.12", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "once_cell", "rustls-tokio-stream", @@ -5225,7 +5290,7 @@ dependencies = [ "base64 0.21.7", "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project", "rand 0.8.5", @@ -5292,9 +5357,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixedbitset" @@ -6262,9 +6327,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "hashify" @@ -6604,9 +6669,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -6635,7 +6700,7 @@ dependencies = [ "futures-util", "headers", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", @@ -6652,7 +6717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -6683,10 +6748,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "log", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-native-certs 0.8.2", "rustls-pki-types", "tokio", @@ -6701,7 +6766,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -6729,7 +6794,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -6750,7 +6815,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.8.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -6771,7 +6836,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -7419,7 +7484,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-http-proxy", "hyper-rustls 0.27.7", "hyper-timeout", @@ -7428,7 +7493,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem 3.0.5", - "rustls 0.23.29", + "rustls 0.23.35", "secrecy", "serde", "serde_json", @@ -7913,7 +7978,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-pki-types", "smtp-proto", "tokio", @@ -8946,7 +9011,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.8.0", + "hyper 1.8.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot 0.12.5", @@ -10246,7 +10311,7 @@ checksum = "7ada44a88ef953a3294f6eb55d2007ba44646015e18613d2f213016379203ef3" dependencies = [ "ahash 0.8.12", "equivalent", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "parking_lot 0.12.5", ] @@ -10262,7 +10327,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.29", + "rustls 0.23.35", "socket2 0.6.1", "thiserror 2.0.17", "tokio", @@ -10282,7 +10347,7 @@ dependencies = [ "rand 0.9.0", "ring 0.17.14", "rustc-hash 2.1.1", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -10712,7 +10777,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", @@ -10724,7 +10789,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-native-certs 0.8.2", "rustls-pki-types", "serde", @@ -10772,7 +10837,7 @@ dependencies = [ "futures", "getrandom 0.2.16", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "parking_lot 0.11.2", "reqwest 0.12.24", "reqwest-middleware", @@ -10785,9 +10850,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "retry-policies" @@ -11146,9 +11211,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.29" +version = "0.23.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" dependencies = [ "aws-lc-rs", "log", @@ -11232,7 +11297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22557157d7395bc30727745b365d923f1ecc230c4c80b176545f3f4f08c46e33" dependencies = [ "futures", - "rustls 0.23.29", + "rustls 0.23.35", "socket2 0.5.10", "tokio", ] @@ -12297,7 +12362,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.29", + "rustls 0.23.35", "serde", "serde_json", "sha2 0.10.9", @@ -13771,7 +13836,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.29", + "rustls 0.23.35", "tokio", ] @@ -13945,7 +14010,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -14603,7 +14668,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.29", + "rustls 0.23.35", "rustls-pki-types", "serde", "serde_json", @@ -15148,7 +15213,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "aws-sdk-config", @@ -15177,7 +15242,7 @@ dependencies = [ "quote", "rand 0.9.0", "reqwest 0.12.24", - "rustls 0.23.29", + "rustls 0.23.35", "serde", "serde_json", "serde_yml", @@ -15204,11 +15269,13 @@ dependencies = [ "windmill-indexer", "windmill-queue", "windmill-worker", + "windows-service", + "windows-sys 0.52.0", ] [[package]] name = "windmill-api" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "argon2", @@ -15251,7 +15318,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15281,7 +15348,7 @@ dependencies = [ "rumqttc", "rust-embed", "rust_decimal", - "rustls 0.23.29", + "rustls 0.23.35", "samael", "serde", "serde_json", @@ -15329,7 +15396,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.575.4" +version = "1.582.2" dependencies = [ "base64 0.22.1", "chrono", @@ -15344,7 +15411,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.575.4" +version = "1.582.2" dependencies = [ "chrono", "lazy_static", @@ -15358,7 +15425,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "axum", @@ -15377,14 +15444,17 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "async-recursion", "async-stream", "async-trait", "aws-config", + "aws-credential-types", + "aws-sdk-rds", "aws-sdk-sts", + "aws-smithy-types", "aws-smithy-types-convert", "axum", "backon", @@ -15405,7 +15475,7 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.8.0", + "hyper 1.8.1", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15452,6 +15522,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "url", + "urlencoding", "uuid", "windmill-macros", "windmill-parser", @@ -15462,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.575.4" +version = "1.582.2" dependencies = [ "regex", "serde", @@ -15477,7 +15548,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "bytes", @@ -15501,7 +15572,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.575.4" +version = "1.582.2" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15513,7 +15584,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.575.4" +version = "1.582.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -15522,7 +15593,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "lazy_static", @@ -15534,7 +15605,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "serde_json", @@ -15546,7 +15617,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "gosyn", @@ -15558,7 +15629,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "lazy_static", @@ -15570,7 +15641,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "serde_json", @@ -15582,7 +15653,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "nu-parser", @@ -15593,7 +15664,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15604,7 +15675,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15616,7 +15687,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "async-recursion", @@ -15639,7 +15710,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "lazy_static", @@ -15653,7 +15724,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15670,7 +15741,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "lazy_static", @@ -15684,7 +15755,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "lazy_static", @@ -15702,7 +15773,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "serde", @@ -15713,7 +15784,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "async-recursion", @@ -15722,12 +15793,14 @@ dependencies = [ "chrono", "chrono-tz", "cron", + "dashmap 6.1.0", "futures", "futures-core", "hex", "hmac", "itertools 0.14.0", "lazy_static", + "once_cell", "prometheus", "quick_cache", "regex", @@ -15748,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.575.4" +version = "1.582.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15758,13 +15831,17 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.575.4" +version = "1.582.2" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-bedrockruntime", + "aws-smithy-types", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -16088,6 +16165,17 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-service" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24d6bcc7f734a4091ecf8d7a64c5f7d7066f45585c1861eba06449909609c8a" +dependencies = [ + "bitflags 2.9.4", + "widestring", + "windows-sys 0.52.0", +] + [[package]] name = "windows-strings" version = "0.4.2" @@ -16540,18 +16628,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +checksum = "43fa6694ed34d6e57407afbccdeecfa268c470a7d2a5b0cf49ce9fcc345afb90" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.27" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +checksum = "c640b22cd9817fae95be82f0d2f90b11f7605f6c319d16705c459b27ac2cbc26" dependencies = [ "proc-macro2", "quote", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 457c9f91dd..45ff97c15e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.575.4" +version = "1.582.2" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.575.4" +version = "1.582.2" authors = ["Ruben Fiszel "] edition = "2021" @@ -161,6 +161,10 @@ nom.workspace = true globset.workspace = true +[target.'cfg(windows)'.dependencies] +windows-service = "0.7" +windows-sys = { version = "0.52", features = ["Win32_System_Services", "Win32_System_Console", "Win32_Foundation"] } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -264,6 +268,7 @@ regex = "^1" semver = "^1" aws-sigv4 = "^1.3.4" aws-sdk-config = "=1.68.0" +aws-sdk-rds = "^1" async-trait = "0.1.88" @@ -328,6 +333,7 @@ dyn-iter = "0.2.0" rsa = "^0" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" +dashmap = "6.1.0" gosyn = "0.2.6" bytes = "1.4.0" gethostname = "0.4.3" @@ -380,11 +386,14 @@ datafusion = "47.0.0" object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7", features = ["aws", "azure", "gcp"] } openidconnect = { version = "4.0.0-rc.1" } aws-config = "^1" +aws-sdk-bedrockruntime = "=1.113.0" +aws-credential-types = "^1" +aws-smithy-types = "^1" aws-sdk-sqs = "=1.77.0" aws-sdk-sts = "=1.79.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" -rustls = "=0.23.29" +rustls = "=0.23.35" async-once-cell = "0.5.4" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f6823b4692..5e8cbdea21 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -31ee9d3449f05cd0328c0fcf43e2b161dee767b9 \ No newline at end of file +c90bfc11c7c643b2c8c5111f092e7ae142318e9d \ No newline at end of file diff --git a/backend/migrations/20251028111213_allow_to_disable_http_trigger.down.sql b/backend/migrations/20251028111213_allow_to_disable_http_trigger.down.sql new file mode 100644 index 0000000000..f743944360 --- /dev/null +++ b/backend/migrations/20251028111213_allow_to_disable_http_trigger.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE http_trigger DROP COLUMN enabled; \ No newline at end of file diff --git a/backend/migrations/20251028111213_allow_to_disable_http_trigger.up.sql b/backend/migrations/20251028111213_allow_to_disable_http_trigger.up.sql new file mode 100644 index 0000000000..ad28eeb7f4 --- /dev/null +++ b/backend/migrations/20251028111213_allow_to_disable_http_trigger.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE http_trigger ADD COLUMN enabled BOOLEAN DEFAULT TRUE NOT NULL; \ No newline at end of file diff --git a/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql b/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql new file mode 100644 index 0000000000..d6fbf14f23 --- /dev/null +++ b/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql @@ -0,0 +1,2 @@ +-- Rollback: Remove job_isolation column from worker_ping table +ALTER TABLE worker_ping DROP COLUMN IF EXISTS job_isolation; diff --git a/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql b/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql new file mode 100644 index 0000000000..b7b2674ebd --- /dev/null +++ b/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql @@ -0,0 +1,4 @@ +-- Add job_isolation column to worker_ping table +-- This tracks which job isolation method the worker is using: 'nsjail', 'unshare', or 'none' +-- Nullable for backwards compatibility - old workers will report NULL +ALTER TABLE worker_ping ADD COLUMN job_isolation TEXT; diff --git a/backend/migrations/20251117222148_email_enabled.down.sql b/backend/migrations/20251117222148_email_enabled.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251117222148_email_enabled.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251117222148_email_enabled.up.sql b/backend/migrations/20251117222148_email_enabled.up.sql new file mode 100644 index 0000000000..861d171af9 --- /dev/null +++ b/backend/migrations/20251117222148_email_enabled.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here + +ALTER TABLE email_trigger ADD COLUMN enabled BOOLEAN DEFAULT TRUE NOT NULL; \ No newline at end of file diff --git a/backend/migrations/20251118190643_unique_jwt_token.down.sql b/backend/migrations/20251118190643_unique_jwt_token.down.sql new file mode 100644 index 0000000000..8e1dca2f24 --- /dev/null +++ b/backend/migrations/20251118190643_unique_jwt_token.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here + +DROP TABLE IF EXISTS unique_ext_jwt_token; diff --git a/backend/migrations/20251118190643_unique_jwt_token.up.sql b/backend/migrations/20251118190643_unique_jwt_token.up.sql new file mode 100644 index 0000000000..bf7613c35d --- /dev/null +++ b/backend/migrations/20251118190643_unique_jwt_token.up.sql @@ -0,0 +1,9 @@ +-- Add up migration script here + +CREATE TABLE IF NOT EXISTS unique_ext_jwt_token ( + jwt_hash BIGINT PRIMARY KEY NOT NULL, + last_used_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_unique_ext_jwt_token_last_used_at ON unique_ext_jwt_token(last_used_at); + diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 5bfe6b20c5..023e41a902 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -46,8 +46,6 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { lazy_static::lazy_static! { static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?\r?$"#).unwrap(); - - static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[([\w\[\]]+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?\r?"#).unwrap(); } fn parse_bash_file(code: &str) -> anyhow::Result>> { @@ -84,6 +82,37 @@ fn parse_bash_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +/// Extract a PowerShell param() block with its preceding attributes, handling nested parentheses. +/// +/// # Arguments +/// * `code` - The PowerShell code to extract from +/// * `include_attributes` - If true, includes attributes like [CmdletBinding()] before param +/// +/// # Returns +/// A tuple of (param_block_with_attributes, remaining_code) or None if not found. +/// - `param_block_with_attributes`: The param block including attributes/comments if include_attributes is true +/// - `remaining_code`: The rest of the code after the param block +/// +/// This function uses the existing extract_powershell_param_block validation, which already +/// ensures that only comments, whitespace, and attributes appear before param. So we can +/// simply return everything from the beginning to the end of the param block. +pub fn extract_powershell_param_block_with_attributes(code: &str, include_attributes: bool) -> Option<(&str, &str)> { + // First, use the existing function to validate and find the param block + let param_block = extract_powershell_param_block(code, true)?; + + // Find where the param block ends in the original code + let param_end_pos = code.find(param_block)? + param_block.len(); + + // If include_attributes is true, we start from the beginning (position 0) + // since we know everything before param is valid (comments/whitespace/attributes) + // Otherwise, start from where param begins + if include_attributes { + Some((&code[..param_end_pos], &code[param_end_pos..])) + } else { + Some((param_block, &code[param_end_pos..])) + } +} + /// Extract a PowerShell param() block, handling nested parentheses. /// /// # Arguments @@ -94,36 +123,89 @@ fn parse_bash_file(code: &str) -> anyhow::Result>> { /// # Returns /// The extracted param block or contents, or None if not found. pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Option<&str> { - // Find "param" keyword (case-insensitive) - let lower_code = code.to_lowercase(); - let param_start = lower_code.find("param")?; - - // Verify that only comments and whitespace appear before "param" - let before_param = &code[..param_start]; - let mut chars = before_param.chars().peekable(); + // Scan through the code looking for "param" while validating everything before it + let mut chars = code.chars().enumerate().peekable(); let mut in_block_comment = false; + let mut in_attribute_bracket = false; + let mut bracket_depth = 0; + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut param_start = None; - while let Some(ch) = chars.next() { + while let Some((idx, ch)) = chars.next() { if in_block_comment { // Check for end of block comment: #> - if ch == '#' && chars.peek() == Some(&'>') { + if ch == '#' && chars.peek().map(|(_, c)| c) == Some(&'>') { chars.next(); // consume '>' in_block_comment = false; } + } else if in_attribute_bracket { + // Handle quotes inside attributes to avoid counting brackets/parens inside strings + if in_single_quote { + if ch == '\'' { + in_single_quote = false; + } + } else if in_double_quote { + if ch == '"' { + in_double_quote = false; + } + } else { + // Track bracket depth to handle nested brackets/parens in attributes + match ch { + '\'' => in_single_quote = true, + '"' => in_double_quote = true, + '[' => bracket_depth += 1, + ']' => { + bracket_depth -= 1; + if bracket_depth == 0 { + in_attribute_bracket = false; + } + } + _ => {} + } + } } else { match ch { // Start of block comment: <# - '<' if chars.peek() == Some(&'#') => { + '<' if chars.peek().map(|(_, c)| c) == Some(&'#') => { chars.next(); // consume '#' in_block_comment = true; } // Single-line comment: consume until end of line '#' => { - while let Some(&next_ch) = chars.peek() { + for (_, next_ch) in chars.by_ref() { if next_ch == '\n' || next_ch == '\r' { break; } - chars.next(); + } + } + // Start of attribute bracket (e.g., [CmdletBinding()]) + '[' => { + in_attribute_bracket = true; + bracket_depth = 1; + } + // Check if we've found "param" (case-insensitive) + 'p' | 'P' => { + // Check if this is the start of "param" keyword + let remaining = &code[idx..]; + if remaining.len() >= 5 { + let next_four = &remaining[1..5]; + if next_four.eq_ignore_ascii_case("aram") { + // Check word boundaries + let before_ok = idx == 0 || { + let before = code.as_bytes()[idx - 1]; + !before.is_ascii_alphanumeric() && before != b'_' + }; + let after_ok = idx + 5 >= code.len() || { + let after = code.as_bytes()[idx + 5]; + !after.is_ascii_alphanumeric() && after != b'_' + }; + + if before_ok && after_ok { + param_start = Some(idx); + break; + } + } } } // Whitespace is allowed @@ -134,11 +216,13 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti } } - // If we're still in a block comment at the end, it's unclosed - invalid - if in_block_comment { + // If we're still in a block comment or unclosed attribute bracket, it's invalid + if in_block_comment || in_attribute_bracket { return None; } + let param_start = param_start?; + // Skip whitespace and tabs after "param" let mut chars = code[param_start + 5..].char_indices(); let mut paren_offset = param_start + 5; @@ -214,108 +298,361 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti None } -enum ParserState { - Normal, - InSingleQuote, - InDoubleQuote, -} -fn split_pwsh_args(code: &str) -> Vec<&str> { - let mut chars = code.char_indices().peekable(); - let mut state = ParserState::Normal; - let mut splits = vec![]; - let mut last_idx = 0; - while let Some((idx, char)) = chars.next() { - match (&state, char) { - (ParserState::Normal, '\'') => { - state = ParserState::InSingleQuote; - } - (ParserState::Normal, '"') => { - state = ParserState::InDoubleQuote; - } - (ParserState::InSingleQuote, '\'') => { - state = ParserState::Normal; - } - (ParserState::InDoubleQuote, '"') => { - state = ParserState::Normal; - } - (ParserState::Normal, ',') => { - splits.push(&code[last_idx..idx]); - last_idx = idx + 1; // skip the comma - } - _ => {} - } - } - - if last_idx < code.len() { - splits.push(&code[last_idx..]); - } - - splits -} - fn parse_powershell_single_typ(typ: &str) -> Typ { match typ.to_lowercase().as_str() { "string" => Typ::Str(None), "int" | "long" => Typ::Int, "decimal" | "double" | "single" => Typ::Float, "datetime" => Typ::Datetime, - "bool" => Typ::Bool, + "bool" | "switch" => Typ::Bool, "pscustomobject" => Typ::Object(ObjectType::new(None, None)), _ => Typ::Str(None), } } -fn parse_powershell_file(code: &str) -> anyhow::Result>> { - let param_wrapper = extract_powershell_param_block(code, false); - let mut args = vec![]; - if let Some(param_wrapper) = param_wrapper { - let params = split_pwsh_args(param_wrapper); - for param in params { - if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) { - let typ = cap.get(1).map(|x| x.as_str().to_string()); - let name = cap.get(2).unwrap().as_str().to_string(); +/// Parse ValidateSet attribute to extract enum values +/// Example: ValidateSet('Red', 'Green', 'Blue') -> Some(vec!["Red", "Green", "Blue"]) +fn parse_validate_set(bracket_content: &str) -> Option> { + // Find the opening parenthesis + let start = bracket_content.find('(')?; + let end = bracket_content.rfind(')')?; - let mut parsed_typ = if let Some(typ) = typ { - if typ.as_str().ends_with("[]") { - Some(Typ::List(Box::new(parse_powershell_single_typ( - typ.as_str().strip_suffix("[]").unwrap(), - )))) - } else { - Some(parse_powershell_single_typ(typ.as_str())) - } + if start >= end { + return None; + } + + let values_str = &bracket_content[start + 1..end]; + let mut values = Vec::new(); + let mut current_value = String::new(); + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escape_next = false; + + for ch in values_str.chars() { + if escape_next { + current_value.push(ch); + escape_next = false; + continue; + } + + match ch { + '`' if in_double_quote => { + escape_next = true; + } + '\'' if !in_double_quote => { + if in_single_quote { + // End of single-quoted string + values.push(current_value.clone()); + current_value.clear(); + in_single_quote = false; } else { - None - }; + // Start of single-quoted string + in_single_quote = true; + } + } + '"' if !in_single_quote => { + if in_double_quote { + // End of double-quoted string + values.push(current_value.clone()); + current_value.clear(); + in_double_quote = false; + } else { + // Start of double-quoted string + in_double_quote = true; + } + } + ',' if !in_single_quote && !in_double_quote => { + // Skip commas outside quotes + continue; + } + c if in_single_quote || in_double_quote => { + current_value.push(c); + } + c if !c.is_whitespace() => { + // Handle unquoted values (though PowerShell typically requires quotes) + current_value.push(c); + } + _ => {} + } + } - let default = if let Some(x) = cap.get(3) { - Some(json!(x.as_str().to_string())) - } else if let Some(x) = cap.get(4) { - if parsed_typ.is_none() { - if x.as_str().parse::().is_ok() { - parsed_typ = Some(Typ::Int); - } else if x.as_str().parse::().is_ok() { - parsed_typ = Some(Typ::Float); + // Handle any remaining unquoted value + if !current_value.is_empty() { + values.push(current_value.trim().to_string()); + } + + if values.is_empty() { + None + } else { + Some(values) + } +} + +/// Single-pass PowerShell parameter parser. +/// Parses the content of a param() block and extracts all parameter information. +/// +/// This function processes PowerShell parameter declarations in a single pass, handling: +/// - Parameter attributes: [Parameter(Mandatory)], [Parameter(Mandatory=$true)], [ValidateSet(...)], etc. +/// - Type annotations: [string], [int[]], [PSCustomObject], etc. +/// - Variable names: $Name, $Value, etc. +/// - Default values: = 'text', = 25, = $env:VAR, etc. +/// - Mandatory detection: Parameters with Mandatory attribute are marked as required +fn parse_powershell_parameters(content: &str) -> anyhow::Result> { + #[derive(Debug, PartialEq)] + enum State { + Normal, + InSingleQuote, + InDoubleQuote, + InBracket, + } + + let mut args = Vec::new(); + let mut chars = content.char_indices().peekable(); + let mut state = State::Normal; + let mut bracket_depth: i32 = 0; + let mut paren_depth: i32 = 0; + + // Current parameter being built + let mut type_annotation: Option = None; + let mut var_name: Option = None; + let mut default_value: Option = None; + let mut is_mandatory = false; + let mut validate_set: Option> = None; + + // Track position for extracting text + let mut last_bracket_start = None; + let mut found_dollar = false; + + while let Some((idx, ch)) = chars.next() { + match state { + State::InSingleQuote => { + if ch == '\'' { + state = State::Normal; + } + } + State::InDoubleQuote => { + if ch == '"' { + // Check for escape character + if idx > 0 && content.chars().nth(idx - 1) != Some('`') { + state = State::Normal; + } + } + } + State::InBracket => { + match ch { + '[' => bracket_depth += 1, + ']' => { + bracket_depth -= 1; + if bracket_depth == 0 { + // Extract the bracket content + if let Some(start) = last_bracket_start { + let bracket_content = &content[start + 1..idx]; + + // Check if this is a Parameter attribute with Mandatory (case-insensitive) + let lower = bracket_content.to_lowercase(); + if lower.starts_with("parameter(") || lower.starts_with("parameter ") { + // Check for Mandatory (case-insensitive) + if lower.contains("mandatory") { + // Check if it's explicitly set to false + if !lower.contains("mandatory=$false") && !lower.contains("mandatory = $false") { + is_mandatory = true; + } + } + } + + // Check if this is a ValidateSet attribute + if lower.starts_with("validateset(") { + // Extract values from ValidateSet('val1', 'val2', ...) + if let Some(values) = parse_validate_set(bracket_content) { + validate_set = Some(values); + } + } + + // Check if this looks like a type (simple word, possibly with []) + let is_type = !bracket_content.contains('(') + && !bracket_content.contains('=') + && (bracket_content.chars().next().unwrap_or(' ').is_alphabetic() + || bracket_content.starts_with('[')); + + if is_type && !found_dollar { + type_annotation = Some(bracket_content.to_string()); + } + } + state = State::Normal; + last_bracket_start = None; } } - serde_json::Number::from_str(x.as_str()) - .ok() - .map(serde_json::Value::Number) - } else { - None - }; + '(' => paren_depth += 1, + ')' => paren_depth = paren_depth.saturating_sub(1), + _ => {} + } + } + State::Normal => { + match ch { + '\'' => state = State::InSingleQuote, + '"' => state = State::InDoubleQuote, + '[' => { + state = State::InBracket; + bracket_depth = 1; + last_bracket_start = Some(idx); + } + '$' => { + found_dollar = true; + // Extract variable name + let name_start = idx + 1; + let mut name_end = name_start; + while let Some(&(_, next_ch)) = chars.peek() { + if next_ch.is_alphanumeric() || next_ch == '_' { + name_end += 1; + chars.next(); + } else { + break; + } + } + var_name = Some(content[name_start..name_end].to_string()); + } + '=' if found_dollar => { + // Extract default value + // Skip whitespace after = + while let Some(&(_, next_ch)) = chars.peek() { + if next_ch.is_whitespace() { + chars.next(); + } else { + break; + } + } - args.push(Arg { - name: name, - typ: parsed_typ.unwrap_or(Typ::Str(None)), - default: default.clone(), - otyp: None, - has_default: default.is_some(), - oidx: None, - }); + let default_start = chars.peek().map(|(i, _)| *i).unwrap_or(content.len()); + let mut default_end = default_start; + let mut in_string = false; + let mut string_char = ' '; + + while let Some((i, ch)) = chars.peek().copied() { + if in_string { + if ch == string_char && content.chars().nth(i.saturating_sub(1)) != Some('`') { + in_string = false; + default_end = i + 1; + chars.next(); + } else { + default_end = i + 1; + chars.next(); + } + } else if ch == '\'' || ch == '"' { + in_string = true; + string_char = ch; + default_end = i + 1; + chars.next(); + } else if ch == ',' { + break; + } else if ch.is_whitespace() && chars.clone().skip(1).next().map(|(_, c)| c) == Some(',') { + break; + } else { + default_end = i + 1; + chars.next(); + } + } + + default_value = Some(content[default_start..default_end].trim().to_string()); + } + ',' => { + // End of parameter, finalize it + if let Some(name) = var_name.take() { + args.push(finalize_parameter(name, type_annotation.take(), default_value.take(), is_mandatory, validate_set.take())?); + } + + // Reset for next parameter + type_annotation = None; + var_name = None; + default_value = None; + is_mandatory = false; + validate_set = None; + found_dollar = false; + } + _ => {} + } } } } - Ok(Some(args)) + + // Finalize last parameter + if let Some(name) = var_name { + args.push(finalize_parameter(name, type_annotation, default_value, is_mandatory, validate_set)?); + } + + Ok(args) +} + +fn finalize_parameter( + name: String, + type_annotation: Option, + default_value: Option, + is_mandatory: bool, + validate_set: Option>, +) -> anyhow::Result { + // Store the original PowerShell type for use in the executor + let otyp = type_annotation.clone(); + + // If ValidateSet is present, use it to create an enum type + let mut parsed_typ = if let Some(ref enum_values) = validate_set { + Some(Typ::Str(Some(enum_values.clone()))) + } else if let Some(typ) = type_annotation { + if typ.ends_with("[]") { + Some(Typ::List(Box::new(parse_powershell_single_typ( + typ.strip_suffix("[]").unwrap(), + )))) + } else { + Some(parse_powershell_single_typ(&typ)) + } + } else { + None + }; + + let default = if let Some(default_str) = default_value { + // Try to parse as string (quoted) + if (default_str.starts_with('"') && default_str.ends_with('"')) + || (default_str.starts_with('\'') && default_str.ends_with('\'')) + { + Some(json!(default_str[1..default_str.len() - 1].to_string())) + } else { + // Try to parse as number + if parsed_typ.is_none() { + if default_str.parse::().is_ok() { + parsed_typ = Some(Typ::Int); + } else if default_str.parse::().is_ok() { + parsed_typ = Some(Typ::Float); + } + } + serde_json::Number::from_str(&default_str) + .ok() + .map(serde_json::Value::Number) + } + } else { + None + }; + + // has_default semantics: + // - true: parameter is optional (has a default value OR is not mandatory) + // - false: parameter is required (marked as Mandatory AND no default value) + // Simplified: A parameter is optional unless it's mandatory without a default + let has_default = default.is_some() || !is_mandatory; + + Ok(Arg { + name, + typ: parsed_typ.unwrap_or(Typ::Str(None)), + default: default.clone(), + otyp, + has_default, + oidx: None, + }) +} + +fn parse_powershell_file(code: &str) -> anyhow::Result>> { + let param_wrapper = extract_powershell_param_block(code, false); + if let Some(param_wrapper) = param_wrapper { + Ok(Some(parse_powershell_parameters(param_wrapper)?)) + } else { + Ok(Some(vec![])) + } } #[cfg(test)] @@ -402,23 +739,23 @@ non_required="${5:-}" star_kwargs: false, args: vec![ Arg { - otyp: None, + otyp: None, // No type annotation name: "Msg".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string".to_string()), // [string] name: "Msg2".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: None, // No type annotation name: "Dflt".to_string(), typ: Typ::Str(None), default: Some(json!("default value, with comma")), @@ -426,7 +763,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: Some("int".to_string()), // [int] name: "Nb".to_string(), typ: Typ::Int, default: Some(json!(3)), @@ -434,7 +771,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // Type inferred from default value name: "Nb2".to_string(), typ: Typ::Float, default: Some(json!(5.0)), @@ -442,7 +779,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // Type inferred from default value name: "Nb3".to_string(), typ: Typ::Int, default: Some(json!(5)), @@ -450,35 +787,35 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // No type annotation name: "Wahoo".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("PSCustomObject".to_string()), // [PSCustomObject] name: "Obj".to_string(), typ: Typ::Object(ObjectType::new(None, None)), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string[]".to_string()), // [string[]] name: "Arr".to_string(), typ: Typ::List(Box::new(Typ::Str(None))), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string".to_string()), // [string] (last type bracket with Mandatory and ValidateSet) name: "Message".to_string(), - typ: Typ::Str(None), + typ: Typ::Str(Some(vec!["Green".to_string(), "Blue".to_string(), "Red".to_string()])), // ValidateSet enum default: None, - has_default: false, + has_default: false, // Required (Mandatory attribute) oidx: None } ], @@ -602,6 +939,405 @@ non_required="${5:-}" extract_powershell_param_block("function test-x{ param($Name)\n}", false), None ); + + // Valid: [CmdletBinding()] before param + assert_eq!( + extract_powershell_param_block("[CmdletBinding()]\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block("[CmdletBinding()]\nparam($Name, $Age)", true), + Some("param($Name, $Age)") + ); + + // Valid: [CmdletBinding()] with options before param + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding(SupportsShouldProcess=$true)]\nparam($Path)", + false + ), + Some("$Path") + ); + + // Valid: Multiple attributes before param + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding()]\n[OutputType([string])]\nparam($Value)", + false + ), + Some("$Value") + ); + + // Valid: CmdletBinding with comments + assert_eq!( + extract_powershell_param_block( + "# My function\n[CmdletBinding()]\nparam($Name)", + false + ), + Some("$Name") + ); + + // Valid: CmdletBinding with whitespace variations + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding()] \n param($Name)", + false + ), + Some("$Name") + ); + + // Invalid: Unclosed attribute bracket + assert_eq!( + extract_powershell_param_block("[CmdletBinding(\nparam($Name)", false), + None + ); + + // Valid: CmdletBinding with DefaultParameterSetName + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding(DefaultParameterSetName='ByName')]\nparam($Name, $Id)", + false + ), + Some("$Name, $Id") + ); + + // Valid: CmdletBinding with complex parameters + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding(DefaultParameterSetName='ByName', SupportsShouldProcess=$true)]\nparam($Path)", + false + ), + Some("$Path") + ); + + // Valid: Multiple attributes with parameters + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding(DefaultParameterSetName='Set1')]\n[OutputType([string])]\nparam($Value)", + false + ), + Some("$Value") + ); + } + + #[test] + fn test_parse_powershell_sig_with_parameter_attributes() -> anyhow::Result<()> { + // Test with [Parameter(Mandatory=$true)] attribute + let code = r#"[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$Name, + [Parameter(Mandatory=$false)] + [int]$Age = 25 +)"#; + let result = parse_powershell_sig(code)?; + assert_eq!(result.args.len(), 2); + assert_eq!(result.args[0].name, "Name"); + assert_eq!(result.args[0].typ, Typ::Str(None)); + assert_eq!(result.args[0].has_default, false); + assert_eq!(result.args[1].name, "Age"); + assert_eq!(result.args[1].typ, Typ::Int); + assert_eq!(result.args[1].has_default, true); + assert_eq!(result.args[1].default, Some(json!(25))); + + // Test with complex attributes including ValidateSet + let code2 = r#"param( + [Parameter(Mandatory=$true, Position=0)] + [ValidateSet('Red', 'Green', 'Blue')] + [string]$Color, + [Parameter(ValueFromPipeline=$true)] + [string[]]$Items +)"#; + let result2 = parse_powershell_sig(code2)?; + assert_eq!(result2.args.len(), 2); + assert_eq!(result2.args[0].name, "Color"); + assert_eq!( + result2.args[0].typ, + Typ::Str(Some(vec![ + "Red".to_string(), + "Green".to_string(), + "Blue".to_string() + ])) + ); + assert_eq!(result2.args[1].name, "Items"); + assert_eq!(result2.args[1].typ, Typ::List(Box::new(Typ::Str(None)))); + + Ok(()) + } + + #[test] + fn test_powershell_single_pass_parser() -> anyhow::Result<()> { + // Test the single-pass parser with a complex real-world example + let code = r#"[CmdletBinding()] +param( + [Parameter(Mandatory=$true, Position=0, HelpMessage="Enter the server name")] + [ValidateNotNullOrEmpty()] + [string]$ServerName, + + [Parameter(Mandatory=$false)] + [ValidateRange(1, 65535)] + [int]$Port = 8080, + + [Parameter(ValueFromPipeline=$true)] + [string[]]$LogFiles, + + [ValidateSet('Debug', 'Info', 'Warning', 'Error')] + [string]$LogLevel = 'Info', + + [PSCustomObject]$Config +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 5); + + // ServerName: mandatory string with no default + assert_eq!(result.args[0].name, "ServerName"); + assert_eq!(result.args[0].typ, Typ::Str(None)); + assert_eq!(result.args[0].has_default, false); + + // Port: optional int with default + assert_eq!(result.args[1].name, "Port"); + assert_eq!(result.args[1].typ, Typ::Int); + assert_eq!(result.args[1].default, Some(json!(8080))); + assert_eq!(result.args[1].has_default, true); + + // LogFiles: string array (no mandatory, so optional) + assert_eq!(result.args[2].name, "LogFiles"); + assert_eq!(result.args[2].typ, Typ::List(Box::new(Typ::Str(None)))); + assert_eq!(result.args[2].has_default, true); // Optional (not mandatory) + + // LogLevel: string with default and ValidateSet (creates enum type) + assert_eq!(result.args[3].name, "LogLevel"); + assert_eq!( + result.args[3].typ, + Typ::Str(Some(vec![ + "Debug".to_string(), + "Info".to_string(), + "Warning".to_string(), + "Error".to_string() + ])) + ); + assert_eq!(result.args[3].default, Some(json!("Info"))); + assert_eq!(result.args[3].has_default, true); + + // Config: PSCustomObject (no mandatory, so optional) + assert_eq!(result.args[4].name, "Config"); + assert_eq!(result.args[4].typ, Typ::Object(ObjectType::new(None, None))); + assert_eq!(result.args[4].has_default, true); // Optional (not mandatory) + + Ok(()) + } + + #[test] + fn test_powershell_mandatory_attribute() -> anyhow::Result<()> { + // Test various forms of the Mandatory attribute + let code = r#"param( + [Parameter(Mandatory)] + [string]$RequiredNoEquals, + + [Parameter(Mandatory=$true)] + [string]$RequiredWithTrue, + + [Parameter(Mandatory = $true)] + [string]$RequiredWithSpaces, + + [Parameter(Mandatory=$false)] + [string]$NotRequired, + + [Parameter(Position=0)] + [string]$NoMandatory, + + [string]$PlainRequired = "default", + + [Parameter(Mandatory=$true)] + [int]$RequiredInt +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 7); + + // RequiredNoEquals: mandatory without =$true + assert_eq!(result.args[0].name, "RequiredNoEquals"); + assert_eq!(result.args[0].has_default, false); // Required (mandatory, no default) + + // RequiredWithTrue: mandatory with =$true + assert_eq!(result.args[1].name, "RequiredWithTrue"); + assert_eq!(result.args[1].has_default, false); // Required + + // RequiredWithSpaces: mandatory with spaces + assert_eq!(result.args[2].name, "RequiredWithSpaces"); + assert_eq!(result.args[2].has_default, false); // Required + + // NotRequired: explicitly Mandatory=$false + assert_eq!(result.args[3].name, "NotRequired"); + assert_eq!(result.args[3].has_default, true); // Optional (not mandatory) + + // NoMandatory: no Mandatory attribute + assert_eq!(result.args[4].name, "NoMandatory"); + assert_eq!(result.args[4].has_default, true); // Optional (not mandatory) + + // PlainRequired: has default value (always optional) + assert_eq!(result.args[5].name, "PlainRequired"); + assert_eq!(result.args[5].has_default, true); // Optional (has default) + assert_eq!(result.args[5].default, Some(json!("default"))); + + // RequiredInt: mandatory int + assert_eq!(result.args[6].name, "RequiredInt"); + assert_eq!(result.args[6].typ, Typ::Int); + assert_eq!(result.args[6].has_default, false); // Required + + Ok(()) + } + + #[test] + fn test_extract_powershell_param_block_with_attributes() { + // Test without attributes + let code = "param($Name, $Age)"; + let result = extract_powershell_param_block_with_attributes(code, true); + assert_eq!(result, Some(("param($Name, $Age)", ""))); + + // Test with simple CmdletBinding + let code2 = "[CmdletBinding()]\nparam($Name)"; + let result2 = extract_powershell_param_block_with_attributes(code2, true); + assert_eq!(result2, Some(("[CmdletBinding()]\nparam($Name)", ""))); + + // Test with CmdletBinding with parameters + let code3 = "[CmdletBinding(DefaultParameterSetName='ByName')]\nparam($Name, $Id)"; + let result3 = extract_powershell_param_block_with_attributes(code3, true); + assert_eq!(result3, Some(("[CmdletBinding(DefaultParameterSetName='ByName')]\nparam($Name, $Id)", ""))); + + // Test with multiple attributes + let code4 = "[CmdletBinding()]\n[OutputType([string])]\nparam($Value)"; + let result4 = extract_powershell_param_block_with_attributes(code4, true); + assert_eq!(result4, Some(("[CmdletBinding()]\n[OutputType([string])]\nparam($Value)", ""))); + + // Test with comment before attributes + let code5 = "# My function\n[CmdletBinding()]\nparam($Name)"; + let result5 = extract_powershell_param_block_with_attributes(code5, true); + assert_eq!(result5, Some(("# My function\n[CmdletBinding()]\nparam($Name)", ""))); + + // Test with include_attributes = false (should only get param block, not attributes) + let code6 = "[CmdletBinding()]\nparam($Name)"; + let result6 = extract_powershell_param_block_with_attributes(code6, false); + assert_eq!(result6, Some(("param($Name)", ""))); + + // Test with code after param + let code7 = "[CmdletBinding()]\nparam($Name)\nWrite-Host 'Hello'"; + let result7 = extract_powershell_param_block_with_attributes(code7, true); + assert_eq!(result7, Some(("[CmdletBinding()]\nparam($Name)", "\nWrite-Host 'Hello'"))); + + // Test with code after param (without attributes) + let code8 = "[CmdletBinding()]\nparam($Name)\nWrite-Host 'Hello'"; + let result8 = extract_powershell_param_block_with_attributes(code8, false); + assert_eq!(result8, Some(("param($Name)", "\nWrite-Host 'Hello'"))); + } + + #[test] + fn test_powershell_sig_with_cmdletbinding_paramsetname() -> anyhow::Result<()> { + // Test with [CmdletBinding(DefaultParameterSetName='ByName')] + let code = r#"[CmdletBinding(DefaultParameterSetName='ByName')] +param( + [Parameter(Mandatory=$true, ParameterSetName='ByName')] + [string]$Name, + + [Parameter(Mandatory=$true, ParameterSetName='ById')] + [int]$Id, + + [string]$Description = "default description" +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 3); + + // Name: mandatory string + assert_eq!(result.args[0].name, "Name"); + assert_eq!(result.args[0].typ, Typ::Str(None)); + assert_eq!(result.args[0].has_default, false); + + // Id: mandatory int + assert_eq!(result.args[1].name, "Id"); + assert_eq!(result.args[1].typ, Typ::Int); + assert_eq!(result.args[1].has_default, false); + + // Description: optional with default + assert_eq!(result.args[2].name, "Description"); + assert_eq!(result.args[2].typ, Typ::Str(None)); + assert_eq!(result.args[2].default, Some(json!("default description"))); + assert_eq!(result.args[2].has_default, true); + + Ok(()) + } + + #[test] + fn test_powershell_case_insensitive_parameter() -> anyhow::Result<()> { + // Test that [parameter(...)] is case-insensitive + let code = r#"param( + [parameter(Mandatory)] + [string]$LowerCase, + + [PARAMETER(MANDATORY=$TRUE)] + [string]$UpperCase, + + [Parameter(mandatory=$true)] + [string]$MixedCase +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 3); + + // All should be detected as mandatory + assert_eq!(result.args[0].name, "LowerCase"); + assert_eq!(result.args[0].has_default, false); + + assert_eq!(result.args[1].name, "UpperCase"); + assert_eq!(result.args[1].has_default, false); + + assert_eq!(result.args[2].name, "MixedCase"); + assert_eq!(result.args[2].has_default, false); + + Ok(()) + } + + #[test] + fn test_powershell_validateset_enum() -> anyhow::Result<()> { + // Test with ValidateSet creating an enum type + let code = r#"param( + [ValidateSet('Red', 'Green', 'Blue')] + [string]$Color, + + [Parameter(Mandatory=$true)] + [ValidateSet("Small", "Medium", "Large")] + [string]$Size +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 2); + + // Color: optional with ValidateSet (enum) + assert_eq!(result.args[0].name, "Color"); + assert_eq!( + result.args[0].typ, + Typ::Str(Some(vec![ + "Red".to_string(), + "Green".to_string(), + "Blue".to_string() + ])) + ); + assert_eq!(result.args[0].has_default, true); // Optional (not mandatory) + + // Size: mandatory with ValidateSet (enum) + assert_eq!(result.args[1].name, "Size"); + assert_eq!( + result.args[1].typ, + Typ::Str(Some(vec![ + "Small".to_string(), + "Medium".to_string(), + "Large".to_string() + ])) + ); + assert_eq!(result.args[1].has_default, false); // Required (mandatory) + + Ok(()) } #[test] diff --git a/backend/run_until_fail.sh b/backend/run_until_fail.sh index d8c461504b..935bacc88c 100755 --- a/backend/run_until_fail.sh +++ b/backend/run_until_fail.sh @@ -1,6 +1,7 @@ #!/bin/bash # Run the command repeatedly until it fails (exits with non-zero code) +cd windmill-duckdb-ffi-internal && ./build_dev.sh && cd .. while true; do DISABLE_EMBEDDING=true \ RUST_LOG=info \ @@ -9,7 +10,7 @@ while true; do GO_PATH=$(which go) \ UV_PATH=$(which uv) \ CARGO_PATH=$(which cargo) \ - cargo test --features enterprise,deno_core,license,python,rust,scoped_cache \ + cargo test --features enterprise,deno_core,license,python,duckdb,rust,scoped_cache \ -- --nocapture --test-threads=8 | tee /tmp/test.log # Capture the exit code of the cargo test command (not tee) diff --git a/backend/src/main.rs b/backend/src/main.rs index 70f428d4ca..a619572f50 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -113,6 +113,10 @@ pub mod ee; mod ee_oss; mod monitor; +// Windows service support - EE feature +#[cfg(all(windows, feature = "enterprise", feature = "private"))] +mod windows_service_ee; + pub fn setup_deno_runtime() -> anyhow::Result<()> { // https://github.com/denoland/deno/blob/main/cli/main.rs#L477 #[cfg(feature = "deno_core")] @@ -207,6 +211,17 @@ lazy_static::lazy_static! { } pub fn main() -> anyhow::Result<()> { + // On Windows with enterprise feature, check if running as a service + #[cfg(all(windows, feature = "enterprise", feature = "private"))] + { + if windows_service_ee::is_running_as_service() { + // Run as Windows service with SCM handlers + return windows_service_ee::run_as_windows_service() + .map_err(|e| anyhow::anyhow!("Failed to run as Windows service: {}", e)); + } + } + + // Normal execution (console/foreground mode) setup_deno_runtime()?; create_and_run_current_thread_inner(windmill_main()) } @@ -496,7 +511,14 @@ async fn windmill_main() -> anyhow::Result<()> { conn } else { // This time we use a pool of connections - let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; + let db = windmill_common::connect_db( + server_mode, + indexer_mode, + worker_mode, + #[cfg(feature = "private")] + killpill_rx.resubscribe(), + ) + .await?; // NOTE: Variable/resource cache initialization moved to API server in windmill-api @@ -817,10 +839,10 @@ Windmill Community Edition {GIT_VERSION} match conn { Connection::Sql(ref db) => { let base_internal_url = base_internal_url.to_string(); - let db_url: String = get_database_url().await?; + let db_url = get_database_url().await?; let db = db.clone(); let h = tokio::spawn(async move { - let mut listener = retry_listen_pg(&db_url).await; + let mut listener = retry_listen_pg(&db_url.as_str().await).await; let mut last_listener_refresh = Instant::now(); let mut monitor_iteration: u64 = 0; let rd_shift: u8 = rand::rng().random_range(0..200); @@ -1154,13 +1176,14 @@ Windmill Community Edition {GIT_VERSION} }, Err(e) => { tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener"); + let db_url = db_url.clone(); tokio::select! { biased; _ = monitor_killpill_rx.recv() => { tracing::info!("received killpill for monitor job"); break; }, - new_listener = retry_listen_pg(&db_url) => { + new_listener = async move { retry_listen_pg(&db_url.as_str().await).await } => { listener = new_listener; continue; } @@ -1174,7 +1197,7 @@ Windmill Community Edition {GIT_VERSION} if let Err(e) = listener.unlisten_all().await { tracing::error!(error = %e, "Could not unlisten to database"); } - listener = retry_listen_pg(&db_url).await; + listener = retry_listen_pg(&db_url.as_str().await).await; initial_load( &conn, tx.clone(), diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 57d7833c9d..0a92dfc74d 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2944,3 +2944,33 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { .await; Ok(()) } + +#[cfg(feature = "duckdb")] +#[sqlx::test(fixtures("base"))] +async fn test_duckdb_ffi(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let content = "-- result_collection=last_statement_first_row_scalar\nSELECT 'Hello world!';"; + + let flow: FlowValue = serde_json::from_value(serde_json::json!({ + "modules": [{ + "value": { + "type": "rawscript", + "language": "duckdb", + "content": content, + }, + }], + })) + .unwrap(); + + let result = + RunJob::from(JobPayload::RawFlow { value: flow.clone(), path: None, restarted_from: None }) + .run_until_complete(&db, false, server.addr.port()) + .await + .json_result() + .unwrap(); + assert_eq!(result, serde_json::json!("Hello world!")); + Ok(()) +} diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e57812e2e2..10acdee52f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.575.4 + version: 1.582.2 title: Windmill API contact: @@ -572,12 +572,12 @@ paths: use_case: type: string responses: - '200': + "200": description: Onboarding data submitted successfully content: application/json: schema: - type: string + type: string /w/{workspace}/users/delete/{username}: delete: @@ -10163,6 +10163,34 @@ paths: schema: type: boolean + /w/{workspace}/http_triggers/setenabled/{path}: + post: + summary: enable/disable http trigger + operationId: setHttpTriggerEnabled + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: http trigger enable/disable + content: + text/plain: + schema: + type: string + /w/{workspace}/websocket_triggers/create: post: summary: create websocket trigger @@ -11970,6 +11998,33 @@ paths: application/json: schema: type: boolean + /w/{workspace}/email_triggers/setenabled/{path}: + post: + summary: enable/disable email trigger + operationId: setEmailTriggerEnabled + tags: + - email_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: email trigger enable/disable + content: + text/plain: + schema: + type: string /groups/list: get: @@ -15085,7 +15140,7 @@ components: schema: type: integer JobTriggerKind: - name: trigger_kind + name: trigger_kind description: trigger kind (schedule, http, websocket...) in: query schema: @@ -15266,8 +15321,7 @@ components: CreatedAfterQueue: name: created_after_queue - description: - filter on jobs created after X for jobs in the queue only + description: filter on jobs created after X for jobs in the queue only in: query schema: type: string @@ -15275,8 +15329,7 @@ components: CreatedBeforeQueue: name: created_before_queue - description: - filter on jobs created before X for jobs in the queue only + description: filter on jobs created before X for jobs in the queue only in: query schema: type: string @@ -15457,6 +15510,8 @@ components: $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatus" FlowStatusModule: $ref: "../../openflow.openapi.yaml#/components/schemas/FlowStatusModule" + FlowNote: + $ref: "../../openflow.openapi.yaml#/components/schemas/FlowNote" # -- INLINE END -- # Do not change line above @@ -15571,6 +15626,7 @@ components: groq, openrouter, togetherai, + aws_bedrock, customai, ] @@ -16722,30 +16778,30 @@ components: ScriptLang: type: string enum: [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - nu, - java, - ruby, - duckdb, - # for related places search: ADD_NEW_LANG - ] + python3, + deno, + go, + bash, + powershell, + postgresql, + mysql, + bigquery, + snowflake, + mssql, + oracledb, + graphql, + nativets, + bun, + php, + rust, + ansible, + csharp, + nu, + java, + ruby, + duckdb, + # for related places search: ADD_NEW_LANG + ] Preview: type: object @@ -17222,6 +17278,8 @@ components: format: date-time is_flow: type: boolean + enabled: + type: boolean required: - path - script_path @@ -17231,6 +17289,7 @@ components: - edited_by - edited_at - is_flow + - enabled AuthenticationMethod: type: string @@ -17456,6 +17515,8 @@ components: type: boolean wrap_body: type: boolean + enabled: + type: boolean raw_string: type: boolean error_handler_path: @@ -17581,8 +17642,6 @@ components: format: date-time error: type: string - enabled: - type: boolean filters: type: array items: @@ -17613,7 +17672,6 @@ components: required: - url - - enabled - filters - can_return_message - can_return_error_result @@ -17802,17 +17860,13 @@ components: format: date-time error: type: string - enabled: - type: boolean error_handler_path: type: string error_handler_args: $ref: "#/components/schemas/ScriptArgs" retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" - required: - - enabled - subscribe_topics - mqtt_resource_path @@ -17934,8 +17988,6 @@ components: format: date-time error: type: string - enabled: - type: boolean error_handler_path: type: string error_handler_args: @@ -17946,7 +17998,6 @@ components: - gcp_resource_path - topic_id - subscription_id - - enabled - delivery_type - subscription_mode @@ -18048,8 +18099,6 @@ components: format: date-time error: type: string - enabled: - type: boolean error_handler_path: type: string error_handler_args: @@ -18060,7 +18109,6 @@ components: required: - queue_url - aws_resource_path - - enabled - aws_auth_resource_type LoggedWizardStatus: @@ -18257,8 +18305,6 @@ components: - $ref: "#/components/schemas/TriggerExtraProperty" type: object properties: - enabled: - type: boolean postgres_resource_path: type: string publication_name: @@ -18279,7 +18325,6 @@ components: retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" required: - - enabled - postgres_resource_path - replication_slot_name - publication_name @@ -18370,8 +18415,6 @@ components: format: date-time error: type: string - enabled: - type: boolean error_handler_path: type: string error_handler_args: @@ -18383,7 +18426,6 @@ components: - kafka_resource_path - group_id - topics - - enabled NewKafkaTrigger: type: object @@ -18475,8 +18517,6 @@ components: format: date-time error: type: string - enabled: - type: boolean error_handler_path: type: string error_handler_args: @@ -18488,7 +18528,6 @@ components: - nats_resource_path - use_jetstream - subjects - - enabled NewNatsTrigger: type: object @@ -18602,7 +18641,8 @@ components: $ref: "#/components/schemas/ScriptArgs" retry: $ref: "../../openflow.openapi.yaml#/components/schemas/Retry" - + enabled: + type: boolean required: - path - script_path @@ -18764,6 +18804,8 @@ components: type: number wm_memory_usage: type: number + job_isolation: + type: string required: - worker - worker_instance diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index e4d07cfdc6..c9d4e69c5d 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -1,3 +1,4 @@ +use crate::bedrock; use crate::db::{ApiAuthed, DB}; use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; @@ -6,12 +7,12 @@ use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use windmill_common::variables::get_variable_or_self; use std::collections::HashMap; use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel, AZURE_API_VERSION}; +use windmill_common::ai_providers::{AIProvider, ProviderConfig, ProviderModel}; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::utils::configure_client; +use windmill_common::variables::get_variable_or_self; lazy_static::lazy_static! { static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new() @@ -66,6 +67,7 @@ struct AIStandardResource { #[serde(alias = "apiKey")] api_key: Option, organization_id: Option, + region: Option, } #[derive(Deserialize, Debug)] @@ -98,7 +100,9 @@ impl AIRequestConfig { ) -> Result { let (api_key, access_token, organization_id, base_url, user) = match resource { AIResource::Standard(resource) => { - let base_url = provider.get_base_url(resource.base_url, db).await?; + let base_url = provider + .get_base_url(resource.base_url, resource.region, db) + .await?; let api_key = if let Some(api_key) = resource.api_key { Some(get_variable_or_self(api_key, db, w_id).await?) } else { @@ -119,7 +123,7 @@ impl AIRequestConfig { None }; let token = Self::get_token_using_oauth(resource, db, w_id).await?; - let base_url = provider.get_base_url(None, db).await?; + let base_url = provider.get_base_url(None, None, db).await?; (None, Some(token), None, base_url, user) } @@ -180,15 +184,34 @@ impl AIRequestConfig { let is_azure = provider.is_azure_openai(base_url); let is_anthropic = matches!(provider, AIProvider::Anthropic); let is_anthropic_sdk = headers.get("X-Anthropic-SDK").is_some(); + let is_bedrock = matches!(provider, AIProvider::AWSBedrock); - let url = if is_azure && method != Method::GET { - let model = AIProvider::extract_model_from_body(&body)?; - AIProvider::build_azure_openai_url(base_url, &model, path) + // Handle AWS Bedrock transformation + let (url, body) = if is_bedrock && method != Method::GET { + let (model, transformed_body, is_streaming) = + bedrock::transform_openai_to_bedrock(&body)?; + let endpoint = if is_streaming { + "converse-stream" + } else { + "converse" + }; + let bedrock_url = format!("{}/model/{}/{}", base_url, model, endpoint); + (bedrock_url, transformed_body) + } else if is_bedrock && (path == "foundation-models" || path == "inference-profiles") { + // AWS Bedrock foundation-models and inference-profiles endpoints use different base URL (without -runtime) + let bedrock_base_url = base_url.replace("bedrock-runtime.", "bedrock."); + let bedrock_url = format!("{}/{}", bedrock_base_url, path); + (bedrock_url, body) + } else if is_azure { + let azure_url = AIProvider::build_azure_openai_url(base_url, path); + (azure_url, body) } else if is_anthropic_sdk { let truncated_base_url = base_url.trim_end_matches("/v1"); - format!("{}/{}", truncated_base_url, path) + let anthropic_url = format!("{}/{}", truncated_base_url, path); + (anthropic_url, body) } else { - format!("{}/{}", base_url, path) + let default_url = format!("{}/{}", base_url, path); + (default_url, body) }; tracing::debug!("AI request URL: {}", url); @@ -205,10 +228,6 @@ impl AIRequestConfig { request = request.body(body); - if is_azure { - request = request.query(&[("api-version", AZURE_API_VERSION)]) - } - if let Some(api_key) = self.api_key { if is_azure { request = request.header("api-key", api_key.clone()) @@ -316,7 +335,7 @@ async fn global_proxy( return Err(Error::BadRequest("API key is required".to_string())); }; - let base_url = provider.get_base_url(None, &db).await?; + let base_url = provider.get_base_url(None, None, &db).await?; let url = format!("{}/{}", base_url, ai_path); @@ -446,6 +465,21 @@ async fn proxy( } }; + // Extract model and streaming flag for Bedrock transformation (only for POST requests) + let (model, is_streaming) = + if matches!(provider, AIProvider::AWSBedrock) && method == Method::POST { + #[derive(Deserialize, Debug)] + struct BedrockRequest { + model: String, + stream: bool, + } + let parsed: BedrockRequest = serde_json::from_slice(&body) + .map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?; + (Some(parsed.model), parsed.stream) + } else { + (None, false) + }; + let request = request_config.prepare_request(&provider, &ai_path, method, headers, body)?; let response = request.send().await.map_err(to_anyhow)?; @@ -469,8 +503,45 @@ async fn proxy( return Err(Error::AIError(err_msg)); } - let status_code = response.status(); - let headers = response.headers().clone(); - let stream = response.bytes_stream(); - Ok((status_code, headers, axum::body::Body::from_stream(stream))) + // Transform Bedrock responses back to OpenAI format + if matches!(provider, AIProvider::AWSBedrock) && model.is_some() { + if is_streaming { + // Transform streaming response + use http::StatusCode; + + let mut response_headers = HeaderMap::new(); + response_headers.insert("content-type", "text/event-stream".parse().unwrap()); + response_headers.insert("cache-control", "no-cache".parse().unwrap()); + response_headers.insert("connection", "keep-alive".parse().unwrap()); + + let stream = response.bytes_stream(); + let transformed_stream = + bedrock::transform_bedrock_stream_to_openai(stream, model.unwrap()); + + Ok(( + StatusCode::OK, + response_headers, + axum::body::Body::from_stream(transformed_stream), + )) + } else { + // Transform non-streaming response + let transformed_body = + bedrock::transform_bedrock_to_openai(response, model.unwrap()).await?; + + let mut response_headers = HeaderMap::new(); + response_headers.insert("content-type", "application/json".parse().unwrap()); + + Ok(( + http::StatusCode::OK, + response_headers, + axum::body::Body::from(transformed_body), + )) + } + } else { + // Pass through for other providers + let status_code = response.status(); + let headers = response.headers().clone(); + let stream = response.bytes_stream(); + Ok((status_code, headers, axum::body::Body::from_stream(stream))) + } } diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 0765f701f9..7f2ccc6876 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -90,6 +90,7 @@ impl AuthCache { w_id.as_ref(), token.trim_start_matches("jwt_ext_"), self.ext_jwks.clone(), + &self.db, ) .await { diff --git a/backend/windmill-api/src/bedrock.rs b/backend/windmill-api/src/bedrock.rs new file mode 100644 index 0000000000..ea93b5b3ea --- /dev/null +++ b/backend/windmill-api/src/bedrock.rs @@ -0,0 +1,602 @@ +use axum::body::Bytes; +use bytes; +use futures; +use uuid; +use windmill_common::error::{Error, Result}; + +/// Transform OpenAI format request to AWS Bedrock Converse format +/// Returns: (model_id, transformed_body, is_streaming) +pub fn transform_openai_to_bedrock(body: &[u8]) -> Result<(String, Bytes, bool)> { + use serde_json::Value; + + // Parse the OpenAI request + let openai_req: Value = serde_json::from_slice(body) + .map_err(|e| Error::internal_err(format!("Failed to parse OpenAI request: {}", e)))?; + + // Extract model and streaming flag + let model = openai_req["model"] + .as_str() + .ok_or_else(|| Error::BadRequest("Missing 'model' field in request".to_string()))? + .to_string(); + + let is_streaming = openai_req["stream"].as_bool().unwrap_or(false); + + // Build Bedrock request + let mut bedrock_req = serde_json::json!({}); + + // Transform messages + if let Some(messages) = openai_req["messages"].as_array() { + let mut system_messages = Vec::new(); + let mut conversation_messages = Vec::new(); + + for msg in messages { + let role = msg["role"].as_str().unwrap_or(""); + + match role { + "system" => { + // Extract system messages to separate array + if let Some(content) = msg["content"].as_str() { + system_messages.push(serde_json::json!({"text": content})); + } + } + "user" | "assistant" => { + // Normalize content to array format + let mut content = if let Some(text) = msg["content"].as_str() { + // Simple string → array of content blocks + vec![serde_json::json!({"text": text})] + } else if let Some(content_array) = msg["content"].as_array() { + // Already an array - transform each item + content_array + .iter() + .filter_map(|item| { + if let Some(text) = item["text"].as_str() { + Some(serde_json::json!({"text": text})) + } else if item["type"].as_str() == Some("text") { + Some(serde_json::json!({"text": item["text"]})) + } else if item["type"].as_str() == Some("image_url") { + // Transform image_url format if needed + // For now, pass through - may need more sophisticated handling + Some(item.clone()) + } else { + None + } + }) + .collect() + } else { + vec![] + }; + + // Handle tool_calls for assistant messages (OpenAI → Bedrock toolUse) + if role == "assistant" { + if let Some(tool_calls) = msg["tool_calls"].as_array() { + for tool_call in tool_calls { + if tool_call["type"].as_str() == Some("function") { + let tool_use_id = tool_call["id"].as_str().unwrap_or(""); + let function_name = + tool_call["function"]["name"].as_str().unwrap_or(""); + let arguments_str = + tool_call["function"]["arguments"].as_str().unwrap_or("{}"); + + // Parse arguments JSON string to object + let input = serde_json::from_str::(arguments_str) + .map_err(|e| { + Error::internal_err(format!( + "Failed to parse tool call arguments: {}", + e + )) + })?; + + content.push(serde_json::json!({ + "toolUse": { + "toolUseId": tool_use_id, + "name": function_name, + "input": input + } + })); + } + } + } + } + + // Only add message if it has content + if !content.is_empty() { + conversation_messages.push(serde_json::json!({ + "role": role, + "content": content + })); + } + } + "tool" => { + // Transform tool response to Bedrock format + let tool_call_id = msg["tool_call_id"].as_str().unwrap_or(""); + let content = msg["content"].as_str().unwrap_or(""); + + // Try to parse content as JSON + // Bedrock requires json field to be an object, not a primitive or array + let tool_result_content = + if let Ok(json_content) = serde_json::from_str::(content) { + if json_content.is_object() { + vec![serde_json::json!({"json": json_content})] + } else { + // Wrap primitives and arrays in an object + vec![serde_json::json!({"json": {"result": json_content}})] + } + } else { + vec![serde_json::json!({"text": content})] + }; + + conversation_messages.push(serde_json::json!({ + "role": "user", + "content": [{ + "toolResult": { + "toolUseId": tool_call_id, + "content": tool_result_content + } + }] + })); + } + _ => {} + } + } + + if !system_messages.is_empty() { + bedrock_req["system"] = Value::Array(system_messages); + } + bedrock_req["messages"] = Value::Array(conversation_messages); + } + + // Transform inference parameters + let mut inference_config = serde_json::json!({}); + if let Some(max_tokens) = openai_req["max_tokens"].as_i64() { + inference_config["maxTokens"] = Value::Number(max_tokens.into()); + } + if let Some(temperature) = openai_req["temperature"].as_f64() { + inference_config["temperature"] = serde_json::json!(temperature); + } + if let Some(top_p) = openai_req["top_p"].as_f64() { + inference_config["topP"] = serde_json::json!(top_p); + } + if let Some(stop) = openai_req["stop"].as_array() { + let stop_sequences: Vec = stop + .iter() + .filter_map(|s| s.as_str().map(|s| s.to_string())) + .collect(); + if !stop_sequences.is_empty() { + inference_config["stopSequences"] = + Value::Array(stop_sequences.into_iter().map(Value::String).collect()); + } + } + if !inference_config.as_object().unwrap().is_empty() { + bedrock_req["inferenceConfig"] = inference_config; + } + + // Transform tools if present + if let Some(tools) = openai_req["tools"].as_array() { + let mut bedrock_tools = Vec::new(); + + for tool in tools { + if tool["type"].as_str() == Some("function") { + if let Some(function) = tool["function"].as_object() { + bedrock_tools.push(serde_json::json!({ + "toolSpec": { + "name": function.get("name"), + "description": function.get("description") + .and_then(|v| v.as_str()) + .filter(|s| !s.is_empty()) + .unwrap_or("Tool function"), + "inputSchema": { + "json": function.get("parameters") + } + } + })); + } + } + } + + if !bedrock_tools.is_empty() { + let mut tool_config = serde_json::json!({ + "tools": bedrock_tools + }); + + // Transform tool_choice + if let Some(tool_choice) = openai_req.get("tool_choice") { + if tool_choice == "auto" { + tool_config["toolChoice"] = serde_json::json!({"auto": {}}); + } else if tool_choice == "required" { + tool_config["toolChoice"] = serde_json::json!({"any": {}}); + } else if let Some(obj) = tool_choice.as_object() { + if obj.get("type").and_then(|v| v.as_str()) == Some("function") { + if let Some(function) = obj.get("function").and_then(|v| v.as_object()) { + if let Some(name) = function.get("name").and_then(|v| v.as_str()) { + tool_config["toolChoice"] = serde_json::json!({ + "tool": {"name": name} + }); + } + } + } + } + } + + bedrock_req["toolConfig"] = tool_config; + } + } + + let transformed_body = serde_json::to_vec(&bedrock_req) + .map_err(|e| Error::internal_err(format!("Failed to serialize Bedrock request: {}", e)))? + .into(); + + Ok((model, transformed_body, is_streaming)) +} + +/// Transform AWS Bedrock Converse response to OpenAI format +pub async fn transform_bedrock_to_openai( + response: reqwest::Response, + model: String, +) -> Result { + use serde_json::Value; + + let bedrock_resp: Value = response + .json() + .await + .map_err(|e| Error::internal_err(format!("Failed to parse Bedrock response: {}", e)))?; + + // Generate unique ID and timestamp + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let created = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + // Extract stop reason and map to finish_reason + let stop_reason = bedrock_resp["stopReason"].as_str().unwrap_or("end_turn"); + let finish_reason = match stop_reason { + "end_turn" => "stop", + "max_tokens" => "length", + "tool_use" => "tool_calls", + "stop_sequence" => "stop", + "guardrail_intervened" | "content_filtered" => "content_filter", + _ => "stop", + }; + + // Extract message content + let message_content = &bedrock_resp["output"]["message"]["content"]; + let mut text_content = String::new(); + let mut tool_calls = Vec::new(); + + if let Some(content_array) = message_content.as_array() { + for (_index, block) in content_array.iter().enumerate() { + if let Some(text) = block["text"].as_str() { + text_content.push_str(text); + } else if let Some(tool_use) = block.get("toolUse") { + // Transform tool use to OpenAI tool_calls format + let tool_call_id = tool_use["toolUseId"].as_str().unwrap_or(""); + let name = tool_use["name"].as_str().unwrap_or(""); + let input = &tool_use["input"]; + + tool_calls.push(serde_json::json!({ + "id": tool_call_id, + "type": "function", + "function": { + "name": name, + "arguments": serde_json::to_string(input).unwrap_or_default() + } + })); + } + } + } + + // Build the message + let message = if !tool_calls.is_empty() { + serde_json::json!({ + "role": "assistant", + "content": if text_content.is_empty() { Value::Null } else { Value::String(text_content) }, + "tool_calls": tool_calls + }) + } else { + serde_json::json!({ + "role": "assistant", + "content": text_content + }) + }; + + // Extract usage information + let usage = if let Some(usage_data) = bedrock_resp.get("usage") { + serde_json::json!({ + "prompt_tokens": usage_data["inputTokens"].as_i64().unwrap_or(0), + "completion_tokens": usage_data["outputTokens"].as_i64().unwrap_or(0), + "total_tokens": usage_data["totalTokens"].as_i64().unwrap_or(0) + }) + } else { + serde_json::json!({ + "prompt_tokens": 0, + "completion_tokens": 0, + "total_tokens": 0 + }) + }; + + // Build OpenAI-format response + let openai_resp = serde_json::json!({ + "id": id, + "object": "chat.completion", + "created": created, + "model": model, + "choices": [{ + "index": 0, + "message": message, + "finish_reason": finish_reason + }], + "usage": usage + }); + + let response_body = serde_json::to_vec(&openai_resp) + .map_err(|e| Error::internal_err(format!("Failed to serialize OpenAI response: {}", e)))? + .into(); + + Ok(response_body) +} + +/// Transform AWS Bedrock streaming response to OpenAI SSE format +/// Bedrock uses AWS event stream binary format, not SSE +pub fn transform_bedrock_stream_to_openai( + stream: impl futures::Stream> + + Send + + 'static, + model: String, +) -> impl futures::Stream> + Send { + use futures::stream::StreamExt; + use serde_json::Value; + use std::collections::HashMap; + + let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple()); + let created = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + + // State to track partial tool calls and binary buffer + struct StreamState { + id: String, + model: String, + created: u64, + tool_calls: HashMap, // index -> (id, name, args) + buffer: Vec, // Binary buffer for AWS event stream + } + + let state = std::sync::Arc::new(tokio::sync::Mutex::new(StreamState { + id: id.clone(), + model: model.clone(), + created, + tool_calls: HashMap::new(), + buffer: Vec::new(), + })); + + stream + .then(move |chunk_result| { + let state = state.clone(); + async move { + match chunk_result { + Ok(chunk) => { + let mut state = state.lock().await; + state.buffer.extend_from_slice(&chunk); + + let mut events = Vec::new(); + + // Parse AWS event stream messages from buffer + loop { + // Need at least 12 bytes for prelude (8) + prelude CRC (4) + if state.buffer.len() < 12 { + break; + } + + // Read prelude: total_length (4 bytes) + headers_length (4 bytes) + let total_length = u32::from_be_bytes([ + state.buffer[0], + state.buffer[1], + state.buffer[2], + state.buffer[3], + ]) as usize; + + // Check if we have the complete message + if state.buffer.len() < total_length { + break; + } + + let headers_length = u32::from_be_bytes([ + state.buffer[4], + state.buffer[5], + state.buffer[6], + state.buffer[7], + ]) as usize; + + // Skip prelude CRC (4 bytes after prelude) + let headers_start = 12; + let payload_start = headers_start + headers_length; + let payload_end = total_length - 4; // Exclude message CRC + + // Parse headers to extract event type + let mut event_type = None; + let mut pos = headers_start; + while pos < payload_start { + if pos + 1 > state.buffer.len() { + break; + } + let name_len = state.buffer[pos] as usize; + pos += 1; + + if pos + name_len > state.buffer.len() { + break; + } + let name = String::from_utf8_lossy(&state.buffer[pos..pos + name_len]).to_string(); + pos += name_len; + + if pos + 3 > state.buffer.len() { + break; + } + let value_type = state.buffer[pos]; + pos += 1; + let value_len = u16::from_be_bytes([state.buffer[pos], state.buffer[pos + 1]]) as usize; + pos += 2; + + if pos + value_len > state.buffer.len() { + break; + } + + if value_type == 7 && name == ":event-type" { + event_type = Some(String::from_utf8_lossy(&state.buffer[pos..pos + value_len]).to_string()); + } + pos += value_len; + } + + // Extract JSON payload (copy to avoid borrow issues) + let payload = state.buffer[payload_start..payload_end].to_vec(); + + // Remove processed message from buffer + state.buffer.drain(0..total_length); + + // Process the event + if let Some(evt_type) = event_type { + if let Ok(payload_str) = std::str::from_utf8(&payload) { + if let Ok(parsed_data) = serde_json::from_str::(payload_str) { + // Transform based on event type + match evt_type.as_str() { + "messageStart" => { + // No output for messageStart + } + "contentBlockStart" => { + let index = parsed_data["contentBlockIndex"].as_u64().unwrap_or(0) as usize; + + if let Some(tool_use) = parsed_data["start"].get("toolUse") { + let tool_id = tool_use["toolUseId"].as_str().unwrap_or("").to_string(); + let name = tool_use["name"].as_str().unwrap_or("").to_string(); + + state.tool_calls.insert(index, (tool_id.clone(), name.clone(), String::new())); + + // Send initial tool call chunk + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": index, + "id": tool_id, + "type": "function", + "function": { + "name": name, + "arguments": "" + } + }] + }, + "finish_reason": Value::Null + }] + }); + + events.push(Ok(bytes::Bytes::from(format!("data: {}\n\n", chunk)))); + } + } + "contentBlockDelta" => { + let index = parsed_data["contentBlockIndex"].as_u64().unwrap_or(0) as usize; + + if let Some(text) = parsed_data["delta"]["text"].as_str() { + // Text content delta + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [{ + "index": 0, + "delta": { + "content": text + }, + "finish_reason": Value::Null + }] + }); + + events.push(Ok(bytes::Bytes::from(format!("data: {}\n\n", chunk)))); + } else if let Some(tool_use_input) = parsed_data["delta"]["toolUse"]["input"].as_str() { + // Tool use arguments delta + if let Some((_tool_id, _name, ref mut args)) = state.tool_calls.get_mut(&index) { + args.push_str(tool_use_input); + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [{ + "index": 0, + "delta": { + "tool_calls": [{ + "index": index, + "function": { + "arguments": tool_use_input + } + }] + }, + "finish_reason": Value::Null + }] + }); + + events.push(Ok(bytes::Bytes::from(format!("data: {}\n\n", chunk)))); + } + } + } + "contentBlockStop" => { + // No output needed + } + "messageStop" => { + let stop_reason = parsed_data["stopReason"].as_str().unwrap_or("end_turn"); + let finish_reason = match stop_reason { + "end_turn" => "stop", + "max_tokens" => "length", + "tool_use" => "tool_calls", + "stop_sequence" => "stop", + "guardrail_intervened" | "content_filtered" => "content_filter", + _ => "stop", + }; + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": finish_reason + }] + }); + + events.push(Ok(bytes::Bytes::from(format!("data: {}\n\n", chunk)))); + } + "metadata" => { + // Could include usage info here if needed + } + _ => {} + } + } + } + } + } // end loop + + events + } + Err(e) => { + vec![Err(std::io::Error::new( + std::io::ErrorKind::Other, + e.to_string(), + ))] + } + } + } + }) + .flat_map(|events| futures::stream::iter(events)) + .chain(futures::stream::iter(vec![ + // Send [DONE] at the end + Ok(bytes::Bytes::from("data: [DONE]\n\n")) + ])) +} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index c5d16ebffe..09da5d1b11 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -53,6 +53,9 @@ lazy_static::lazy_static! { ).to_string()), (20221105003256, "DELETE FROM workspace_invite WHERE workspace_id = 'demo' AND email = 'ruben@windmill.dev';".to_string()), (20221123151919, "".to_string()), + (20251105100125, include_str!( + "../../migrations/20251105100125_legacy_sql_result_flag.up.sql" + ).replace("✅", "")), ].into_iter().collect(); } @@ -166,6 +169,7 @@ impl Migrate for CustomMigrator { if let Some(migration_sql) = OVERRIDDEN_MIGRATIONS.get(&migration.version) { tracing::info!("Using custom migration for version {}", migration.version); + // tracing::info!("Migration SQL: {}", migration_sql); self.inner .execute(&**migration_sql) diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index 30756af8ab..24eea5ab85 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -26,6 +26,7 @@ pub async fn jwt_ext_auth( _w_id: Option<&String>, _token: &str, _external_jwks: Option>>, + _db: &crate::db::DB, ) -> anyhow::Result<(crate::db::ApiAuthed, usize)> { // Implementation is not open source diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index dd141d46f6..a2e8e307e6 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -783,8 +783,8 @@ async fn update_flow( sqlx::query!( " - UPDATE - flow + UPDATE + flow SET path = $1, summary = $2, @@ -800,7 +800,7 @@ async fn update_flow( schema = $9::text::json, edited_by = $10, edited_at = now() - WHERE + WHERE path = $11 AND workspace_id = $12", if is_new_path { flow_path } else { &nf.path }, nf.summary, @@ -824,8 +824,8 @@ async fn update_flow( if is_new_path { // if new path, must clone flow to new path and delete old flow for flow_version foreign key constraint sqlx::query!( - "INSERT INTO flow - (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) + "INSERT INTO flow + (workspace_id, path, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at) SELECT workspace_id, $1, summary, description, archived, extra_perms, dependency_job, draft_only, tag, ws_error_handler_muted, dedicated_worker, timeout, visible_to_runner_only, on_behalf_of_email, concurrency_key, versions, value, schema, edited_by, edited_at FROM flow WHERE path = $2 AND workspace_id = $3", @@ -893,6 +893,8 @@ async fn update_flow( .warn_after_seconds(10) .await??; + // tracing::error!("Updating flow: {:?}", nf.value.get()); + // This will lock anyone who is trying to iterate on flow_versions with given path and parameters. let version = sqlx::query_scalar!( "INSERT INTO flow_version (workspace_id, path, value, schema, created_by) VALUES ($1, $2, $3, $4::text::json, $5) RETURNING id", @@ -1143,11 +1145,11 @@ async fn get_flow_by_path( favorite.path IS NOT NULL AS starred FROM flow LEFT JOIN favorite - ON favorite.favorite_kind = 'flow' - AND favorite.workspace_id = flow.workspace_id - AND favorite.path = flow.path + ON favorite.favorite_kind = 'flow' + AND favorite.workspace_id = flow.workspace_id + AND favorite.path = flow.path AND favorite.usr = $3 - LEFT JOIN flow_version + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = $1 AND flow.workspace_id = $2 "#, @@ -1182,7 +1184,7 @@ async fn get_flow_by_path( flow_version.created_by AS edited_by, NULL AS starred FROM flow - LEFT JOIN flow_version + LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = $1 AND flow.workspace_id = $2 "#, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index cd2f963680..96bf0ed085 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -80,6 +80,7 @@ pub mod args; mod assets; mod audit; pub mod auth; +mod bedrock; mod capture; mod concurrency_groups; mod configs; @@ -154,6 +155,7 @@ mod smtp_server_oss; pub mod teams_approvals_ee; mod teams_approvals_oss; +mod public_app_layer; mod static_assets; #[cfg(all(feature = "stripe", feature = "enterprise", feature = "private"))] pub mod stripe_ee; @@ -183,7 +185,6 @@ pub mod workspaces_ee; mod workspaces_export; mod workspaces_extra; mod workspaces_oss; -mod public_app_layer; #[cfg(feature = "mcp")] mod mcp; @@ -324,6 +325,11 @@ pub async fn run_server( #[cfg(feature = "embedding")] load_embeddings_db(&db); + #[cfg(feature = "cloud")] + if *CLOUD_HOSTED { + windmill_queue::init_usage_buffer(db.clone()); + } + let mut start_smtp_server = false; if let Some(smtp_settings) = load_value_from_global_settings(&db, EMAIL_DOMAIN_SETTING).await? diff --git a/backend/windmill-api/src/mcp/server.rs b/backend/windmill-api/src/mcp/server.rs index 8a09459202..0e1a585383 100644 --- a/backend/windmill-api/src/mcp/server.rs +++ b/backend/windmill-api/src/mcp/server.rs @@ -39,6 +39,7 @@ use super::utils::{ FlowInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo, ToolableItem, WorkspaceId, }, schema::transform_schema_for_resources, + scope_matcher::{is_resource_allowed, parse_mcp_scopes}, transform::{reverse_transform, reverse_transform_key}, }; @@ -151,6 +152,10 @@ impl ServerHandler for Runner { check_scopes(authed)?; + // Parse MCP scopes for authorization + let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); + let scope_config = parse_mcp_scopes(scopes)?; + if request.name.ends_with("_TRUNC") { return Ok(CallToolResult::error( vec![ @@ -197,6 +202,19 @@ impl ServerHandler for Runner { let endpoint_tools = all_endpoint_tools(); for endpoint_tool in endpoint_tools { if endpoint_tool.name.as_ref() == request.name { + // Validate endpoint scope + if scope_config.granular + && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) + { + return Err(ErrorData::internal_error( + format!( + "Access denied: endpoint '{}' not in token scope", + endpoint_tool.name + ), + None, + )); + } + // This is an endpoint tool, forward to the actual HTTP endpoint let result = call_endpoint_tool(&endpoint_tool, args.clone(), &workspace_id, &authed) @@ -212,6 +230,21 @@ impl ServerHandler for Runner { ErrorData::internal_error(format!("Failed to reverse transform path: {}", e), None) })?; + // Validate script/flow scope + if !is_hub && scope_config.granular { + if tool_type == "script" && !is_resource_allowed(&path, &scope_config.scripts) { + return Err(ErrorData::internal_error( + format!("Access denied: script '{}' not in token scope", path), + None, + )); + } else if tool_type == "flow" && !is_resource_allowed(&path, &scope_config.flows) { + return Err(ErrorData::internal_error( + format!("Access denied: flow '{}' not in token scope", path), + None, + )); + } + } + let item_schema = if is_hub { get_hub_script_schema(&format!("hub/{}", path), db).await? } else { @@ -337,53 +370,23 @@ impl ServerHandler for Runner { }) .map(|w_id| w_id.0.clone())?; - let scopes = authed.scopes.as_ref(); - let owned_scope = scopes.and_then(|scopes| { - scopes - .iter() - .find(|scope| scope.starts_with("mcp:") && !scope.contains("hub")) - }); - let hub_scope = - scopes.and_then(|scopes| scopes.iter().find(|scope| scope.starts_with("mcp:hub"))); - let (scope_type, scope_path) = owned_scope.map_or(("all", None), |scope| { - let parts = scope.split(":").collect::>(); - ( - parts[1], - if parts.len() == 3 { - Some(parts[2]) - } else { - None - }, - ) - }); - let scope_integrations = hub_scope.and_then(|scope| { - let parts = scope.split(":").collect::>(); - if parts.len() == 3 { - Some(parts[2]) - } else { - None - } - }); + // Parse MCP scopes to determine what to expose + let scopes = authed.scopes.as_ref().map(|s| s.as_slice()).unwrap_or(&[]); + let scope_config = parse_mcp_scopes(scopes)?; - let scripts_fn = get_items::( - user_db, - authed, - &workspace_id, - scope_type, - "script", - scope_path.as_deref(), - ); - let flows_fn = get_items::( - user_db, - authed, - &workspace_id, - scope_type, - "flow", - scope_path.as_deref(), - ); + let scope_type = if scope_config.favorites { + "favorites" + } else { + // Fetch all items if either all or granular scope set (we filter later for granular scopes) + "all" + }; + + let scripts_fn = + get_items::(user_db, authed, &workspace_id, scope_type, "script"); + let flows_fn = get_items::(user_db, authed, &workspace_id, scope_type, "flow"); let resources_types_fn = get_resources_types(user_db, authed, &workspace_id); - let hub_scripts_fn = get_scripts_from_hub(db, scope_integrations.as_deref()); - let (scripts, flows, resources_types, hub_scripts) = if scope_integrations.is_some() { + let hub_scripts_fn = get_scripts_from_hub(db, scope_config.hub_apps.as_deref()); + let (scripts, flows, resources_types, hub_scripts) = if scope_config.hub_apps.is_some() { let (scripts, flows, resources_types, hub_scripts) = try_join!(scripts_fn, flows_fn, resources_types_fn, hub_scripts_fn)?; (scripts, flows, resources_types, hub_scripts) @@ -396,7 +399,13 @@ impl ServerHandler for Runner { let mut resources_cache: HashMap> = HashMap::new(); let mut tools: Vec = Vec::new(); + // Filter and add scripts based on scope for script in scripts { + // For granular scopes, filter by path + if scope_config.granular && !is_resource_allowed(&script.path, &scope_config.scripts) { + continue; + } + tools.push( Runner::create_tool_from_item( &script, @@ -410,7 +419,13 @@ impl ServerHandler for Runner { ); } + // Filter and add flows based on scope for flow in flows { + // For granular scopes, filter by path + if scope_config.granular && !is_resource_allowed(&flow.path, &scope_config.flows) { + continue; + } + tools.push( Runner::create_tool_from_item( &flow, @@ -438,10 +453,23 @@ impl ServerHandler for Runner { ); } - // Add endpoint tools from the generated MCP tools + // Add endpoint tools from the generated MCP tools, filtered by scope let endpoint_tools = all_endpoint_tools(); - let mcp_tools_converted = endpoint_tools_to_mcp_tools(endpoint_tools); - tools.extend(mcp_tools_converted); + for endpoint_tool in endpoint_tools { + // For granular scopes, filter by endpoint name + if scope_config.granular + && !is_resource_allowed(&endpoint_tool.name, &scope_config.endpoints) + { + continue; + } + + tools.push( + endpoint_tools_to_mcp_tools(vec![endpoint_tool]) + .into_iter() + .next() + .unwrap(), + ); + } Ok(ListToolsResult { tools, next_cursor: None }) } diff --git a/backend/windmill-api/src/mcp/utils/database.rs b/backend/windmill-api/src/mcp/utils/database.rs index 39aac1ae6e..32da184638 100644 --- a/backend/windmill-api/src/mcp/utils/database.rs +++ b/backend/windmill-api/src/mcp/utils/database.rs @@ -18,11 +18,10 @@ use crate::HTTP_CLIENT; pub fn check_scopes(authed: &ApiAuthed) -> Result<(), ErrorData> { let scopes = authed.scopes.as_ref(); if scopes.is_none() - || scopes.unwrap().iter().all(|scope| { - !scope.starts_with("mcp:all") - && !scope.starts_with("mcp:favorites") - && !scope.starts_with("mcp:hub:") - }) + || scopes + .unwrap() + .iter() + .all(|scope| !scope.starts_with("mcp:")) { tracing::error!("Unauthorized: missing mcp scope"); return Err(ErrorData::internal_error( @@ -141,7 +140,6 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen workspace_id: &str, scope_type: &str, item_type: &str, - scope_path: Option<&str>, ) -> Result, ErrorData> { let mut sqlb = SqlBuilder::select_from(&format!("{} as o", item_type)); let fields = vec!["o.path", "o.summary", "o.description", "o.schema"]; @@ -159,23 +157,6 @@ pub async fn get_items sqlx::FromRow<'a, sqlx::postgres::PgRow> + Sen sqlb.and_where("(o.no_main_func IS NOT TRUE OR o.no_main_func IS NULL)"); } - // scope path is always a folder path, format is f/my_folder/* - if let Some(scope_path) = scope_path { - if scope_path.split("/").count() != 3 - || !scope_path.starts_with("f/") - || !scope_path.ends_with("/*") - { - return Err(ErrorData::internal_error( - format!( - "Invalid folder format: {}, expected format is f/my_folder/*", - scope_path - ), - None, - )); - } - sqlb.and_where_like_left("o.path", &scope_path[..scope_path.len() - 2]); - } - sqlb.order_by( if item_type == "flow" { "o.edited_at" diff --git a/backend/windmill-api/src/mcp/utils/mod.rs b/backend/windmill-api/src/mcp/utils/mod.rs index 04464be481..e4072313a0 100644 --- a/backend/windmill-api/src/mcp/utils/mod.rs +++ b/backend/windmill-api/src/mcp/utils/mod.rs @@ -6,4 +6,5 @@ pub mod models; pub mod database; pub mod schema; -pub mod transform; \ No newline at end of file +pub mod transform; +pub mod scope_matcher; \ No newline at end of file diff --git a/backend/windmill-api/src/mcp/utils/scope_matcher.rs b/backend/windmill-api/src/mcp/utils/scope_matcher.rs new file mode 100644 index 0000000000..05fdafb706 --- /dev/null +++ b/backend/windmill-api/src/mcp/utils/scope_matcher.rs @@ -0,0 +1,229 @@ +//! MCP Scope matching utilities +//! +//! Contains utilities for parsing and matching MCP token scopes to determine +//! which scripts, flows, and endpoints a token has access to. + +use rmcp::ErrorData; + +/// Configuration for MCP scopes parsed from token scopes +#[derive(Debug, Clone, Default)] +pub struct McpScopeConfig { + /// Script paths/patterns allowed by this token + pub scripts: Vec, + /// Flow paths/patterns allowed by this token + pub flows: Vec, + /// Endpoint names/patterns allowed by this token + pub endpoints: Vec, + /// Whether this is a legacy "all" scope + pub all: bool, + /// Whether this is a "favorites" scope + pub favorites: bool, + /// Whether a granular scope is detected + pub granular: bool, + /// Hub app filter (if any) + pub hub_apps: Option, +} + +/// Parse MCP scopes from token scope strings +pub fn parse_mcp_scopes(scopes: &[String]) -> Result { + let mut config = McpScopeConfig::default(); + + for scope in scopes { + if !scope.starts_with("mcp:") { + continue; + } + + if scope == "mcp:all" { + // Legacy scope: grant access to everything + config.all = true; + config.scripts.push("*".to_string()); + config.flows.push("*".to_string()); + config.endpoints.push("*".to_string()); + continue; + } + + if scope == "mcp:favorites" { + // Legacy favorites scope + config.favorites = true; + continue; + } + + // Legacy folder scope: mcp:all:f/folder/* + if scope.starts_with("mcp:all:") { + if let Some(folder_pattern) = scope.strip_prefix("mcp:all:") { + // Parse as folder pattern - add to both scripts and flows. Also add all endpoints. + config.scripts.push(folder_pattern.to_string()); + config.flows.push(folder_pattern.to_string()); + config.endpoints.push("*".to_string()); + } + continue; + } + + if scope.starts_with("mcp:hub:") { + // Legacy hub scope + if let Some(apps) = scope.strip_prefix("mcp:hub:") { + config.hub_apps = Some(apps.to_string()); + } + continue; + } + + if let Some(resources) = scope.strip_prefix("mcp:scripts:") { + // New granular script scope: mcp:scripts:path1,path2,f/folder/* + config.scripts.extend(parse_resource_list(resources)?); + continue; + } + + if let Some(resources) = scope.strip_prefix("mcp:flows:") { + // New granular flow scope: mcp:flows:path1,path2,f/folder/* + config.flows.extend(parse_resource_list(resources)?); + continue; + } + + if let Some(resources) = scope.strip_prefix("mcp:endpoints:") { + // New granular endpoint scope: mcp:endpoints:name1,name2 + config.endpoints.extend(parse_resource_list(resources)?); + continue; + } + + tracing::warn!("Unrecognized MCP scope format: {}", scope); + } + + config.granular = !config.all && !config.favorites; + + Ok(config) +} + +/// Parse comma-separated resource list +fn parse_resource_list(resources: &str) -> Result, ErrorData> { + if resources.is_empty() { + return Ok(vec![]); + } + + Ok(resources + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect()) +} + +/// Check if a resource path matches any pattern in the allowed list +pub fn is_resource_allowed(resource_path: &str, allowed_patterns: &[String]) -> bool { + if allowed_patterns.is_empty() { + return false; + } + + // Wildcard grants all access + if allowed_patterns.contains(&"*".to_string()) { + return true; + } + + // Check against each pattern + for pattern in allowed_patterns { + if resource_matches_pattern(resource_path, pattern) { + return true; + } + } + + false +} + +/// Check if a resource path matches a pattern (supports wildcards like f/folder/*) +fn resource_matches_pattern(resource_path: &str, pattern: &str) -> bool { + // Exact match + if pattern == resource_path { + return true; + } + + // Wildcard pattern matching + if pattern.ends_with("/*") { + let prefix = &pattern[..pattern.len() - 2]; + + if !resource_path.starts_with(prefix) { + return false; + } + + // If the resource is exactly the prefix, it matches + if resource_path.len() == prefix.len() { + return true; + } + + // If the resource is longer, the next character must be '/' for a valid match + // This prevents "u/user" from matching "u/use/*" + return resource_path.chars().nth(prefix.len()) == Some('/'); + } + + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_legacy_scopes() { + let scopes = vec!["mcp:all".to_string()]; + let config = parse_mcp_scopes(&scopes).unwrap(); + assert!(config.all); + assert_eq!(config.scripts, vec!["*"]); + assert_eq!(config.flows, vec!["*"]); + assert_eq!(config.endpoints, vec!["*"]); + + let scopes = vec!["mcp:favorites".to_string()]; + let config = parse_mcp_scopes(&scopes).unwrap(); + assert!(config.favorites); + + let scopes = vec!["mcp:hub:slack".to_string()]; + let config = parse_mcp_scopes(&scopes).unwrap(); + assert_eq!(config.hub_apps, Some("slack".to_string())); + } + + #[test] + fn test_parse_granular_scopes() { + let scopes = vec![ + "mcp:scripts:u/admin/script1,u/admin/script2".to_string(), + "mcp:flows:f/automation/*".to_string(), + "mcp:endpoints:list_jobs,get_job".to_string(), + ]; + let config = parse_mcp_scopes(&scopes).unwrap(); + + assert_eq!(config.scripts, vec!["u/admin/script1", "u/admin/script2"]); + assert_eq!(config.flows, vec!["f/automation/*"]); + assert_eq!(config.endpoints, vec!["list_jobs", "get_job"]); + } + + #[test] + fn test_resource_matching() { + // Exact match + assert!(resource_matches_pattern("u/admin/script", "u/admin/script")); + + // Wildcard folder match + assert!(resource_matches_pattern("f/folder/script", "f/folder/*")); + assert!(resource_matches_pattern( + "f/folder/sub/script", + "f/folder/*" + )); + + // Should NOT match - prefix is not complete + assert!(!resource_matches_pattern("u/username", "u/user/*")); + + // Should match - exact prefix + assert!(resource_matches_pattern("u/user/script", "u/user/*")); + } + + #[test] + fn test_is_resource_allowed() { + let patterns = vec!["u/admin/script1".to_string(), "f/folder/*".to_string()]; + + assert!(is_resource_allowed("u/admin/script1", &patterns)); + assert!(is_resource_allowed("f/folder/anything", &patterns)); + assert!(!is_resource_allowed("u/other/script", &patterns)); + + // Test wildcard + let wildcard = vec!["*".to_string()]; + assert!(is_resource_allowed("any/path", &wildcard)); + + // Test empty patterns + let empty: Vec = vec![]; + assert!(!is_resource_allowed("any/path", &empty)); + } +} diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index ec1eafe08a..503c60e1f8 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -472,7 +472,7 @@ pub async fn get_resource_value_interpolated_internal( // This is a special syntax to help debugging ducklake catalogs stored in the instance if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") { require_super_admin(db, &authed.email).await?; - let pg_creds = parse_postgres_url(&get_database_url().await?)?; + let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; return Ok(Some(serde_json::json!({ "dbname": dbname, "host": pg_creds.host, @@ -738,6 +738,31 @@ async fn create_resource( let res_value = resource.value.unwrap_or_default(); let raw_json = sqlx::types::Json(res_value.as_ref()); + if resource.path.starts_with("f/app_themes/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_custom/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } else if resource.path.starts_with("f/app_groups/") { + sqlx::query!( + "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", + w_id, + authed.username, + ) + .execute(&db) + .await?; + } sqlx::query!( "INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 62cb87d391..bff2b282b6 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -647,8 +647,7 @@ async fn setup_ducklake_catalog_db_inner( ) -> Result<()> { require_super_admin(db, &authed.email).await?; logs.super_admin = "OK".to_string(); - let pg_creds = &get_database_url().await?; - let pg_creds = parse_postgres_url(pg_creds)?; + let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; logs.database_credentials = "OK".to_string(); // Validate name to ensure it only contains alphanumeric characters @@ -695,7 +694,7 @@ async fn setup_ducklake_catalog_db_inner( "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}", user = urlencoding::encode(&pg_creds.username.unwrap_or_else(|| "postgres".to_string())), password = urlencoding::encode(&pg_creds.password.as_deref().unwrap_or("")), - host = urlencoding::encode(&pg_creds.host), + host = &pg_creds.host, port = pg_creds.port.unwrap_or(5432), dbname = dbname, sslmode = ssl_mode diff --git a/backend/windmill-api/src/triggers/email/handler_oss.rs b/backend/windmill-api/src/triggers/email/handler_oss.rs index 2f3a318bdd..03a377df98 100644 --- a/backend/windmill-api/src/triggers/email/handler_oss.rs +++ b/backend/windmill-api/src/triggers/email/handler_oss.rs @@ -28,7 +28,6 @@ impl TriggerCrud for EmailTrigger { const TABLE_NAME: &'static str = ""; const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/email_triggers"; diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-api/src/triggers/gcp/handler_oss.rs index 4b52ff9c68..b75a9721eb 100644 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ b/backend/windmill-api/src/triggers/gcp/handler_oss.rs @@ -25,7 +25,6 @@ impl TriggerCrud for GcpTrigger { const TABLE_NAME: &'static str = ""; const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/gcp_triggers"; diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs index 0e22ea2f34..05042bd385 100644 --- a/backend/windmill-api/src/triggers/handler.rs +++ b/backend/windmill-api/src/triggers/handler.rs @@ -50,7 +50,6 @@ pub trait TriggerCrud: Send + Sync + 'static { const TABLE_NAME: &'static str; const TRIGGER_TYPE: &'static str; - const SUPPORTS_ENABLED: bool; const SUPPORTS_SERVER_STATE: bool; const SUPPORTS_TEST_CONNECTION: bool; const ROUTE_PREFIX: &'static str; @@ -144,10 +143,11 @@ pub trait TriggerCrud: Send + Sync + 'static { "email", "edited_at", "extra_perms", + "enabled", ]; if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); } fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); @@ -206,6 +206,10 @@ pub trait TriggerCrud: Send + Sync + 'static { Ok(deleted > 0) } + async fn set_enabled_extra_action(&self, _: &mut PgConnection) -> Result<()> { + Ok(()) + } + async fn set_enabled( &self, authed: &ApiAuthed, @@ -214,38 +218,59 @@ pub trait TriggerCrud: Send + Sync + 'static { path: &str, enabled: bool, ) -> Result { - if !Self::SUPPORTS_SERVER_STATE { - return Err(anyhow::anyhow!( - "Enable/disable not supported for this trigger type".to_string(), - ) - .into()); - } + let updated = if Self::SUPPORTS_SERVER_STATE { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + enabled = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(enabled) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + } else { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + enabled = $1, + email = $2, + edited_by = $3, + edited_at = now() + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(enabled) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + }; - let updated = sqlx::query(&format!( - r#" - UPDATE - {} - SET - enabled = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(enabled) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); + self.set_enabled_extra_action(&mut *tx).await?; Ok(updated > 0) } @@ -296,10 +321,11 @@ pub trait TriggerCrud: Send + Sync + 'static { "email", "edited_at", "extra_perms", + "enabled", ]; if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); } fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); @@ -333,6 +359,7 @@ pub trait TriggerCrud: Send + Sync + 'static { .sql() .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + tracing::info!("SQL: {}", sql); let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; Ok(triggers) @@ -346,11 +373,8 @@ pub fn trigger_routes() -> Router { .route("/get/*path", get(get_trigger::)) .route("/update/*path", post(update_trigger::)) .route("/delete/*path", delete(delete_trigger::)) - .route("/exists/*path", get(exists_trigger::)); - - if T::SUPPORTS_ENABLED { - router = router.route("/setenabled/*path", post(set_enabled_trigger::)); - } + .route("/exists/*path", get(exists_trigger::)) + .route("/setenabled/*path", post(set_enabled_trigger::)); if T::SUPPORTS_TEST_CONNECTION { router = router.route("/test", post(test_connection::)); diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index fd051e05be..cf1a004c32 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -196,6 +196,7 @@ pub async fn insert_new_trigger_into_db( summary, description, is_flow, + enabled, request_type, authentication_method, http_method, @@ -209,7 +210,7 @@ pub async fn insert_new_trigger_into_db( retry ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22 + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 ) "#, w_id, @@ -224,6 +225,7 @@ pub async fn insert_new_trigger_into_db( trigger.config.summary, trigger.config.description, trigger.base.is_flow, + trigger.base.enabled.unwrap_or(true), request_type as _, trigger.config.authentication_method as _, trigger.config.http_method as _, @@ -356,7 +358,6 @@ impl TriggerCrud for HttpTrigger { const TABLE_NAME: &'static str = "http_trigger"; const TRIGGER_TYPE: &'static str = "http"; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/http_triggers"; @@ -482,22 +483,23 @@ impl TriggerCrud for HttpTrigger { script_path = $7, path = $8, is_flow = $9, - http_method = $10, - static_asset_config = $11, - edited_by = $12, - email = $13, - request_type = $14, - authentication_method = $15, - summary = $16, - description = $17, + enabled = $10, + http_method = $11, + static_asset_config = $12, + edited_by = $13, + email = $14, + request_type = $15, + authentication_method = $16, + summary = $17, + description = $18, edited_at = now(), - is_static_website = $18, - error_handler_path = $19, - error_handler_args = $20, - retry = $21 + is_static_website = $19, + error_handler_path = $20, + error_handler_args = $21, + retry = $22 WHERE - workspace_id = $22 AND - path = $23 + workspace_id = $23 AND + path = $24 "#, route_path, &route_path_key, @@ -508,6 +510,7 @@ impl TriggerCrud for HttpTrigger { trigger.base.script_path, trigger.base.path, trigger.base.is_flow, + trigger.base.enabled.unwrap_or(true), trigger.config.http_method as _, trigger.config.static_asset_config as _, &authed.username, @@ -539,22 +542,23 @@ impl TriggerCrud for HttpTrigger { script_path = $4, path = $5, is_flow = $6, - http_method = $7, - static_asset_config = $8, - edited_by = $9, - email = $10, - request_type = $11, - authentication_method = $12, - summary = $13, - description = $14, + enabled = $7, + http_method = $8, + static_asset_config = $9, + edited_by = $10, + email = $11, + request_type = $12, + authentication_method = $13, + summary = $14, + description = $15, edited_at = now(), - is_static_website = $15, - error_handler_path = $16, - error_handler_args = $17, - retry = $18 + is_static_website = $16, + error_handler_path = $17, + error_handler_args = $18, + retry = $19 WHERE - workspace_id = $19 AND - path = $20 + workspace_id = $20 AND + path = $21 "#, trigger.config.wrap_body, trigger.config.raw_string, @@ -562,6 +566,7 @@ impl TriggerCrud for HttpTrigger { trigger.base.script_path, trigger.base.path, trigger.base.is_flow, + trigger.base.enabled.unwrap_or(true), trigger.config.http_method as _, trigger.config.static_asset_config as _, &authed.username, @@ -586,6 +591,10 @@ impl TriggerCrud for HttpTrigger { Ok(()) } + async fn set_enabled_extra_action(&self, tx: &mut PgConnection) -> Result<()> { + increase_trigger_version(tx).await + } + async fn delete_by_path( &self, tx: &mut PgConnection, diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs index 55844e0fb1..96944eccad 100644 --- a/backend/windmill-api/src/triggers/http/mod.rs +++ b/backend/windmill-api/src/triggers/http/mod.rs @@ -261,7 +261,8 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route FROM http_trigger WHERE - http_method = $1 + http_method = $1 AND + enabled is TRUE "#, &http_method as &HttpMethod ) diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-api/src/triggers/kafka/handler_oss.rs index bf5537b941..9fda83916f 100644 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ b/backend/windmill-api/src/triggers/kafka/handler_oss.rs @@ -28,7 +28,6 @@ impl TriggerCrud for KafkaTrigger { const TABLE_NAME: &'static str = ""; const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/kafka_triggers"; diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs index af9d0e32ce..23e0dfd936 100644 --- a/backend/windmill-api/src/triggers/listener.rs +++ b/backend/windmill-api/src/triggers/listener.rs @@ -69,12 +69,12 @@ pub trait Listener: TriggerCrud + TriggerJobArgs { "email", "edited_at", "extra_perms", + "enabled", + "error_handler_path", + "error_handler_args", + "retry", ]; - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); - } - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index 9a7f22e3e5..6c7004dac4 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -47,6 +47,7 @@ pub struct BaseTrigger { pub workspace_id: String, pub path: String, pub script_path: String, + pub enabled: Option, pub is_flow: bool, pub edited_by: String, pub email: String, @@ -56,7 +57,6 @@ pub struct BaseTrigger { #[derive(Debug, FromRow, Clone, Serialize, Deserialize)] pub struct ServerState { - pub enabled: bool, #[serde(skip_serializing_if = "Option::is_none")] pub server_id: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-api/src/triggers/mqtt/handler.rs b/backend/windmill-api/src/triggers/mqtt/handler.rs index d8e314a836..fd6054666c 100644 --- a/backend/windmill-api/src/triggers/mqtt/handler.rs +++ b/backend/windmill-api/src/triggers/mqtt/handler.rs @@ -26,7 +26,6 @@ impl TriggerCrud for MqttTrigger { const TABLE_NAME: &'static str = "mqtt_trigger"; const TRIGGER_TYPE: &'static str = "mqtt"; - const SUPPORTS_ENABLED: bool = true; const SUPPORTS_SERVER_STATE: bool = true; const SUPPORTS_TEST_CONNECTION: bool = true; const ROUTE_PREFIX: &'static str = "/mqtt_triggers"; diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-api/src/triggers/nats/handler_oss.rs index 41f060003f..3ab3430e5f 100644 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ b/backend/windmill-api/src/triggers/nats/handler_oss.rs @@ -25,7 +25,6 @@ impl TriggerCrud for NatsTrigger { const TABLE_NAME: &'static str = ""; const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/nats_triggers"; diff --git a/backend/windmill-api/src/triggers/postgres/handler.rs b/backend/windmill-api/src/triggers/postgres/handler.rs index 5dce0345c6..3b0a1b6972 100644 --- a/backend/windmill-api/src/triggers/postgres/handler.rs +++ b/backend/windmill-api/src/triggers/postgres/handler.rs @@ -47,7 +47,6 @@ impl TriggerCrud for PostgresTrigger { const TABLE_NAME: &'static str = "postgres_trigger"; const TRIGGER_TYPE: &'static str = "postgres"; - const SUPPORTS_ENABLED: bool = true; const SUPPORTS_SERVER_STATE: bool = true; const SUPPORTS_TEST_CONNECTION: bool = true; const ROUTE_PREFIX: &'static str = "/postgres_triggers"; @@ -64,7 +63,6 @@ impl TriggerCrud for PostgresTrigger { DeployedObject::PostgresTrigger { path } } - async fn create_trigger( &self, db: &DB, diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs index 191c57b166..ea267721a2 100644 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -350,14 +350,14 @@ pub async fn get_raw_postgres_connection( } let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - let client = if let Some(connector) = connector { let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; tokio::spawn(async move { + tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); + tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); }; - tracing::info!("Successfully Connected into database"); + tracing::info!("PostgreSQL trigger connection closed"); }); client } else { diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-api/src/triggers/sqs/handler_oss.rs index 6b6d18b72e..21200e8e38 100644 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ b/backend/windmill-api/src/triggers/sqs/handler_oss.rs @@ -26,7 +26,6 @@ impl TriggerCrud for SqsTrigger { const TABLE_NAME: &'static str = ""; const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_ENABLED: bool = false; const SUPPORTS_SERVER_STATE: bool = false; const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/sqs_triggers"; diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-api/src/triggers/websocket/handler.rs index 510f6e97ae..2439be8292 100644 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ b/backend/windmill-api/src/triggers/websocket/handler.rs @@ -30,7 +30,6 @@ impl TriggerCrud for WebsocketTrigger { const TABLE_NAME: &'static str = "websocket_trigger"; const TRIGGER_TYPE: &'static str = "websocket"; - const SUPPORTS_ENABLED: bool = true; const SUPPORTS_SERVER_STATE: bool = true; const SUPPORTS_TEST_CONNECTION: bool = true; const ROUTE_PREFIX: &'static str = "/websocket_triggers"; diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 4e6c5ab086..1f49ec5492 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -69,6 +69,8 @@ struct WorkerPing { memory_usage: Option, #[serde(skip_serializing_if = "Option::is_none")] wm_memory_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + job_isolation: Option, } // #[derive(Serialize, Deserialize)] @@ -97,8 +99,8 @@ async fn list_worker_pings( let rows = sqlx::query_as!( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, - CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage + CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index c7053209da..cea6c2ba5c 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2455,38 +2455,6 @@ async fn create_workspace( .execute(&mut *tx) .await?; - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_themes', 'App Themes', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - nw.id, - username, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_custom', 'App Custom Components', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - nw.id, - username, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by, edited_at) VALUES ($1, 'app_groups', 'App Groups', ARRAY[]::TEXT[], '{\"g/all\": false}', $2, now()) ON CONFLICT DO NOTHING", - nw.id, - username, - ) - .execute(&mut *tx) - .await?; - - sqlx::query!( - "INSERT INTO resource (workspace_id, path, value, description, resource_type, created_by, edited_at) VALUES ($1, 'f/app_themes/theme_0', '{\"name\": \"Default Theme\", \"value\": \"\"}', 'The default app theme', 'app_theme', $2, now()) ON CONFLICT DO NOTHING", - nw.id, - username, - ) - .execute(&mut *tx) - .await?; - audit_log( &mut *tx, &authed, diff --git a/backend/windmill-api/src/workspaces_extra.rs b/backend/windmill-api/src/workspaces_extra.rs index 2df23f3ce9..ca4f9db6a7 100644 --- a/backend/windmill-api/src/workspaces_extra.rs +++ b/backend/windmill-api/src/workspaces_extra.rs @@ -448,6 +448,17 @@ pub(crate) async fn delete_workspace( require_super_admin(&db, &authed.email).await?; } + sqlx::query!("DELETE FROM ai_agent_memory WHERE workspace_id = $1", &w_id) + .execute(&mut *tx) + .await?; + + sqlx::query!( + "DELETE FROM flow_conversation WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + sqlx::query!("DELETE FROM workspace_env WHERE workspace_id = $1", &w_id) .execute(&mut *tx) .await?; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 8317940079..c0a52c6677 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -7,14 +7,14 @@ edition.workspace = true [features] default = [] enterprise = [] -private = [] +private = ["dep:aws-sdk-rds"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] prometheus = ["dep:prometheus"] loki = ["dep:tracing-loki"] benchmark = [] -parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"] -aws_auth = ["dep:aws-sdk-sts", "dep:aws-config"] +parquet = ["dep:object_store", "dep:aws-sdk-sts", "dep:aws-smithy-types-convert", "dep:datafusion"] +aws_auth = ["dep:aws-sdk-sts"] otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"] smtp = ["dep:mail-send"] @@ -62,12 +62,15 @@ tracing-loki = { version = "^0", optional = true } magic-crypt.workspace = true object_store = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } -aws-config = { workspace = true, optional = true } +aws-config.workspace = true aws-sdk-sts = { workspace = true, optional = true } +aws-credential-types.workspace = true +aws-smithy-types.workspace = true base64.workspace = true bitflags.workspace = true aws-smithy-types-convert = { workspace = true, optional = true } +aws-sdk-rds = { workspace = true, optional = true } indexmap.workspace = true bytes.workspace = true mail-send = { workspace = true, optional = true } @@ -86,6 +89,7 @@ openidconnect = { workspace = true, optional = true } strum.workspace = true strum_macros.workspace = true url.workspace = true +urlencoding.workspace = true async-recursion.workspace = true semver.workspace = true diff --git a/backend/windmill-common/src/ai_providers.rs b/backend/windmill-common/src/ai_providers.rs index 46d5d687c3..b8c1aaf8fe 100644 --- a/backend/windmill-common/src/ai_providers.rs +++ b/backend/windmill-common/src/ai_providers.rs @@ -10,7 +10,6 @@ lazy_static::lazy_static! { static ref OPENAI_AZURE_BASE_PATH: Option = std::env::var("OPENAI_AZURE_BASE_PATH").ok(); } -pub const AZURE_API_VERSION: &str = "2025-04-01-preview"; pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] @@ -27,11 +26,18 @@ pub enum AIProvider { OpenRouter, TogetherAI, CustomAI, + #[serde(rename = "aws_bedrock")] + AWSBedrock, } impl AIProvider { /// Get the base URL for the AI provider - pub async fn get_base_url(&self, resource_base_url: Option, db: &DB) -> Result { + pub async fn get_base_url( + &self, + resource_base_url: Option, + region: Option, + db: &DB, + ) -> Result { match self { AIProvider::OpenAI => { // Check for Azure base path override @@ -74,6 +80,10 @@ impl AIProvider { ))) } } + AIProvider::AWSBedrock => Ok(format!( + "https://bedrock-runtime.{}.amazonaws.com", + region.unwrap_or_else(|| "us-east-1".to_string()) + )), } } @@ -89,13 +99,12 @@ impl AIProvider { } /// Build Azure OpenAI URL with deployment model path - pub fn build_azure_openai_url(base_url: &str, model: &str, path: &str) -> String { + pub fn build_azure_openai_url(base_url: &str, path: &str) -> String { let base_url = base_url.trim_end_matches('/'); - - if base_url.ends_with("/deployments") { - format!("{}/{}/{}", base_url, model, path) - } else if base_url.ends_with("/openai") { - format!("{}/deployments/{}/{}", base_url, model, path) + if base_url.ends_with("/openai") { + format!("{}/v1/{}", base_url, path) + } else if base_url.ends_with("/deployments") { + format!("{}/v1/{}", base_url.trim_end_matches("/deployments"), path) } else { format!("{}/{}", base_url, path) } diff --git a/backend/windmill-common/src/auth.rs b/backend/windmill-common/src/auth.rs index e48cf3728e..89d4831e38 100644 --- a/backend/windmill-common/src/auth.rs +++ b/backend/windmill-common/src/auth.rs @@ -158,6 +158,21 @@ impl JWTAuthClaims { .as_ref() .is_some_and(|token_w_ids| token_w_ids.iter().any(|token_w_id| w_id == token_w_id)) } + + pub fn compute_ext_jwt_hash(&self) -> i64 { + let mut hasher = DefaultHasher::new(); + self.email.hash(&mut hasher); + self.username.hash(&mut hasher); + self.is_admin.hash(&mut hasher); + self.is_operator.hash(&mut hasher); + self.groups.hash(&mut hasher); + self.folders.hash(&mut hasher); + self.workspace_id.hash(&mut hasher); + self.workspace_ids.hash(&mut hasher); + self.label.hash(&mut hasher); + self.scopes.hash(&mut hasher); + hasher.finish() as i64 + } } #[derive(Deserialize, Debug)] diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index c7972d3a7e..df0ef6842a 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -285,6 +285,21 @@ pub struct FlowData { pub flow: FlowValue, } +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FlowNotes { + pub notes: Option>, +} + +impl FlowData { + pub fn notes(&self) -> Option { + serde_json::from_str::(self.raw_flow.get()) + .map_err(|e| { + tracing::error!("Failed to parse notes into FlowNotes: {}", e); + error::Error::internal_err(format!("Failed to parse notes into FlowNotes: {}", e)) + }) + .ok() + } +} /// !!!Shouldn't be used. Reverted optimization for ai agent steps.!!! #[derive(Deserialize)] struct RevertedFlowNodeFlow { diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 5b81d35f75..27044c1dd3 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -194,7 +194,7 @@ pub struct FlowValue { #[serde(skip_serializing_if = "Option::is_none")] pub chat_input_enabled: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub flow_env: Option>> + pub flow_env: Option>>, } impl FlowValue { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b91dcfaf95..04e59cfac9 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -35,6 +35,8 @@ pub mod bench; pub mod cache; pub mod client; pub mod db; +#[cfg(all(feature = "enterprise", feature = "private"))] +mod db_iam_ee; #[cfg(feature = "private")] pub mod ee; pub mod ee_oss; @@ -341,7 +343,12 @@ pub fn parse_postgres_url(url: &str) -> Result { let scheme = parsed_url.scheme().to_string(); let username = parsed_url.username().to_string(); + let username = urlencoding::decode(&username).map_err(to_anyhow)?.to_string(); let password = parsed_url.password().map(|p| p.to_string()); + let password = match password { + Some(p) => Some(urlencoding::decode(&p).map_err(to_anyhow)?.to_string()), + None => None, + }; let host = parsed_url .host_str() .ok_or_else(|| Error::BadConfig("Missing host in PostgreSQL URL".to_string()))? @@ -370,27 +377,120 @@ pub fn parse_postgres_url(url: &str) -> Result { }) } -pub async fn get_database_url() -> Result { - use std::env::var; - use tokio::fs::File; - use tokio::io::AsyncReadExt; - match var("DATABASE_URL_FILE") { - Ok(file_path) => { - let mut file = File::open(file_path).await?; - let mut contents = String::new(); - file.read_to_string(&mut contents).await?; - Ok(contents.trim().to_string()) +#[derive(Clone)] +pub enum DatabaseUrl { + #[cfg(all(feature = "enterprise", feature = "private"))] + IamRds(std::sync::Arc>), + Static(String), +} + +impl DatabaseUrl { + pub async fn as_str(&self) -> String { + match self { + #[cfg(all(feature = "enterprise", feature = "private"))] + DatabaseUrl::IamRds(rds_url) => { + let guard = rds_url.read().await; + guard.as_str().to_string() + } + DatabaseUrl::Static(url) => url.clone(), } - Err(_) => var("DATABASE_URL").map_err(|_| { - Error::BadConfig( - "Either DATABASE_URL_FILE or DATABASE_URL env var is missing".to_string(), - ) - }), } + + pub async fn refresh(&self) -> anyhow::Result<()> { + match self { + #[cfg(all(feature = "enterprise", feature = "private"))] + DatabaseUrl::IamRds(rds_url) => rds_url.write().await.refresh().await, + DatabaseUrl::Static(_) => Ok(()), + } + } + + +} + +static DATABASE_URL_CACHE: tokio::sync::OnceCell = + tokio::sync::OnceCell::const_new(); + +pub async fn get_database_url() -> Result { + let database_url = DATABASE_URL_CACHE + .get_or_try_init(|| async { + use std::env::var; + use tokio::fs::File; + use tokio::io::AsyncReadExt; + + let url = match var("DATABASE_URL_FILE") { + Ok(file_path) => { + let mut file = File::open(file_path).await?; + let mut contents = String::new(); + file.read_to_string(&mut contents).await?; + Ok(contents.trim().to_string()) + } + Err(_) => var("DATABASE_URL").map_err(|_| { + Error::BadConfig( + "Either DATABASE_URL_FILE or DATABASE_URL env var is missing".to_string(), + ) + }), + }?; + + let parsed_url = url::Url::parse(&url)?; + + if parsed_url.password().is_some_and(|x| x == "iamrds") { + let region = var("AWS_REGION").map_err(|_| { + Error::BadConfig( + "AWS_REGION env var is required for IAM RDS authentication".to_string(), + ) + })?; + + tracing::info!("iamrds mode detected, generating IAM RDS URL for region: {region}"); + #[cfg(all(feature = "enterprise", feature = "private"))] + { + let rds_url = db_iam_ee::generate_database_url(&url, ®ion) + .await + .map_err(|e| { + Error::InternalErr(format!("Failed to generate IAM database URL: {}", e)) + })?; + tracing::info!("IAM RDS URL generated successfully"); + Ok::(DatabaseUrl::IamRds( + std::sync::Arc::new(tokio::sync::RwLock::new(rds_url)) + )) + } + + #[cfg(not(all(feature = "enterprise", feature = "private")))] + { + return Err(Error::BadConfig("IAM RDS authentication is not enabled in OSS mode".to_string())); + } + } else { + Ok::(DatabaseUrl::Static(url.to_string())) + } + }) + .await?; + + // Check if we need to refresh and do so if necessary + #[cfg(all(feature = "enterprise", feature = "private"))] + if let DatabaseUrl::IamRds(ref rds_url_lock) = database_url { + // Check if refresh is needed + let needs_refresh = { + let read_guard = rds_url_lock.read().await; + read_guard.needs_refresh() + }; + + // If refresh is needed, acquire write lock and refresh + if needs_refresh { + let mut write_guard = rds_url_lock.write().await; + // Double-check after acquiring write lock (another task might have refreshed) + if write_guard.needs_refresh() { + write_guard.refresh().await.map_err(|e| { + Error::InternalErr(format!("Failed to refresh IAM token: {}", e)) + })?; + } + } + } + + // Return the URL string + Ok(database_url.clone()) } pub async fn initial_connection() -> Result, error::Error> { - let database_url = get_database_url().await?; + let database_url = get_database_url().await?.as_str().await; sqlx::postgres::PgPoolOptions::new() .max_connections(2) .connect_with(sqlx::postgres::PgConnectOptions::from_str(&database_url)?) @@ -402,6 +502,8 @@ pub async fn connect_db( server_mode: bool, indexer_mode: bool, worker_mode: bool, + #[cfg(feature = "private")] + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> anyhow::Result> { use anyhow::Context; @@ -426,11 +528,58 @@ pub async fn connect_db( } }; - Ok(connect(&database_url, max_connections, worker_mode).await?) + + let pool = connect(database_url.clone(), max_connections, worker_mode).await?; + #[cfg(all(feature = "enterprise", feature = "private"))] + let pool2 = pool.clone(); + #[cfg(all(feature = "enterprise", feature = "private"))] + if let DatabaseUrl::IamRds(database_url) = database_url { + tokio::spawn(async move { + loop { + tokio::select! { + _ = killpill_rx.recv() => { + break; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => { + let needs_refresh = { + let read_guard = database_url.read().await; + read_guard.needs_refresh() + }; + if needs_refresh { + let new_url = tokio::time::timeout(std::time::Duration::from_secs(10), get_database_url()).await; + match new_url { + Ok(Ok(new_url)) => { + let new_url = new_url.as_str().await; + let connect_options = sqlx::postgres::PgConnectOptions::from_str(&new_url); + if let Err(e) = connect_options { + tracing::error!("Error parsing IAM RDS URL as connect options, retrying in 10s: {}", e); + continue; + } + pool2.set_connect_options(connect_options.unwrap()); + tracing::info!("Refreshed IAM RDS URL successfully"); + } + Ok(Err(e)) => { + tracing::error!("Error refreshing IAM RDS URL, trying again in 10s: {}", e); + continue; + } + Err(e) => { + tracing::error!("Timeout after 10s refreshing IAM RDS URL, trying again in 10 seconds: {}", e); + continue; + } + } + } + } + } + + } + }); + } + + Ok(pool) } pub async fn connect( - database_url: &str, + database_url: DatabaseUrl, max_connections: u32, worker_mode: bool, ) -> Result, error::Error> { @@ -479,7 +628,7 @@ pub async fn connect( } }) .connect_with( - sqlx::postgres::PgConnectOptions::from_str(database_url)?.statement_cache_capacity(400), + sqlx::postgres::PgConnectOptions::from_str(&database_url.as_str().await)?.statement_cache_capacity(400), ) .await .map_err(|err| Error::ConnectingToDatabase(err.to_string())) @@ -490,9 +639,7 @@ type Tag = String; pub use db::DB; use crate::{ - auth::{PermsCache, FLOW_PERMS_CACHE, HASH_PERMS_CACHE}, - db::{AuthedRef, UserDbWithAuthed}, - scripts::ScriptHash, + auth::{FLOW_PERMS_CACHE, HASH_PERMS_CACHE, PermsCache}, db::{AuthedRef, UserDbWithAuthed}, error::to_anyhow, scripts::ScriptHash }; #[derive(Clone)] diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index 899e3a2fd5..69fbaa4b29 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -396,6 +396,21 @@ pub struct S3Resource { pub port: Option, } +impl S3Resource { + pub fn endpoint_with_region_fallback(&self, region_fallback: Option) -> String { + if self.endpoint.is_empty() { + let final_region = if self.region.is_empty() { + region_fallback.unwrap_or_else(|| "us-east-1".to_string()) + } else { + self.region.clone() + }; + format!("s3.{}.amazonaws.com", final_region) + } else { + self.endpoint.clone() + } + } +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct AzureBlobResource { pub endpoint: Option, @@ -642,7 +657,7 @@ pub async fn build_s3_client(s3_resource_ref: &S3Resource) -> error::Result { pub value: T, pub expiry: std::time::Instant, } + +impl ExpiringCacheEntry { + pub fn is_expired(&self) -> bool { + self.expiry < std::time::Instant::now() + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 0b0055ed0d..2918d40b0c 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1301,6 +1301,7 @@ pub struct Ping { pub occupancy_rate_15s: Option, pub occupancy_rate_5m: Option, pub occupancy_rate_30m: Option, + pub job_isolation: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -1348,6 +1349,7 @@ pub async fn update_ping_http( &insert_ping.version.unwrap(), insert_ping.vcpus, insert_ping.memory, + insert_ping.job_isolation, db, ) .await?; @@ -1363,6 +1365,7 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_15s, insert_ping.occupancy_rate_5m, insert_ping.occupancy_rate_30m, + insert_ping.job_isolation, db, ) .await?; @@ -1476,10 +1479,11 @@ pub async fn insert_ping_query( version: &str, vcpus: Option, memory: Option, + job_isolation: Option, db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (worker) DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", worker_instance, worker_name, @@ -1489,7 +1493,8 @@ pub async fn insert_ping_query( dw, version, vcpus, - memory + memory, + job_isolation.as_deref() ) .execute(db) .await?; @@ -1506,11 +1511,12 @@ pub async fn update_worker_ping_from_job_query( occupancy_rate_15s: Option, occupancy_rate_5m: Option, occupancy_rate_30m: Option, + job_isolation: Option, db: &DB, ) -> anyhow::Result<()> { sqlx::query!( "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4, - occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9, job_isolation = $10 WHERE worker = $5", job_id, w_id, memory_usage, @@ -1520,6 +1526,7 @@ pub async fn update_worker_ping_from_job_query( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + job_isolation, ) .execute(db) .await?; diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index edd9c8abe5..0fa8769ffd 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -208,7 +208,7 @@ pub async fn get_ducklake_from_db_unchecked( let catalog_resource = if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance { - let pg_creds = parse_postgres_url(&get_database_url().await?)?; + let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; json!({ "dbname": ducklake.catalog.resource_path, "host": pg_creds.host, diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.lock b/backend/windmill-duckdb-ffi-internal/Cargo.lock index a53a988f30..3ae3e5b066 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.lock +++ b/backend/windmill-duckdb-ffi-internal/Cargo.lock @@ -415,8 +415,8 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "duckdb" -version = "1.4.1" -source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" +version = "1.4.2" +source = "git+https://github.com/windmill-labs/duckdb-rs.git?rev=fe0702529de6ec5a568337726bba9355503157d2#fe0702529de6ec5a568337726bba9355503157d2" dependencies = [ "arrow", "cast", @@ -702,8 +702,8 @@ checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libduckdb-sys" -version = "1.4.1" -source = "git+https://github.com/diegoimbert/duckdb-rs?branch=main#0df52a6941c9996d7ec60d585eaf6430db8c48cf" +version = "1.4.2" +source = "git+https://github.com/windmill-labs/duckdb-rs.git?rev=fe0702529de6ec5a568337726bba9355503157d2#fe0702529de6ec5a568337726bba9355503157d2" dependencies = [ "cc", "flate2", diff --git a/backend/windmill-duckdb-ffi-internal/Cargo.toml b/backend/windmill-duckdb-ffi-internal/Cargo.toml index 9560e1d0e6..b1535daba9 100644 --- a/backend/windmill-duckdb-ffi-internal/Cargo.toml +++ b/backend/windmill-duckdb-ffi-internal/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" [dependencies] chrono = "0.4.41" -duckdb = { git = "https://github.com/diegoimbert/duckdb-rs", branch = "main", features = ["bundled"] } +duckdb = { rev = "fe0702529de6ec5a568337726bba9355503157d2", git = "https://github.com/windmill-labs/duckdb-rs.git", features = ["bundled"] } rust_decimal = "1.37.2" serde = { version = "1.0", features = ["derive"] } serde_json = { version = "^1", features = ["preserve_order", "raw_value"] } diff --git a/backend/windmill-duckdb-ffi-internal/build_dev.sh b/backend/windmill-duckdb-ffi-internal/build_dev.sh index 0b42352732..6d6a7f66c5 100755 --- a/backend/windmill-duckdb-ffi-internal/build_dev.sh +++ b/backend/windmill-duckdb-ffi-internal/build_dev.sh @@ -1,2 +1,2 @@ -cargo build --release -p windmill_duckdb_ffi_internal +CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release -p windmill_duckdb_ffi_internal cp target/release/libwindmill_duckdb_ffi_internal.* ../target/debug/ \ No newline at end of file diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index 501b5c5dee..ee5396d45a 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -265,13 +265,22 @@ fn do_duckdb_inner( None => { type_aliases = Some( (0..stmt.column_count()) - .map(|i| stmt.column_logical_type(i).get_alias()) + .map(|i| { + let logical_type = stmt.column_logical_type(i); + if logical_type.is_invalid() { + None + } else { + logical_type.get_alias() + } + }) .collect::>(), ); type_aliases.as_ref().unwrap() } }; + // let type_aliases = (0..stmt.column_count()).map(|_| None).collect::>(); + let row = row_to_value(row, &column_names.as_slice(), &type_aliases.as_slice()) .map_err(|e| e.to_string())?; rows_vec.push(row); diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 6349aee84e..16178502fe 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -47,3 +47,5 @@ regex.workspace = true backon.workspace = true quick_cache.workspace = true thiserror.workspace = true +dashmap.workspace = true +once_cell.workspace = true diff --git a/backend/windmill-queue/src/cloud_usage.rs b/backend/windmill-queue/src/cloud_usage.rs new file mode 100644 index 0000000000..7ded6977ef --- /dev/null +++ b/backend/windmill-queue/src/cloud_usage.rs @@ -0,0 +1,203 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use chrono::Datelike; +use dashmap::DashMap; +use sqlx::{Pool, Postgres}; +use std::sync::Arc; +use tokio::sync::Notify; + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +struct UsageKey { + id: String, + is_workspace: bool, + month: i32, +} + +pub struct UsageBuffer { + buffer: Arc>, + db: Pool, + shutdown_notify: Arc, +} + +impl UsageBuffer { + pub fn new(db: Pool) -> Arc { + let buffer = Arc::new(Self { + buffer: Arc::new(DashMap::new()), + db, + shutdown_notify: Arc::new(Notify::new()), + }); + + // Spawn the periodic flush task + let buffer_clone = buffer.clone(); + tokio::spawn(async move { + buffer_clone.flush_loop().await; + }); + + buffer + } + + pub fn increment(&self, workspace_id: String, email: Option) { + let month = Self::current_month(); + + // Increment workspace usage + self.buffer + .entry(UsageKey { id: workspace_id, is_workspace: true, month }) + .and_modify(|counter| *counter += 1) + .or_insert(1); + + // Increment user usage if email is provided + if let Some(email) = email { + self.buffer + .entry(UsageKey { id: email, is_workspace: false, month }) + .and_modify(|counter| *counter += 1) + .or_insert(1); + } + } + + async fn flush_loop(&self) { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(60)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + _ = interval.tick() => { + self.flush().await; + } + _ = self.shutdown_notify.notified() => { + // Final flush on shutdown + self.flush().await; + break; + } + } + } + } + + async fn flush(&self) { + if self.buffer.is_empty() { + return; + } + + // Drain all buffered usage counts + let mut to_flush = Vec::new(); + self.buffer.retain(|key, value| { + to_flush.push((key.clone(), *value)); + false + }); + + if to_flush.is_empty() { + return; + } + + tracing::debug!( + "Flushing {} buffered usage entries to database", + to_flush.len() + ); + + // Batch update to database + for (key, count) in to_flush { + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, $2, $3, $4) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + $4", + &key.id, + key.is_workspace, + key.month, + count + ) + .execute(&self.db), + ) + .await; + + match result { + Ok(Ok(_)) => {} + Ok(Err(e)) => { + tracing::error!( + "Failed to flush usage for {} (is_workspace: {}): {:#}", + key.id, + key.is_workspace, + e + ); + } + Err(_) => { + tracing::error!( + "Usage flush timed out for {} (is_workspace: {})", + key.id, + key.is_workspace + ); + } + } + } + } + + fn current_month() -> i32 { + let now = chrono::Utc::now(); + (now.year() * 12 + now.month() as i32) as i32 + } +} + +lazy_static::lazy_static! { + static ref USAGE_BUFFER: once_cell::sync::OnceCell> = once_cell::sync::OnceCell::new(); +} + +pub fn init_usage_buffer(db: Pool) { + USAGE_BUFFER.get_or_init(|| UsageBuffer::new(db)); +} + +pub fn increment_usage_async(db: Pool, workspace_id: String, email: Option) { + if let Some(buffer) = USAGE_BUFFER.get() { + buffer.increment(workspace_id, email); + } else { + tracing::warn!("Usage buffer not initialized, falling back to direct database update"); + // Fallback to old implementation if buffer not initialized + tokio::task::spawn(async move { + let result = tokio::time::timeout(std::time::Duration::from_secs(10), async { + // Update workspace usage + let workspace_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + &workspace_id + ) + .execute(&db) + .await; + + if let Err(e) = workspace_result { + tracing::error!("Failed to update workspace usage for {}: {:#}", workspace_id, e); + } + + // Update user usage if email is provided (non-premium workspaces only) + if let Some(ref email) = email { + let user_result = sqlx::query!( + "INSERT INTO usage (id, is_workspace, month_, usage) + VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) + ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", + email + ) + .execute(&db) + .await; + + if let Err(e) = user_result { + tracing::error!("Failed to update user usage for {}: {:#}", email, e); + } + } + }) + .await; + + if let Err(_) = result { + tracing::error!( + "Usage update timed out after 10s for workspace {} and email {:?}", + workspace_id, + email + ); + } + }); + } +} diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 7783751944..c050d56659 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -3809,50 +3809,7 @@ async fn check_usage_limits( } #[cfg(feature = "cloud")] -fn increment_usage_async(db: Pool, workspace_id: String, email: Option) { - tokio::task::spawn(async move { - let result = tokio::time::timeout(std::time::Duration::from_secs(10), async { - // Update workspace usage - let workspace_result = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", - &workspace_id - ) - .execute(&db) - .await; - - if let Err(e) = workspace_result { - tracing::error!("Failed to update workspace usage for {}: {:#}", workspace_id, e); - } - - // Update user usage if email is provided (non-premium workspaces only) - if let Some(ref email) = email { - let user_result = sqlx::query!( - "INSERT INTO usage (id, is_workspace, month_, usage) - VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), 1) - ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + 1", - email - ) - .execute(&db) - .await; - - if let Err(e) = user_result { - tracing::error!("Failed to update user usage for {}: {:#}", email, e); - } - } - }) - .await; - - if let Err(_) = result { - tracing::error!( - "Usage update timed out after 10s for workspace {} and email {:?}", - workspace_id, - email - ); - } - }); -} +use crate::cloud_usage::increment_usage_async; // #[instrument(level = "trace", skip_all)] pub async fn push<'c, 'd>( diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index 9b6025e515..3ea2e2c1aa 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -14,3 +14,8 @@ pub mod schedule; pub use jobs::*; pub mod flow_status; pub mod tags; + +#[cfg(feature = "cloud")] +pub mod cloud_usage; +#[cfg(feature = "cloud")] +pub use cloud_usage::init_usage_buffer; diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index d7e0f66685..19ebb88db5 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -58,6 +58,10 @@ windmill-parser-graphql.workspace = true windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true rmcp = { version = "0.8.1", features = ["client", "transport-streamable-http-client", "transport-streamable-http-client-reqwest"] } +aws-sdk-bedrockruntime.workspace = true +aws-config.workspace = true +aws-credential-types.workspace = true +aws-smithy-types.workspace = true flume.workspace = true sqlx.workspace = true uuid.workspace = true diff --git a/backend/windmill-worker/src/ai/image_handler.rs b/backend/windmill-worker/src/ai/image_handler.rs index c668ded17f..946c922fed 100644 --- a/backend/windmill-worker/src/ai/image_handler.rs +++ b/backend/windmill-worker/src/ai/image_handler.rs @@ -4,6 +4,8 @@ use ulid; use windmill_common::{client::AuthedClient, error::Error, s3_helpers::S3Object}; use windmill_queue::MiniPulledJob; +use crate::ai::types::*; + /// Upload image to S3 and return S3Object pub async fn upload_image_to_s3( base64_image: &str, @@ -66,3 +68,53 @@ pub async fn download_and_encode_s3_image( Ok((mime_type.to_string(), base64_data)) } + +/// Prepare messages for API by converting S3Objects to base64 ImageUrls +pub async fn prepare_messages_for_api( + messages: &[OpenAIMessage], + client: &AuthedClient, + workspace_id: &str, +) -> Result, Error> { + let mut prepared_messages = Vec::new(); + + for message in messages { + let mut prepared_message = message.clone(); + + if let Some(content) = &message.content { + match content { + OpenAIContent::Text(text) => { + prepared_message.content = Some(OpenAIContent::Text(text.clone())); + } + OpenAIContent::Parts(parts) => { + let mut prepared_content = Vec::new(); + + for part in parts { + match part { + ContentPart::S3Object { s3_object } => { + // Convert S3Object to base64 image URL + let (mime_type, image_bytes) = + download_and_encode_s3_image(s3_object, client, workspace_id) + .await?; + prepared_content.push(ContentPart::ImageUrl { + image_url: ImageUrlData { + url: format!("data:{};base64,{}", mime_type, image_bytes), + }, + }); + } + other => { + // Keep Text and ImageUrl as-is + prepared_content.push(other.clone()); + } + } + } + + prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); + } + } + } + + prepared_messages.push(prepared_message); + } + + Ok(prepared_messages) +} diff --git a/backend/windmill-worker/src/ai/providers/bedrock.rs b/backend/windmill-worker/src/ai/providers/bedrock.rs new file mode 100644 index 0000000000..18d08d0a41 --- /dev/null +++ b/backend/windmill-worker/src/ai/providers/bedrock.rs @@ -0,0 +1,804 @@ +use crate::ai::{ + image_handler::prepare_messages_for_api, + providers::openai::{OpenAIFunction, OpenAIToolCall}, + query_builder::{ParsedResponse, StreamEventProcessor}, + types::StreamingEvent, + types::{ContentPart, OpenAIContent, OpenAIMessage, ToolDef}, +}; +use aws_config::BehaviorVersion; +use aws_credential_types::provider::token::ProvideToken; +use aws_sdk_bedrockruntime::types::{ + ContentBlock, ConversationRole, ConverseStreamOutput, ImageBlock, ImageFormat, ImageSource, + InferenceConfiguration, Message, SystemContentBlock, Tool, ToolInputSchema, ToolSpecification, +}; +use aws_sdk_bedrockruntime::Client as BedrockRuntimeClient; +use std::collections::HashMap; +use windmill_common::{client::AuthedClient, error::Error}; + +/// Constants for commonly used strings to avoid allocations +const FUNCTION_TYPE: &str = "function"; +const EMPTY_JSON: &str = "{}"; + +#[derive(Debug, Clone)] +pub struct BearerTokenProvider { + token: String, +} + +impl BearerTokenProvider { + pub fn new(token: String) -> Self { + Self { token } + } +} + +impl ProvideToken for BearerTokenProvider { + fn provide_token<'a>(&'a self) -> aws_credential_types::provider::future::ProvideToken<'a> + where + Self: 'a, + { + aws_credential_types::provider::future::ProvideToken::ready(Ok( + aws_credential_types::Token::new(self.token.clone(), None), + )) + } +} + +pub struct BedrockClient { + client: BedrockRuntimeClient, +} + +impl BedrockClient { + pub async fn from_bearer_token(bearer_token: String, region: &str) -> Result { + let config = aws_sdk_bedrockruntime::config::Builder::new() + .region(aws_config::Region::new(region.to_string())) + .behavior_version(BehaviorVersion::latest()) + .token_provider(BearerTokenProvider::new(bearer_token)) + .build(); + + Ok(Self { client: BedrockRuntimeClient::from_conf(config) }) + } + + pub fn client(&self) -> &BedrockRuntimeClient { + &self.client + } +} + +/// Format AWS SDK errors with detailed information +fn format_bedrock_error(error: &aws_sdk_bedrockruntime::error::SdkError) -> String +where + E: std::fmt::Debug + std::fmt::Display, + R: std::fmt::Debug, +{ + use aws_sdk_bedrockruntime::error::SdkError; + + match error { + SdkError::ServiceError(err) => { + // Include both the display and debug representations for maximum detail + format!("Service error: {} (details: {:?})", err.err(), err) + } + SdkError::ConstructionFailure(err) => { + format!("Request construction failed: {:?}", err) + } + SdkError::DispatchFailure(err) => { + format!("Request dispatch failed: {:?}", err) + } + SdkError::ResponseError(err) => { + format!("Response error: {:?}", err) + } + SdkError::TimeoutError(err) => { + format!("Request timeout: {:?}", err) + } + _ => format!("{:?}", error), + } +} + +/// Convert AWS Smithy Document to serde_json::Value +fn document_to_json(doc: &aws_smithy_types::Document) -> serde_json::Value { + use aws_smithy_types::Document; + + match doc { + Document::Object(map) => { + let mut obj = serde_json::Map::new(); + for (k, v) in map { + obj.insert(k.clone(), document_to_json(v)); + } + serde_json::Value::Object(obj) + } + Document::Array(arr) => { + serde_json::Value::Array(arr.iter().map(document_to_json).collect()) + } + Document::Number(num) => { + // Try to parse as different number types + serde_json::Value::Number( + serde_json::Number::from_f64(num.to_f64_lossy()) + .unwrap_or(serde_json::Number::from(0)), + ) + } + Document::String(s) => serde_json::Value::String(s.clone()), + Document::Bool(b) => serde_json::Value::Bool(*b), + Document::Null => serde_json::Value::Null, + } +} + +/// Convert serde_json::Value to AWS Smithy Document +fn json_to_document(value: serde_json::Value) -> aws_smithy_types::Document { + use aws_smithy_types::Document; + use serde_json::Value; + + match value { + Value::Object(map) => { + let mut doc_map = std::collections::HashMap::new(); + for (k, v) in map { + doc_map.insert(k, json_to_document(v)); + } + Document::Object(doc_map) + } + Value::Array(arr) => Document::Array(arr.into_iter().map(json_to_document).collect()), + Value::Number(num) => { + if let Some(i) = num.as_i64() { + Document::Number(aws_smithy_types::Number::PosInt(i as u64)) + } else if let Some(f) = num.as_f64() { + Document::Number(aws_smithy_types::Number::Float(f)) + } else { + Document::Number(aws_smithy_types::Number::PosInt(0)) + } + } + Value::String(s) => Document::String(s), + Value::Bool(b) => Document::Bool(b), + Value::Null => Document::Null, + } +} + +/// Convert OpenAI-style messages to Bedrock format +/// +/// Separates system messages from conversation messages as required by Bedrock API. +/// +/// # Returns +/// Tuple of (conversation_messages, system_prompts) +pub fn openai_messages_to_bedrock( + messages: &[OpenAIMessage], +) -> Result<(Vec, Vec), Error> { + let mut bedrock_messages = Vec::new(); + let mut system_prompts = Vec::new(); + + for msg in messages { + match msg.role.as_str() { + "system" => { + // Extract system messages separately + if let Some(ref content) = msg.content { + let text = content_to_text(content); + if !text.is_empty() { + system_prompts.push(SystemContentBlock::Text(text)); + } + } + } + "user" | "assistant" => { + bedrock_messages.push(convert_message(msg)?); + } + "tool" => { + // Tool results are handled as user messages with ToolResult content + bedrock_messages.push(convert_tool_message(msg)?); + } + _ => { + return Err(Error::BadRequest(format!("Unsupported role: {}", msg.role))); + } + } + } + + Ok((bedrock_messages, system_prompts)) +} + +/// Helper to extract text from OpenAIContent (ignoring images) +fn content_to_text(content: &OpenAIContent) -> String { + match content { + OpenAIContent::Text(text) => text.to_string(), + OpenAIContent::Parts(parts) => { + // Extract only text parts and join them + let text_parts: Vec<&str> = parts + .iter() + .filter_map(|part| match part { + ContentPart::Text { text } => Some(text.as_str()), + _ => None, + }) + .collect(); + text_parts.join(" ") + } + } +} + +/// Parse image data URL and extract format and base64 data +fn parse_image_data_url(url: &str) -> Result<(ImageFormat, Vec), Error> { + if !url.starts_with("data:") { + return Err(Error::internal_err("Image URL must be a data URL")); + } + + // Parse data:image/png;base64, + let base64_start = url + .find("base64,") + .ok_or_else(|| Error::internal_err("Invalid data URL format"))?; + + let base64_data = &url[base64_start + 7..]; + let mime_type = url + .split(';') + .next() + .and_then(|s| s.strip_prefix("data:")) + .unwrap_or("image/png"); + + // Extract format from MIME type (e.g., "image/png" -> "png") + let format_str = mime_type + .rsplit_once('/') + .map(|(_, format)| format) + .unwrap_or("png"); + + // Map to ImageFormat enum + let format = match format_str { + "png" => ImageFormat::Png, + "jpeg" | "jpg" => ImageFormat::Jpeg, + "gif" => ImageFormat::Gif, + "webp" => ImageFormat::Webp, + _ => ImageFormat::Png, // Default to PNG + }; + + // Decode base64 + let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, base64_data) + .map_err(|e| Error::internal_err(format!("Failed to decode base64 image: {}", e)))?; + + Ok((format, bytes)) +} + +/// Convert a ContentPart to Bedrock ContentBlock +fn content_part_to_block(part: &ContentPart) -> Result, Error> { + match part { + ContentPart::Text { text } => { + if text.is_empty() { + Ok(None) + } else { + Ok(Some(ContentBlock::Text(text.clone()))) + } + } + ContentPart::ImageUrl { image_url } => { + let (format, bytes) = parse_image_data_url(&image_url.url)?; + + let image_source = ImageSource::Bytes(bytes.into()); + let image_block = ImageBlock::builder() + .format(format) + .source(image_source) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build image block: {}", e)))?; + + Ok(Some(ContentBlock::Image(image_block))) + } + ContentPart::S3Object { .. } => { + // S3Objects are already converted to ImageUrl by prepare_messages_for_api + // If we somehow get here, skip it + Ok(None) + } + } +} + +/// Convert a single OpenAI message to Bedrock Message +fn convert_message(msg: &OpenAIMessage) -> Result { + let role = match msg.role.as_str() { + "user" => ConversationRole::User, + "assistant" => ConversationRole::Assistant, + _ => { + return Err(Error::internal_err(format!( + "Unsupported role: {}", + msg.role + ))); + } + }; + + let mut content_blocks = Vec::new(); + + // Handle content (text and/or images) + if let Some(ref content) = msg.content { + match content { + OpenAIContent::Text(text) => { + if !text.is_empty() { + content_blocks.push(ContentBlock::Text(text.clone())); + } + } + OpenAIContent::Parts(parts) => { + for part in parts { + if let Some(block) = content_part_to_block(part)? { + content_blocks.push(block); + } + } + } + } + } + + // Handle tool calls (for assistant messages) + if let Some(ref tool_calls) = msg.tool_calls { + for tc in tool_calls { + content_blocks.push(convert_tool_call_to_content(tc)?); + } + } + + // Bedrock requires at least one content block + if content_blocks.is_empty() { + content_blocks.push(ContentBlock::Text(String::new())); + } + + Message::builder() + .role(role) + .set_content(Some(content_blocks)) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build message: {}", e))) +} + +/// Convert OpenAI tool call to Bedrock ToolUse content block +fn convert_tool_call_to_content(tool_call: &OpenAIToolCall) -> Result { + let input = json_to_document( + serde_json::from_str(&tool_call.function.arguments) + .unwrap_or_else(|_| serde_json::json!({})), + ); + Ok(ContentBlock::ToolUse( + aws_sdk_bedrockruntime::types::ToolUseBlock::builder() + .tool_use_id(&tool_call.id) + .name(&tool_call.function.name) + .input(input) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build tool use: {}", e)))?, + )) +} + +/// Convert tool result message to Bedrock format +fn convert_tool_message(msg: &OpenAIMessage) -> Result { + let tool_call_id = msg + .tool_call_id + .as_ref() + .ok_or_else(|| Error::internal_err("Tool message missing tool_call_id"))?; + + let content_str = msg + .content + .as_ref() + .map(|c| content_to_text(c)) + .unwrap_or_default(); + + // Try to parse as JSON, otherwise use text + let tool_result_content = + if let Ok(json_val) = serde_json::from_str::(&content_str) { + if json_val.is_object() { + vec![aws_sdk_bedrockruntime::types::ToolResultContentBlock::Json( + json_to_document(json_val), + )] + } else { + // Wrap primitives and arrays in an object + vec![aws_sdk_bedrockruntime::types::ToolResultContentBlock::Json( + json_to_document(serde_json::json!({"result": json_val})), + )] + } + } else { + vec![aws_sdk_bedrockruntime::types::ToolResultContentBlock::Text( + content_str.to_string(), + )] + }; + + let tool_result = ContentBlock::ToolResult( + aws_sdk_bedrockruntime::types::ToolResultBlock::builder() + .tool_use_id(tool_call_id) + .set_content(Some(tool_result_content)) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build tool result: {}", e)))?, + ); + + Message::builder() + .role(ConversationRole::User) + .content(tool_result) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build tool result message: {}", e))) +} + +/// Convert OpenAI tool definitions to Bedrock format +pub fn openai_tools_to_bedrock(tools: &[ToolDef]) -> Result, Error> { + tools + .iter() + .map(|tool_def| { + let spec = &tool_def.function; + + // Convert parameters (RawValue) to Document via serde_json::Value + let param_value: serde_json::Value = serde_json::from_str(spec.parameters.get()) + .map_err(|e| Error::internal_err(format!("Invalid tool schema: {}", e)))?; + let input_schema = ToolInputSchema::Json(json_to_document(param_value)); + + let tool_spec = ToolSpecification::builder() + .name(&spec.name) + .set_description(spec.description.clone()) + .input_schema(input_schema) + .build() + .map_err(|e| Error::internal_err(format!("Failed to build tool spec: {}", e)))?; + + Ok(Tool::ToolSpec(tool_spec)) + }) + .collect() +} + +/// Create inference configuration from parameters +pub fn create_inference_config( + temperature: Option, + max_tokens: Option, +) -> Option { + if temperature.is_none() && max_tokens.is_none() { + return None; + } + + let mut builder = InferenceConfiguration::builder(); + + if let Some(temp) = temperature { + builder = builder.temperature(temp); + } + + if let Some(max_tok) = max_tokens { + builder = builder.max_tokens(max_tok); + } + + Some(builder.build()) +} + +/// Extract text content and tool calls from Bedrock Converse response +pub fn bedrock_response_to_openai( + output: &aws_sdk_bedrockruntime::operation::converse::ConverseOutput, +) -> Result<(Option, Vec), Error> { + let mut text_content = String::new(); + let mut tool_calls = Vec::new(); + + if let Some(message) = output.output().and_then(|o| o.as_message().ok()) { + let content_blocks = message.content(); + if !content_blocks.is_empty() { + for block in content_blocks { + match block { + ContentBlock::Text(text) => { + text_content.push_str(&text); + } + ContentBlock::ToolUse(tool_use) => { + // Convert to OpenAI tool call format + // Convert aws_smithy_types::Document to serde_json::Value + let input_value = document_to_json(tool_use.input()); + let arguments = serde_json::to_string(&input_value) + .unwrap_or_else(|_| EMPTY_JSON.to_string()); + + tool_calls.push(OpenAIToolCall { + id: tool_use.tool_use_id().to_string(), + r#type: FUNCTION_TYPE.to_string(), + function: OpenAIFunction { + name: tool_use.name().to_string(), + arguments, + }, + }); + } + _ => {} + } + } + } + } + + let content = if text_content.is_empty() { + None + } else { + Some(text_content) + }; + + Ok((content, tool_calls)) +} + +/// Extract text delta from Bedrock stream event +pub fn bedrock_stream_event_to_text(event: &ConverseStreamOutput) -> Option { + match event { + ConverseStreamOutput::ContentBlockDelta(delta) => delta + .delta() + .and_then(|d| d.as_text().ok()) + .map(|s| s.to_string()), + _ => None, + } +} + +/// Represents a streaming tool call being accumulated +#[derive(Debug, Clone)] +pub struct StreamingToolCall { + pub id: String, + pub name: String, + pub arguments: String, +} + +/// Extract tool use start event from stream +pub fn bedrock_stream_event_to_tool_start( + event: &ConverseStreamOutput, +) -> Option { + match event { + ConverseStreamOutput::ContentBlockStart(start) => { + if let Some(tool_use) = start.start().and_then(|s| s.as_tool_use().ok()) { + Some(StreamingToolCall { + id: tool_use.tool_use_id().to_string(), + name: tool_use.name().to_string(), + arguments: String::new(), + }) + } else { + None + } + } + _ => None, + } +} + +/// Extract tool use input delta from stream +pub fn bedrock_stream_event_to_tool_delta(event: &ConverseStreamOutput) -> Option { + match event { + ConverseStreamOutput::ContentBlockDelta(delta) => delta + .delta() + .and_then(|d| d.as_tool_use().ok()) + .map(|tool_use| tool_use.input().to_string()), + _ => None, + } +} + +/// Check if stream event indicates content block stop +pub fn bedrock_stream_event_is_block_stop(event: &ConverseStreamOutput) -> bool { + matches!(event, ConverseStreamOutput::ContentBlockStop(_)) +} + +/// Convert accumulated streaming tool calls to OpenAI format +pub fn streaming_tool_calls_to_openai(tool_calls: Vec) -> Vec { + tool_calls + .into_iter() + .map(|tc| OpenAIToolCall { + id: tc.id, + function: OpenAIFunction { name: tc.name, arguments: tc.arguments }, + r#type: FUNCTION_TYPE.to_string(), + }) + .collect() +} + +#[derive(Default)] +pub struct BedrockQueryBuilder; + +impl BedrockQueryBuilder { + /// Execute Bedrock request (streaming or non-streaming) + pub async fn execute_request( + &self, + messages: &[OpenAIMessage], + tools: Option<&[ToolDef]>, + model: &str, + temperature: Option, + max_tokens: Option, + api_key: &str, + region: &str, + should_stream: bool, + stream_event_processor: Option, + client: &AuthedClient, + workspace_id: &str, + structured_output_tool_name: Option<&str>, + ) -> Result { + // Create Bedrock client with bearer token authentication + let bedrock_client = BedrockClient::from_bearer_token(api_key.to_string(), region).await?; + + // Prepare messages: convert S3Objects to ImageUrls by downloading from S3 + let prepared_messages = prepare_messages_for_api(messages, client, workspace_id).await?; + + // Convert messages to Bedrock format (separates system prompts) + let (bedrock_messages, system_prompts) = openai_messages_to_bedrock(&prepared_messages)?; + + // Build inference configuration + let inference_config = create_inference_config(temperature, max_tokens.map(|t| t as i32)); + + // Build tool configuration with optional ToolChoice + let tool_config = self.build_tool_config(tools, structured_output_tool_name.is_some())?; + + if should_stream { + self.execute_converse_stream( + &bedrock_client, + model, + bedrock_messages, + system_prompts, + inference_config, + tool_config, + stream_event_processor, + ) + .await + } else { + self.execute_converse( + &bedrock_client, + model, + bedrock_messages, + system_prompts, + inference_config, + tool_config, + ) + .await + } + } + + /// Build tool configuration with optional ToolChoice for structured output + fn build_tool_config( + &self, + tools: Option<&[ToolDef]>, + force_tool_use: bool, + ) -> Result, Error> { + if let Some(tools) = tools { + let bedrock_tools = openai_tools_to_bedrock(tools)?; + let mut tool_config_builder = + aws_sdk_bedrockruntime::types::ToolConfiguration::builder() + .set_tools(Some(bedrock_tools)); + + // For structured output, force the model to use the tool + if force_tool_use { + tool_config_builder = tool_config_builder.tool_choice( + aws_sdk_bedrockruntime::types::ToolChoice::Any( + aws_sdk_bedrockruntime::types::AnyToolChoice::builder().build(), + ), + ); + } + + Ok(Some(tool_config_builder.build().map_err(|e| { + Error::internal_err(format!("Failed to build tool configuration: {}", e)) + })?)) + } else { + Ok(None) + } + } + + /// Execute non-streaming Bedrock request + async fn execute_converse( + &self, + bedrock_client: &BedrockClient, + model: &str, + bedrock_messages: Vec, + system_prompts: Vec, + inference_config: Option, + tool_config: Option, + ) -> Result { + let mut request_builder = bedrock_client + .client() + .converse() + .model_id(model) + .set_messages(Some(bedrock_messages)); + + if !system_prompts.is_empty() { + request_builder = request_builder.set_system(Some(system_prompts)); + } + + if let Some(config) = inference_config { + request_builder = request_builder.inference_config(config); + } + + if let Some(config) = tool_config { + request_builder = request_builder.set_tool_config(Some(config)); + } + + // Execute the request + let response = request_builder.send().await.map_err(|e| { + let error_msg = format!("Bedrock API error: {}", format_bedrock_error(&e)); + Error::internal_err(error_msg) + })?; + + // Convert response back to OpenAI format + let (content, tool_calls) = bedrock_response_to_openai(&response)?; + + Ok(ParsedResponse::Text { content, tool_calls, events_str: None }) + } + + /// Execute streaming Bedrock request + async fn execute_converse_stream( + &self, + bedrock_client: &BedrockClient, + model: &str, + bedrock_messages: Vec, + system_prompts: Vec, + inference_config: Option, + tool_config: Option, + stream_event_processor: Option, + ) -> Result { + // Build streaming request + let mut request_builder = bedrock_client + .client() + .converse_stream() + .model_id(model) + .set_messages(Some(bedrock_messages)); + + if !system_prompts.is_empty() { + request_builder = request_builder.set_system(Some(system_prompts)); + } + + if let Some(config) = inference_config { + request_builder = request_builder.inference_config(config); + } + + if let Some(config) = tool_config { + request_builder = request_builder.set_tool_config(Some(config)); + } + + // Execute streaming request + let mut stream = request_builder + .send() + .await + .map_err(|e| { + let error_msg = + format!("Bedrock streaming API error: {}", format_bedrock_error(&e)); + Error::internal_err(error_msg) + })? + .stream; + + let mut accumulated_text = String::new(); + let mut events_str = String::new(); + let mut accumulated_tool_calls: HashMap = HashMap::new(); + let mut current_tool_use_id: Option = None; + + // Process stream events + loop { + match stream.recv().await { + Ok(Some(event)) => { + // Handle tool use start + if let Some(tool_call) = bedrock_stream_event_to_tool_start(&event) { + current_tool_use_id = Some(tool_call.id.clone()); + accumulated_tool_calls.insert(tool_call.id.clone(), tool_call); + } + + // Handle text delta + if let Some(text_delta) = bedrock_stream_event_to_text(&event) { + accumulated_text.push_str(&text_delta); + if let Some(processor) = stream_event_processor.as_ref() { + processor + .send( + StreamingEvent::TokenDelta { content: text_delta }, + &mut events_str, + ) + .await?; + } + } + + // Handle tool use input delta + if let Some(input_delta) = bedrock_stream_event_to_tool_delta(&event) { + if let Some(tool_id) = ¤t_tool_use_id { + if let Some(tool_call) = accumulated_tool_calls.get_mut(tool_id) { + tool_call.arguments.push_str(&input_delta); + } + } + } + + // Handle content block stop + if bedrock_stream_event_is_block_stop(&event) { + current_tool_use_id = None; + } + } + Ok(None) => break, // Stream ended + Err(e) => { + return Err(Error::internal_err(format!("Bedrock stream error: {}", e))); + } + } + } + + // Send tool call events to stream processor + if let Some(processor) = stream_event_processor.as_ref() { + for tool_call in accumulated_tool_calls.values() { + processor + .send( + StreamingEvent::ToolCallArguments { + call_id: tool_call.id.clone(), + function_name: tool_call.name.clone(), + arguments: tool_call.arguments.clone(), + }, + &mut events_str, + ) + .await?; + } + } + + let content = if accumulated_text.is_empty() { + None + } else { + Some(accumulated_text) + }; + + let tool_calls = + streaming_tool_calls_to_openai(accumulated_tool_calls.into_values().collect()); + + Ok(ParsedResponse::Text { + content, + tool_calls, + events_str: if events_str.is_empty() { + None + } else { + Some(events_str) + }, + }) + } +} diff --git a/backend/windmill-worker/src/ai/providers/google_ai.rs b/backend/windmill-worker/src/ai/providers/google_ai.rs index b94d4f29e5..0248c8d7e4 100644 --- a/backend/windmill-worker/src/ai/providers/google_ai.rs +++ b/backend/windmill-worker/src/ai/providers/google_ai.rs @@ -255,7 +255,13 @@ impl QueryBuilder for GoogleAIQueryBuilder { .await } - fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + fn get_endpoint( + &self, + base_url: &str, + model: &str, + output_type: &OutputType, + _stream: bool, + ) -> String { match output_type { OutputType::Text => format!("{}/chat/completions", base_url), // Use OpenAI-compatible endpoint OutputType::Image => { diff --git a/backend/windmill-worker/src/ai/providers/mod.rs b/backend/windmill-worker/src/ai/providers/mod.rs index 13cf766e28..e86d70650a 100644 --- a/backend/windmill-worker/src/ai/providers/mod.rs +++ b/backend/windmill-worker/src/ai/providers/mod.rs @@ -1,3 +1,4 @@ +pub mod bedrock; pub mod google_ai; pub mod openai; pub mod openrouter; diff --git a/backend/windmill-worker/src/ai/providers/openai.rs b/backend/windmill-worker/src/ai/providers/openai.rs index d764cc11ad..b3205b529a 100644 --- a/backend/windmill-worker/src/ai/providers/openai.rs +++ b/backend/windmill-worker/src/ai/providers/openai.rs @@ -4,14 +4,13 @@ use serde_json; use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; use crate::ai::{ - image_handler::download_and_encode_s3_image, + image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, sse::{OpenAISSEParser, SSEParser}, types::*, - utils::is_claude_model, + utils::should_use_structured_output_tool, }; -// OpenAI-specific types #[derive(Deserialize, Serialize, Clone, Debug)] pub struct OpenAIFunction { pub name: String, @@ -114,62 +113,6 @@ impl OpenAIQueryBuilder { Self { provider_kind } } - pub async fn prepare_messages_for_api( - &self, - messages: &[OpenAIMessage], - client: &AuthedClient, - workspace_id: &str, - ) -> Result, Error> { - let mut prepared_messages = Vec::new(); - - for message in messages { - let mut prepared_message = message.clone(); - - if let Some(content) = &message.content { - match content { - OpenAIContent::Text(text) => { - prepared_message.content = Some(OpenAIContent::Text(text.clone())); - } - OpenAIContent::Parts(parts) => { - let mut prepared_content = Vec::new(); - - for part in parts { - match part { - ContentPart::S3Object { s3_object } => { - // Convert S3Object to base64 image URL - let (mime_type, image_bytes) = download_and_encode_s3_image( - s3_object, - client, - workspace_id, - ) - .await?; - prepared_content.push(ContentPart::ImageUrl { - image_url: ImageUrlData { - url: format!( - "data:{};base64,{}", - mime_type, image_bytes - ), - }, - }); - } - other => { - // Keep Text and ImageUrl as-is - prepared_content.push(other.clone()); - } - } - } - - prepared_message.content = Some(OpenAIContent::Parts(prepared_content)); - } - } - } - - prepared_messages.push(prepared_message); - } - - Ok(prepared_messages) - } - async fn build_text_request( &self, args: &BuildRequestArgs<'_>, @@ -177,9 +120,8 @@ impl OpenAIQueryBuilder { workspace_id: &str, stream: bool, ) -> Result { - let prepared_messages = self - .prepare_messages_for_api(args.messages, client, workspace_id) - .await?; + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; // Check if we need to add response_format for structured output let has_output_properties = args @@ -203,9 +145,10 @@ impl OpenAIQueryBuilder { None }; - let is_claude_model = is_claude_model(&args.model); + let should_use_structured_output_tool = + should_use_structured_output_tool(&self.provider_kind, args.model); // Force usage of structured output tool for Claude models when structured output provided - let tool_choice = if is_claude_model && response_format.is_some() { + let tool_choice = if should_use_structured_output_tool && response_format.is_some() { Some(ToolChoice::Required) } else { None @@ -405,14 +348,20 @@ impl QueryBuilder for OpenAIQueryBuilder { }) } - fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String { + fn get_endpoint( + &self, + base_url: &str, + _model: &str, + output_type: &OutputType, + _stream: bool, + ) -> String { let path = match output_type { OutputType::Text => "chat/completions", OutputType::Image => "responses", }; if self.provider_kind.is_azure_openai(base_url) { - AIProvider::build_azure_openai_url(base_url, model, path) + AIProvider::build_azure_openai_url(base_url, path) } else { format!("{}/{}", base_url, path) } diff --git a/backend/windmill-worker/src/ai/providers/openrouter.rs b/backend/windmill-worker/src/ai/providers/openrouter.rs index 9e22a63552..f3e215cada 100644 --- a/backend/windmill-worker/src/ai/providers/openrouter.rs +++ b/backend/windmill-worker/src/ai/providers/openrouter.rs @@ -4,6 +4,7 @@ use serde_json; use windmill_common::{ai_providers::AIProvider, client::AuthedClient, error::Error}; use crate::ai::{ + image_handler::prepare_messages_for_api, providers::openai::{OpenAIQueryBuilder, OpenAIResponse}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventProcessor}, types::*, @@ -91,11 +92,9 @@ impl QueryBuilder for OpenRouterQueryBuilder { } OutputType::Image => { // For image generation, we need to add modalities field - // First, prepare the messages using the OpenAI builder's logic - let openai_builder = &self.openai_builder; - let prepared_messages = openai_builder - .prepare_messages_for_api(args.messages, client, workspace_id) - .await?; + // First, prepare the messages + let prepared_messages = + prepare_messages_for_api(args.messages, client, workspace_id).await?; // Check if we need to add response_format for structured output let has_output_properties = args @@ -204,7 +203,13 @@ impl QueryBuilder for OpenRouterQueryBuilder { .await } - fn get_endpoint(&self, base_url: &str, _model: &str, _output_type: &OutputType) -> String { + fn get_endpoint( + &self, + base_url: &str, + _model: &str, + _output_type: &OutputType, + _stream: bool, + ) -> String { // OpenRouter uses the same endpoint for both text and image generation format!("{}/chat/completions", base_url) } diff --git a/backend/windmill-worker/src/ai/query_builder.rs b/backend/windmill-worker/src/ai/query_builder.rs index e332c21f73..603f428c2e 100644 --- a/backend/windmill-worker/src/ai/query_builder.rs +++ b/backend/windmill-worker/src/ai/query_builder.rs @@ -69,7 +69,13 @@ pub trait QueryBuilder: Send + Sync { } /// Get the API endpoint for this provider - fn get_endpoint(&self, base_url: &str, model: &str, output_type: &OutputType) -> String; + fn get_endpoint( + &self, + base_url: &str, + model: &str, + output_type: &OutputType, + stream: bool, + ) -> String; /// Get the authentication headers for this provider fn get_auth_headers( diff --git a/backend/windmill-worker/src/ai/types.rs b/backend/windmill-worker/src/ai/types.rs index 457eb3b215..eb22066436 100644 --- a/backend/windmill-worker/src/ai/types.rs +++ b/backend/windmill-worker/src/ai/types.rs @@ -126,6 +126,7 @@ pub struct ProviderResource { pub api_key: String, #[serde(alias = "baseUrl")] pub base_url: Option, + pub region: Option, } #[derive(Deserialize, Debug)] @@ -146,9 +147,17 @@ impl ProviderWithResource { pub async fn get_base_url(&self, db: &DB) -> Result { self.kind - .get_base_url(self.resource.base_url.clone(), db) + .get_base_url( + self.resource.base_url.clone(), + self.resource.region.clone(), + db, + ) .await } + + pub fn get_region(&self) -> Option<&str> { + self.resource.region.as_deref() + } } #[derive(Serialize)] diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 947b56f549..34b1ad69c4 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -8,6 +8,7 @@ use std::{ }; use uuid::Uuid; use windmill_common::{ + ai_providers::AIProvider, db::DB, error::Error, flow_conversations::{add_message_to_conversation_tx, MessageType}, @@ -311,9 +312,9 @@ pub fn get_step_name_from_flow( ) } -/// Claude models starts with claude if provider is anthropic, or anthropic for openrouter and other providers -pub fn is_claude_model(model: &str) -> bool { - model.starts_with("claude") || model.starts_with("anthropic") +/// AWS Bedrock do not handle structured output query param, so we use a tool for structured output. Same for every Claude models. +pub fn should_use_structured_output_tool(provider: &AIProvider, model: &str) -> bool { + model.contains("claude") || provider == &AIProvider::AWSBedrock } /// Cleanup MCP clients by gracefully shutting down connections diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 481717a207..d18e32e845 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -2,9 +2,9 @@ use crate::ai::tools::{execute_tool_calls, ToolExecutionContext}; use crate::ai::utils::{ add_message_to_conversation, any_tool_needs_previous_result, cleanup_mcp_clients, filter_schema_by_input_transforms, find_unique_tool_name, get_flow_context, - get_flow_job_runnable_and_raw_flow, get_step_name_from_flow, is_claude_model, load_mcp_tools, - parse_raw_script_schema, update_flow_status_module_with_actions, - update_flow_status_module_with_actions_success, + get_flow_job_runnable_and_raw_flow, get_step_name_from_flow, load_mcp_tools, + parse_raw_script_schema, should_use_structured_output_tool, + update_flow_status_module_with_actions, update_flow_status_module_with_actions_success, }; use crate::memory_oss::{read_from_memory, write_to_memory}; use crate::worker_flow::{get_previous_job_result, get_transform_context}; @@ -15,7 +15,7 @@ use std::{collections::HashMap, sync::Arc}; use uuid::Uuid; use windmill_common::mcp_client::McpClient; use windmill_common::{ - ai_providers::AZURE_API_VERSION, + ai_providers::AIProvider, cache, client::AuthedClient, db::DB, @@ -376,6 +376,7 @@ pub async fn run_agent( let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); let base_url = args.provider.get_base_url(db).await?; let api_key = args.provider.get_api_key(); + let region = args.provider.get_region(); // Create the query builder for the provider let query_builder = create_query_builder(&args.provider); @@ -498,14 +499,15 @@ pub async fn run_agent( .map(|props| !props.is_empty()) .unwrap_or(false); - let is_claude_model = is_claude_model(&args.provider.model); + let should_use_structured_output_tool = + should_use_structured_output_tool(&args.provider.kind, &args.provider.model); let mut used_structured_output_tool = false; let mut structured_output_tool_name: Option = None; // For text output with schema, handle structured output if has_output_properties && output_type == &OutputType::Text { let schema = args.output_schema.as_ref().unwrap(); - if is_claude_model { + if should_use_structured_output_tool { // Anthropic uses a tool for structured output let unique_tool_name = find_unique_tool_name("structured_output", tool_defs.as_deref()); structured_output_tool_name = Some(unique_tool_name.clone()); @@ -551,253 +553,285 @@ pub async fn run_agent( break; } - // For text output or image output with tools - let build_args = BuildRequestArgs { - messages: &messages, - tools: tool_defs.as_deref(), - model: args.provider.get_model(), - temperature: args.temperature, - max_tokens: args.max_completion_tokens, - output_schema: args.output_schema.as_ref(), - output_type, - system_prompt: args.system_prompt.as_deref(), - user_message: &args.user_message, - images: args.user_images.as_deref(), - }; + // Special handling for AWS Bedrock using the official SDK + let parsed = if args.provider.kind == AIProvider::AWSBedrock { + let Some(region) = region else { + return Err(Error::internal_err( + "AWS Bedrock region is required".to_string(), + )); + }; + // Use Bedrock SDK via dedicated query builder + crate::ai::providers::bedrock::BedrockQueryBuilder::default() + .execute_request( + &messages, + tool_defs.as_deref(), + args.provider.get_model(), + args.temperature, + args.max_completion_tokens, + api_key, + region, + should_stream, + stream_event_processor.clone(), + client, + &job.workspace_id, + structured_output_tool_name.as_deref(), + ) + .await? + } else { + // For non-Bedrock providers, use HTTP client + let build_args = BuildRequestArgs { + messages: &messages, + tools: tool_defs.as_deref(), + model: args.provider.get_model(), + temperature: args.temperature, + max_tokens: args.max_completion_tokens, + output_schema: args.output_schema.as_ref(), + output_type, + system_prompt: args.system_prompt.as_deref(), + user_message: &args.user_message, + images: args.user_images.as_deref(), + }; - let request_body = query_builder - .build_request(&build_args, client, &job.workspace_id, should_stream) - .await?; + let request_body = query_builder + .build_request(&build_args, client, &job.workspace_id, should_stream) + .await?; - let endpoint = - query_builder.get_endpoint(&base_url, args.provider.get_model(), output_type); - let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); + let endpoint = query_builder.get_endpoint( + &base_url, + args.provider.get_model(), + output_type, + should_stream, + ); + let auth_headers = query_builder.get_auth_headers(api_key, &base_url, output_type); - let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout) - .await - .0; + let timeout = resolve_job_timeout(conn, &job.workspace_id, job.id, job.timeout) + .await + .0; - let mut request = HTTP_CLIENT - .post(&endpoint) - .timeout(timeout) - .header("Content-Type", "application/json"); + let mut request = HTTP_CLIENT + .post(&endpoint) + .timeout(timeout) + .header("Content-Type", "application/json"); - // Apply authentication headers - for (header_name, header_value) in &auth_headers { - request = request.header(*header_name, header_value.clone()); - } + // Apply authentication headers + for (header_name, header_value) in &auth_headers { + request = request.header(*header_name, header_value.clone()); + } - // Apply custom headers from AI_HTTP_HEADERS environment variable - for (header_name, header_value) in AI_HTTP_HEADERS.iter() { - request = request.header(header_name.as_str(), header_value.as_str()); - } + // Apply custom headers from AI_HTTP_HEADERS environment variable + for (header_name, header_value) in AI_HTTP_HEADERS.iter() { + request = request.header(header_name.as_str(), header_value.as_str()); + } - if args.provider.kind.is_azure_openai(&base_url) { - request = request.query(&[("api-version", AZURE_API_VERSION)]) - } + let resp = request + .body(request_body) + .send() + .await + .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; - let resp = request - .body(request_body) - .send() - .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; - - match resp.error_for_status_ref() { - Ok(_) => { - let parsed = if let Some(stream_event_processor) = stream_event_processor.clone() { - query_builder - .parse_streaming_response(resp, stream_event_processor) - .await? - } else { - // Handle non-streaming response - query_builder.parse_response(resp).await? - }; - - match parsed { - ParsedResponse::Text { content: response_content, tool_calls, events_str } => { - if let Some(events_str) = events_str { - final_events_str.push_str(&events_str); - } - - if let Some(ref response_content) = response_content { - actions.push(AgentAction::Message {}); - messages.push(OpenAIMessage { - role: "assistant".to_string(), - content: Some(OpenAIContent::Text(response_content.clone())), - agent_action: Some(AgentAction::Message {}), - ..Default::default() - }); - - update_flow_status_module_with_actions(db, parent_job, &actions) - .await?; - update_flow_status_module_with_actions_success(db, parent_job, true) - .await?; - - content = Some(OpenAIContent::Text(response_content.clone())); - - // Add assistant message to conversation if chat_input_enabled - let chat_enabled = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.chat_input_enabled) - .unwrap_or(false); - if chat_enabled && !response_content.is_empty() { - if let Some(memory_id) = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.memory_id) - { - let agent_job_id = job.id; - let db_clone = db.clone(); - let message_content = response_content.clone(); - let step_name = get_step_name_from_flow( - summary.as_deref(), - job.flow_step_id.as_deref(), - ); - - // Spawn task because we do not need to wait for the result - tokio::spawn(async move { - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - Some(agent_job_id), - &message_content, - MessageType::Assistant, - &step_name, - true, - ) - .await - { - tracing::warn!("Failed to add assistant message to conversation {}: {}", memory_id, e); - } - }); - } - } - } - - if tool_calls.is_empty() { - break; - } else if i == MAX_AGENT_ITERATIONS - 1 { - return Err(Error::internal_err( - "AI agent reached max iterations, but there are still tool calls" - .to_string(), - )); - } - - messages.push(OpenAIMessage { - role: "assistant".to_string(), - tool_calls: Some(tool_calls.clone()), - ..Default::default() - }); - - // Handle tool calls using extracted tools module - let tool_execution_ctx = ToolExecutionContext { - db, - conn, - job, - parent_job, - summary: &summary, - client, - worker_dir, - base_internal_url, - worker_name, - hostname, - occupancy_metrics, - job_completed_tx, - killpill_rx, - stream_event_processor: stream_event_processor.as_ref(), - flow_context: &mut flow_context, - previous_result: &previous_result, - id_context: &id_context, - }; - - let (tool_messages, tool_content, tool_used_structured_output) = - execute_tool_calls( - tool_execution_ctx, - &tool_calls, - &tools, - mcp_clients, - &mut actions, - &mut final_events_str, - &structured_output_tool_name, - ) - .await?; - - messages.extend(tool_messages); - if let Some(tc) = tool_content { - content = Some(tc); - } - used_structured_output_tool = tool_used_structured_output; - } - ParsedResponse::Image { base64_data } => { - // For image output, upload to S3 and track in conversation - let s3_object = upload_image_to_s3(&base64_data, job, client).await?; - - let content = to_raw_value(&s3_object); - - // Add assistant message to conversation if chat_input_enabled - let chat_enabled = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.chat_input_enabled) - .unwrap_or(false); - if chat_enabled { - if let Some(memory_id) = flow_context - .flow_status - .as_ref() - .and_then(|fs| fs.memory_id) - { - let agent_job_id = job.id; - let db_clone = db.clone(); - let flow_step_id_owned = job.flow_step_id.clone(); - let summary_owned = summary.map(|s| s.to_string()); - - // Create extended version with type discriminator for conversation storage - // This avoids conflicts with outputs that are of the same format as S3 objects - let s3_with_type = S3ObjectWithType { - s3_object: s3_object.clone(), - r#type: "windmill_s3_object".to_string(), - }; - - let message_content = serde_json::to_string(&s3_with_type) - .unwrap_or_else(|_| content.get().to_string()); - - // Spawn task because we do not need to wait for the result - tokio::spawn(async move { - let step_name = get_step_name_from_flow( - summary_owned.as_deref(), - flow_step_id_owned.as_deref(), - ); - - if let Err(e) = add_message_to_conversation( - &db_clone, - &memory_id, - Some(agent_job_id), - &message_content, - MessageType::Assistant, - &step_name, - true, - ) - .await - { - tracing::warn!("Failed to add assistant message to conversation {}: {}", memory_id, e); - } - }); - } - } - - // Return early since image generation is complete - return Ok(content); + match resp.error_for_status_ref() { + Ok(_) => { + if let Some(stream_event_processor) = stream_event_processor.clone() { + query_builder + .parse_streaming_response(resp, stream_event_processor) + .await? + } else { + // Handle non-streaming response + query_builder.parse_response(resp).await? } } + Err(e) => { + let _status = resp.status(); + let text = resp + .text() + .await + .unwrap_or_else(|_| "".to_string()); + return Err(Error::internal_err(format!("API error: {} - {}", e, text))); + } } - Err(e) => { - let _status = resp.status(); - let text = resp - .text() - .await - .unwrap_or_else(|_| "".to_string()); - return Err(Error::internal_err(format!("API error: {} - {}", e, text))); + }; + + match parsed { + ParsedResponse::Text { content: response_content, tool_calls, events_str } => { + if let Some(events_str) = events_str { + final_events_str.push_str(&events_str); + } + + if let Some(ref response_content) = response_content { + actions.push(AgentAction::Message {}); + messages.push(OpenAIMessage { + role: "assistant".to_string(), + content: Some(OpenAIContent::Text(response_content.clone())), + agent_action: Some(AgentAction::Message {}), + ..Default::default() + }); + + update_flow_status_module_with_actions(db, parent_job, &actions).await?; + update_flow_status_module_with_actions_success(db, parent_job, true).await?; + + content = Some(OpenAIContent::Text(response_content.clone())); + + // Add assistant message to conversation if chat_input_enabled + let chat_enabled = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.chat_input_enabled) + .unwrap_or(false); + if chat_enabled && !response_content.is_empty() { + if let Some(memory_id) = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + let agent_job_id = job.id; + let db_clone = db.clone(); + let message_content = response_content.clone(); + let step_name = get_step_name_from_flow( + summary.as_deref(), + job.flow_step_id.as_deref(), + ); + + // Spawn task because we do not need to wait for the result + tokio::spawn(async move { + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + Some(agent_job_id), + &message_content, + MessageType::Assistant, + &step_name, + true, + ) + .await + { + tracing::warn!( + "Failed to add assistant message to conversation {}: {}", + memory_id, + e + ); + } + }); + } + } + } + + if tool_calls.is_empty() { + break; + } else if i == MAX_AGENT_ITERATIONS - 1 { + return Err(Error::internal_err( + "AI agent reached max iterations, but there are still tool calls" + .to_string(), + )); + } + + messages.push(OpenAIMessage { + role: "assistant".to_string(), + tool_calls: Some(tool_calls.clone()), + ..Default::default() + }); + + // Handle tool calls using extracted tools module + let tool_execution_ctx = ToolExecutionContext { + db, + conn, + job, + parent_job, + summary: &summary, + client, + worker_dir, + base_internal_url, + worker_name, + hostname, + occupancy_metrics, + job_completed_tx, + killpill_rx, + stream_event_processor: stream_event_processor.as_ref(), + flow_context: &mut flow_context, + previous_result: &previous_result, + id_context: &id_context, + }; + + let (tool_messages, tool_content, tool_used_structured_output) = + execute_tool_calls( + tool_execution_ctx, + &tool_calls, + &tools, + mcp_clients, + &mut actions, + &mut final_events_str, + &structured_output_tool_name, + ) + .await?; + + messages.extend(tool_messages); + if let Some(tc) = tool_content { + content = Some(tc); + } + used_structured_output_tool = tool_used_structured_output; + } + ParsedResponse::Image { base64_data } => { + // For image output, upload to S3 and track in conversation + let s3_object = upload_image_to_s3(&base64_data, job, client).await?; + + let content = to_raw_value(&s3_object); + + // Add assistant message to conversation if chat_input_enabled + let chat_enabled = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.chat_input_enabled) + .unwrap_or(false); + if chat_enabled { + if let Some(memory_id) = flow_context + .flow_status + .as_ref() + .and_then(|fs| fs.memory_id) + { + let agent_job_id = job.id; + let db_clone = db.clone(); + let flow_step_id_owned = job.flow_step_id.clone(); + let summary_owned = summary.map(|s| s.to_string()); + + // Create extended version with type discriminator for conversation storage + // This avoids conflicts with outputs that are of the same format as S3 objects + let s3_with_type = S3ObjectWithType { + s3_object: s3_object.clone(), + r#type: "windmill_s3_object".to_string(), + }; + + let message_content = serde_json::to_string(&s3_with_type) + .unwrap_or_else(|_| content.get().to_string()); + + // Spawn task because we do not need to wait for the result + tokio::spawn(async move { + let step_name = get_step_name_from_flow( + summary_owned.as_deref(), + flow_step_id_owned.as_deref(), + ); + + if let Err(e) = add_message_to_conversation( + &db_clone, + &memory_id, + Some(agent_job_id), + &message_content, + MessageType::Assistant, + &step_name, + true, + ) + .await + { + tracing::warn!( + "Failed to add assistant message to conversation {}: {}", + memory_id, + e + ); + } + }); + } + } + + // Return early since image generation is complete + return Ok(content); } } } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index ac1ae4e233..9aed0e18ad 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -29,13 +29,13 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ bash_executor::BIN_BASH, common::{ - check_executor_binary_exists, get_reserved_variables, read_and_check_result, - start_child_process, transform_json, OccupancyMetrics, + build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, + read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, PyVAlias, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1240,7 +1240,11 @@ fi .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str()); + let ansible_args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect(); + let mut ansible_cmd = build_command_with_isolation( + ANSIBLE_PLAYBOOK_PATH.as_str(), + &ansible_args, + ); ansible_cmd .current_dir(job_dir) .env_clear() @@ -1250,7 +1254,6 @@ fi .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index d147aaff9b..640f3c9fae 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -35,7 +35,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ - build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, + build_args_map, build_command_with_isolation, get_reserved_variables, read_file, read_file_content, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -200,7 +200,10 @@ exit $exit_status } else { let mut cmd_args = vec!["wrapper.sh"]; cmd_args.extend(&args); - let mut bash_cmd = Command::new(BIN_BASH.as_str()); + let mut bash_cmd = build_command_with_isolation( + BIN_BASH.as_str(), + &cmd_args.iter().map(|s| s.as_ref()).collect::>(), + ); bash_cmd .current_dir(job_dir) .env_clear() @@ -209,7 +212,6 @@ exit $exit_status .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 757656b934..b8723ad815 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -17,7 +17,7 @@ use crate::common::build_envs_map; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, - read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, + build_command_with_isolation, read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -1430,15 +1430,15 @@ try {{ } else { let cmd = if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); + let args = vec!["--preserve-symlinks", script_path.as_str()]; - let mut bun_cmd = Command::new(&*NODE_BIN_PATH); + let mut bun_cmd = build_command_with_isolation(&*NODE_BIN_PATH, &args); bun_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_bun_proc_envs) - .args(vec!["--preserve-symlinks", &script_path]) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1450,8 +1450,7 @@ try {{ } else { let script_path = format!("{job_dir}/wrapper.mjs"); - let mut bun_cmd = Command::new(&*BUN_PATH); - let args = if codebase.is_some() || has_bundle_cache { + let args: Vec<&str> = if codebase.is_some() || has_bundle_cache { vec!["run", &script_path] } else { vec![ @@ -1463,13 +1462,13 @@ try {{ &script_path, ] }; + let mut bun_cmd = build_command_with_isolation(&*BUN_PATH, &args); bun_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_bun_proc_envs) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1480,16 +1479,12 @@ try {{ bun_cmd }; - start_child_process( - cmd, - if annotation.nodejs { - &*NODE_BIN_PATH - } else { - &*BUN_PATH - }, - false, - ) - .await? + let executable = if annotation.nodejs { + &*NODE_BIN_PATH + } else { + &*BUN_PATH + }; + start_child_process(cmd, executable, false).await? }; let stream_notifier = StreamNotifier::new(conn, job); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 4a6edbd55d..2d995ea9bb 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -528,6 +528,7 @@ pub async fn update_worker_ping_for_failed_init_script( memory: None, memory_usage: None, wm_memory_usage: None, + job_isolation: None, ping_type: PingType::InitScript, }, ) @@ -628,6 +629,38 @@ lazy_static! { static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok(); } +pub fn build_command_with_isolation( + program: &str, + args: &[&str], +) -> Command { + use tokio::process::Command; + + if *crate::ENABLE_UNSHARE_PID { + if let Some(unshare_path) = crate::UNSHARE_PATH.as_ref() { + let mut cmd = Command::new(unshare_path); + + let flags = crate::UNSHARE_ISOLATION_FLAGS.as_str(); + for flag in flags.split_whitespace() { + cmd.arg(flag); + } + + cmd.arg("--"); + cmd.arg(program); + cmd.args(args); + cmd + } else { + panic!( + "BUG: ENABLE_UNSHARE_PID is true but UNSHARE_PATH is None. \ + This should have been caught at worker startup." + ); + } + } else { + let mut cmd = Command::new(program); + cmd.args(args); + cmd + } +} + pub async fn start_child_process( cmd: Command, executable: &str, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 51bdf2f8e8..d5a91ab61e 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -27,12 +27,12 @@ use windmill_queue::CanceledBy; #[cfg(feature = "csharp")] use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, }, handle_child::handle_child, - CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, - NUGET_CONFIG, PATH_ENV, TZ_ENV, + CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, + NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, TZ_ENV, }; use crate::common::OccupancyMetrics; @@ -588,7 +588,8 @@ pub async fn handle_csharp_job( } else { format!("{job_dir}/Main.exe") }; - let mut run_csharp = Command::new(&compiled_executable_name); + + let mut run_csharp = build_command_with_isolation(&compiled_executable_name, &[]); run_csharp .current_dir(job_dir) .env_clear() diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index cda1f78fdc..332cd142db 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,7 +7,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -406,14 +406,17 @@ try {{ args.push("-A"); } args.push(&script_path); - let mut deno_cmd = Command::new(DENO_PATH.as_str()); + + let mut deno_cmd = build_command_with_isolation( + DENO_PATH.as_str(), + &args.iter().map(|s| s.as_ref()).collect::>(), + ); deno_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_deno_proc_envs) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -589,7 +592,7 @@ BigInt.prototype.toJSON = function () {{ {dates} -console.log('start\n'); +console.log('start\n'); const decoder = new TextDecoder(); for await (const chunk of Deno.stdin.readable) {{ @@ -601,7 +604,7 @@ for await (const chunk of Deno.stdin.readable) {{ break; }} try {{ - let {{ {spread} }} = JSON.parse(line) + let {{ {spread} }} = JSON.parse(line) {dates} let res: any = await main(...[ {spread} ]); console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 0409ceba95..bf3957c430 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -19,7 +19,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - capitalize, create_args_and_out_file, get_reserved_variables, read_result, + build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -371,10 +371,7 @@ func Run(req Req) (interface{{}}, error){{ #[cfg(windows)] let compiled_executable_name = format!("{}/main.exe", job_dir); - #[cfg(unix)] - let mut run_go = Command::new(&compiled_executable_name); - #[cfg(windows)] - let mut run_go = Command::new(&compiled_executable_name); + let mut run_go = build_command_with_isolation(&compiled_executable_name, &[]); run_go .current_dir(job_dir) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 92ef69da16..b553b9eff1 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -21,13 +21,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child, universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency}, - COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, - MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, + JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; @@ -653,11 +653,13 @@ async fn run<'a>( ) .await; - let mut cmd = Command::new(if cfg!(windows) { + let java_executable = if cfg!(windows) { "java" } else { JAVA_PATH.as_str() - }); + }; + + let mut cmd = build_command_with_isolation(java_executable, &[]); cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) @@ -703,7 +705,7 @@ async fn run<'a>( std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "java", false).await? + start_child_process(cmd, java_executable, false).await? }; handle_child::handle_child( &job.id, @@ -836,7 +838,7 @@ fn wrap(inner_content: &str) -> Result { }) .collect_vec() .join(" "); - Ok(r#" + Ok(r#" package net.script; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.FileInputStream; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 2497757d9d..917c5333ee 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -13,10 +13,11 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, - handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, }; use windmill_common::client::AuthedClient; @@ -181,7 +182,7 @@ fn wrap(inner_content: &str) -> Result { .collect_vec() .join(" "); Ok( - r#" + r#" $env.config.table.mode = 'basic' def nullguard [ name: string ] { @@ -194,11 +195,11 @@ def nullguard [ name: string ] { # TODO: Probably needs rework in order for LSP to work def get_variable [ pat ] { let addr = $"($env.BASE_INTERNAL_URL)/api/w/($env.WM_WORKSPACE)/variables/get_value/($pat)" ; - http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in + http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in } def get_resource [ pat ] { let addr = $"($env.BASE_INTERNAL_URL)/api/w/($env.WM_WORKSPACE)/resources/get_value_interpolated/($pat)" ; - http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in + http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in } def 'main --wrapped' [] { @@ -285,11 +286,14 @@ async fn run<'a>( // let plugin_registry = format!("{job_dir}/plugin-registry"); // File::create(&plugin_registry).await?; // - let mut cmd = Command::new(if cfg!(windows) { + let nu_executable = if cfg!(windows) { "nu" } else { NU_PATH.as_str() - }); + }; + + let args = vec!["main.nu", "--wrapped"]; + let mut cmd = build_command_with_isolation(nu_executable, &args); cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) @@ -297,20 +301,16 @@ async fn run<'a>( .envs(envs) .envs(reserved_variables) .envs(PROXY_ENVS.clone()) - .args(&[ - "main.nu", - "--wrapped", - // TODO(v1): - // "--plugins", - // &format!( - // "[{}]", - // plugins - // .into_iter() - // .map(|pl| format!("{NU_CACHE_DIR}/plugins/bin/nu_plugin_{pl}")) - // .collect_vec() - // .join(",") - // ), - ]) + // TODO(v1): + // "--plugins", + // &format!( + // "[{}]", + // plugins + // .into_iter() + // .map(|pl| format!("{NU_CACHE_DIR}/plugins/bin/nu_plugin_{pl}")) + // .collect_vec() + // .join(",") + // ), .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -324,7 +324,7 @@ async fn run<'a>( std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "nu", false).await? + start_child_process(cmd, nu_executable, false).await? }; handle_child::handle_child( &job.id, diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index e240b5d87b..15cc6ccf30 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -241,7 +241,7 @@ pub async fn do_postgresql( "postgres://{user}:{password}@{host}:{port}/{dbname}?sslmode={sslmode}", user = encode(&database.user.unwrap_or("postgres".to_string())), password = encode(&database.password.unwrap_or("".to_string())), - host = encode(&database.host), + host = database.host, port = database.port.unwrap_or(5432), dbname = database.dbname, sslmode = sslmode diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index b272d631da..4ead742f63 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -16,11 +16,12 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, OccupancyMetrics, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, + NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -309,25 +310,22 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let cmd = { - let script_path = format!("{job_dir}/wrapper.php"); + let script_path = format!("{job_dir}/wrapper.php"); + let args = vec![script_path.as_str()]; - let mut php_cmd = Command::new(&*PHP_PATH); - let args = vec![&script_path]; - php_cmd - .current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - php_cmd - }; - start_child_process(cmd, &*PHP_PATH, false).await? + let mut php_cmd = build_command_with_isolation(&*PHP_PATH, &args); + php_cmd + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .env("BASE_INTERNAL_URL", base_internal_url) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + start_child_process(php_cmd, &*PHP_PATH, false).await? }; handle_child( diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 4a4b85f8ac..b2a81234a3 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -23,12 +23,13 @@ lazy_static::lazy_static! { use crate::{ common::{ - build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, - OccupancyMetrics, + build_args_map, build_command_with_isolation, get_reserved_variables, read_file, + read_file_content, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, - POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, + TZ_ENV, }; fn val_to_pwsh_param(v: serde_json::Value) -> String { @@ -81,14 +82,14 @@ $credentials = $null if ($hasPrivateRepo) { $repoName = "windmill-private-$jobId" $repoUri = "$privateRepoUrl" - + # Create PSCredential for authentication $username = "token" $patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force $credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) - + Write-Host "Registering temporary repository: $repoName" - + # Remove repository if it already exists Unregister-PSResourceRepository -Name $repoName -ErrorAction SilentlyContinue Register-PSResourceRepository -Name $repoName -Uri $repoUri -Trusted @@ -99,7 +100,7 @@ try { foreach ($moduleRequest in $moduleRequests) { $moduleName = $moduleRequest.Name $requiredVersion = $moduleRequest.Version - + # Check if module is already installed with the required version (case-insensitive) $isInstalled = $false if ($requiredVersion) { @@ -107,32 +108,32 @@ try { } else { $isInstalled = $availableModules | Where-Object { $_.Name -eq $moduleName } } - + if (-not $isInstalled) { $moduleFound = $false - + # First try private repository if configured if ($hasPrivateRepo) { $findParams = @{ Name = $moduleName; Repository = $repoName; ErrorAction = 'SilentlyContinue'; Credential = $credentials } if ($requiredVersion) { $findParams.Version = $requiredVersion } - + $privateModule = Find-PSResource @findParams if ($privateModule) { $moduleFound = $true $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } Write-Host "Found module $moduleName$versionInfo in private repository, installing from there..." - + $saveParams = @{ Name = $moduleName; Path = $path; Repository = $repoName; Credential = $credentials } if ($requiredVersion) { $saveParams.Version = $requiredVersion } Save-PSResource @saveParams } } - + # If not found in private repo (or no private repo configured), try all repositories if (-not $moduleFound) { $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } Write-Host "Installing module $moduleName$versionInfo from public repositories..." - + $saveParams = @{ Name = $moduleName; Path = $path; TrustRepository = $true } if ($requiredVersion) { $saveParams.Version = $requiredVersion } Save-PSResource @saveParams @@ -255,20 +256,42 @@ pub async fn handle_powershell_job( job.args.as_ref() }; - let args_owned = windmill_parser_bash::parse_powershell_sig(&content)? + let parsed_sig = windmill_parser_bash::parse_powershell_sig(&content)?; + + parsed_sig .args .iter() - .map(|arg| { - ( - arg.name.clone(), - job_args.and_then(|x| x.get(&arg.name).map(|x| raw_to_pwsh_param(x.get()))), - ) - }) - .collect::)>>(); + .filter_map(|arg| { + let value_opt = job_args.and_then(|x| x.get(&arg.name)); - args_owned - .into_iter() - .filter_map(|(n, v)| v.map(|v| format!("-{n} {v}"))) + // Check if this is a switch parameter (only [switch], not [bool]) + let is_switch = arg.otyp.as_ref().map(|t| { + t.to_lowercase() == "switch" + }).unwrap_or(false); + + if is_switch { + // Handle switch parameters: -SwitchName or omit + if let Some(value) = value_opt { + match serde_json::from_str::(value.get()) { + Ok(serde_json::Value::Bool(true)) => { + // Switch is enabled: just pass -SwitchName + Some(format!("-{}", arg.name)) + } + Ok(serde_json::Value::Bool(false)) | _ => { + // Switch is disabled or invalid: omit the parameter + None + } + } + } else { + // No value provided, omit the switch (defaults to false) + None + } + } else { + // Regular parameter (including [bool]): format as -ParamName Value + // For [bool] parameters, this will be -ParamName $true or -ParamName $false + value_opt.map(|v| format!("-{} {}", arg.name, raw_to_pwsh_param(v.get()))) + } + }) .collect::>() .join(" ") }; @@ -425,15 +448,16 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", exit 1\n\ }\n"; - // make sure param() is first - let param_match = windmill_parser_bash::extract_powershell_param_block(&content, true); - let content: String = if let Some(param_match) = param_match { + // make sure param() with its attributes is first + let content: String = if let Some((param_block, remaining_code)) = + windmill_parser_bash::extract_powershell_param_block_with_attributes(&content, true) + { format!( "{}\n{}\n{}\n{}\n{}", - param_match, + param_block, profile, strict_termination_start, - content.replace(param_match, ""), + remaining_code, strict_termination_end ) } else { @@ -503,7 +527,6 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut cmd = Command::new(POWERSHELL_PATH.as_str()); let cmd_args; #[cfg(unix)] @@ -516,6 +539,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", cmd_args = vec![r".\wrapper.ps1"]; } + let mut cmd = build_command_with_isolation(POWERSHELL_PATH.as_str(), &cmd_args); + cmd.current_dir(job_dir) .env_clear() .envs(envs) @@ -524,7 +549,6 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(&cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 7ae80a97b6..bc84a2e19f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -120,7 +120,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_file, read_result, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -835,9 +835,12 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut python_cmd = Command::new(&python_path); - let args = vec!["-u", "-m", "wrapper"]; + + let mut python_cmd = build_command_with_isolation( + &python_path, + &args, + ); python_cmd .current_dir(job_dir) .env_clear() @@ -847,7 +850,6 @@ mount {{ .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index cd25663889..0039e6e8c2 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,12 +23,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child::{self}, universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, - DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, + DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + RUBY_CACHE_DIR, RUBY_REPOS, }; lazy_static::lazy_static! { static ref RUBY_CONCURRENT_DOWNLOADS: usize = std::env::var("RUBY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); @@ -140,7 +141,7 @@ pub async fn prepare<'a>( File::create(format!("{}/inline.rb", &mini_wm_path)) .await? .write_all(&wrap( - r#" + r#" class GemfileProxy def initialize @gem_calls = [] @@ -213,7 +214,7 @@ end .await? .write_all( &wrap( - r##" + r##" require 'net/http' require 'uri' require 'json' @@ -223,17 +224,17 @@ def get_variable(path) base_url = ENV['BASE_INTERNAL_URL'] workspace = ENV['WM_WORKSPACE'] token = ENV['WM_TOKEN'] - + uri = URI("#{base_url}/api/w/#{workspace}/variables/get_value/#{path}") - + http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' - + request = Net::HTTP::Get.new(uri) request['Authorization'] = "Bearer #{token}" - + response = http.request(request) - + if response.code == '200' JSON.parse(response.body) else @@ -245,17 +246,17 @@ def get_resource(path) base_url = ENV['BASE_INTERNAL_URL'] workspace = ENV['WM_WORKSPACE'] token = ENV['WM_TOKEN'] - + uri = URI("#{base_url}/api/w/#{workspace}/resources/get_value_interpolated/#{path}") - + http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' - + request = Net::HTTP::Get.new(uri) request['Authorization'] = "Bearer #{token}" - + response = http.request(request) - + if response.code == '200' JSON.parse(response.body) else @@ -823,11 +824,14 @@ mount {{ ) .await; - let mut cmd = Command::new(if cfg!(windows) { + let ruby_executable = if cfg!(windows) { "ruby.exe" } else { RUBY_PATH.as_str() - }); + }; + + let args = vec!["main.rb"]; + let mut cmd = build_command_with_isolation(ruby_executable, &args); #[cfg(windows)] let rubylib = rubylib.replace(":", ";"); @@ -842,8 +846,7 @@ mount {{ .envs(PROXY_ENVS.clone()) .envs(envs); - cmd.args(&["main.rb"]) - .stdin(Stdio::null()) + cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -856,7 +859,7 @@ mount {{ std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "ruby", false).await? + start_child_process(cmd, ruby_executable, false).await? }; handle_child::handle_child( &job.id, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 6a05de459c..c43a7d2d97 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,7 +19,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -567,7 +567,7 @@ pub async fn handle_rust_job( start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let compiled_executable_name = "./main"; - let mut run_rust = Command::new(compiled_executable_name); + let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) .env_clear() diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e58d521a5d..9b47aa5113 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -303,6 +303,128 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(true); + pub static ref ENABLE_UNSHARE_PID: bool = std::env::var("ENABLE_UNSHARE_PID") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + + pub static ref UNSHARE_ISOLATION_FLAGS: String = { + std::env::var("UNSHARE_ISOLATION_FLAGS") + .unwrap_or_else(|_| "--user --map-root-user --pid --fork --mount-proc".to_string()) + }; + + pub static ref UNSHARE_PATH: Option = { + let flags = UNSHARE_ISOLATION_FLAGS.as_str(); + let mut test_cmd_args: Vec<&str> = flags.split_whitespace().collect(); + test_cmd_args.push("--"); + test_cmd_args.push("true"); + + let test_result = std::process::Command::new("unshare") + .args(&test_cmd_args) + .output(); + + match test_result { + Ok(output) if output.status.success() => { + tracing::info!("PID namespace isolation enabled. Flags: {}", flags); + Some("unshare".to_string()) + }, + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + + if *ENABLE_UNSHARE_PID { + panic!( + "ENABLE_UNSHARE_PID is set but unshare test failed.\n\ + Error: {}\n\ + Flags: {}\n\ + \n\ + Solutions:\n\ + • Check if user namespaces are enabled: 'sysctl kernel.unprivileged_userns_clone'\n\ + • For Docker: Requires 'privileged: true' in docker-compose for --mount-proc flag\n\ + • For Kubernetes: Requires 'privileged: true' in securityContext for --mount-proc flag\n\ + • Try different flags via UNSHARE_ISOLATION_FLAGS env var (remove --mount-proc if privileged mode not possible)\n\ + • Alternative: Use NSJAIL instead\n\ + • Disable: Set ENABLE_UNSHARE_PID=false", + stderr.trim(), + flags + ); + } + + tracing::warn!( + "unshare test failed: {}. Flags: {}. Set ENABLE_UNSHARE_PID=true to fail on error.", + stderr.trim(), + flags + ); + None + }, + Err(e) => { + if *ENABLE_UNSHARE_PID { + if e.kind() == std::io::ErrorKind::NotFound { + panic!( + "ENABLE_UNSHARE_PID is set but unshare binary not found.\n\ + Install util-linux package or set ENABLE_UNSHARE_PID=false" + ); + } else { + panic!( + "ENABLE_UNSHARE_PID is set but failed to test unshare: {}", + e + ); + } + } + + if e.kind() == std::io::ErrorKind::NotFound { + tracing::debug!("unshare binary not found"); + } else { + tracing::warn!("Failed to test unshare: {}", e); + } + None + } + } + }; + + pub static ref NSJAIL_AVAILABLE: Option = { + if *DISABLE_NSJAIL { + None + } else { + let nsjail_path = NSJAIL_PATH.as_str(); + + let test_result = std::process::Command::new(nsjail_path) + .arg("--help") + .output(); + + match test_result { + Ok(output) if output.status.success() => { + tracing::info!("NSJAIL sandboxing available at: {}", nsjail_path); + Some(nsjail_path.to_string()) + }, + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + tracing::warn!( + "nsjail test failed: {}. Jobs will run without nsjail sandboxing. \ + To enable nsjail: install nsjail binary or use windmill image with -nsjail suffix", + stderr.trim() + ); + None + }, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + tracing::warn!( + "nsjail not found at '{}'. Jobs will run without nsjail sandboxing. \ + To enable nsjail: install nsjail binary or use windmill image with -nsjail suffix", + nsjail_path + ); + } else { + tracing::warn!( + "Failed to test nsjail at '{}': {}. Jobs will run without nsjail sandboxing.", + nsjail_path, + e + ); + } + None + } + } + } + }; + pub static ref KEEP_JOB_DIR: AtomicBool = AtomicBool::new(std::env::var("KEEP_JOB_DIR") .ok() .and_then(|x| x.parse::().ok()) @@ -981,6 +1103,19 @@ pub async fn run_worker( ); } + // Force UNSHARE_PATH initialization now to fail-fast if unshare doesn't work + // This ensures we panic at startup rather than lazily when first accessed during job execution + if *ENABLE_UNSHARE_PID { + // Access UNSHARE_PATH to trigger lazy_static initialization and test + let _ = &*UNSHARE_PATH; + + tracing::info!( + worker = %worker_name, hostname = %hostname, + "PID namespace isolation enabled via unshare with flags: {}", + UNSHARE_ISOLATION_FLAGS.as_str() + ); + } + let start_time = Instant::now(); let worker_dir = format!("{TMP_DIR}/{worker_name}"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 0b840884f5..824a45c661 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -9,6 +9,7 @@ use crate::scoped_dependency_map::ScopedDependencyMap; use async_recursion::async_recursion; use chrono::{Duration, Utc}; use itertools::Itertools; +use serde::Serialize; use serde_json::value::RawValue; use serde_json::{from_value, json, Value}; use sha2::Digest; @@ -765,15 +766,16 @@ pub async fn handle_flow_dependency_job( // `JobKind::FlowDependencies` job store either: // - A saved flow version `id` in the `script_hash` column. // - Preview raw flow in the `queue` or `job` table. - let mut flow = match job.runnable_id { - Some(ScriptHash(id)) => cache::flow::fetch_version(db, id).await?, + let (mut flow, notes) = match job.runnable_id { + Some(ScriptHash(id)) => { + let flow = cache::flow::fetch_version(db, id).await?; + (flow.value().clone(), flow.notes()) + } _ => match preview_data { - Some(RawData::Flow(data)) => data.clone(), + Some(RawData::Flow(data)) => (data.value().clone(), data.notes()), _ => return Err(Error::internal_err("expected script hash")), }, - } - .value() - .clone(); + }; let mut tx = db.begin().await?; @@ -861,7 +863,22 @@ pub async fn handle_flow_dependency_job( .await?; } - let new_flow_value = Json(serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?); + #[derive(Debug, Clone, Serialize)] + struct FlowValueWithNotes<'a> { + #[serde(flatten)] + value: &'a FlowValue, + + #[serde(skip_serializing_if = "Option::is_none")] + notes: Option>, // TODO: Make this a Vec + } + + let new_flow_value = Json( + serde_json::value::to_raw_value(&FlowValueWithNotes { + value: &flow, + notes: notes.and_then(|n| n.notes).map(|n| n.into()), + }) + .map_err(to_anyhow)?, + ); // Re-check cancellation to ensure we don't accidentally override a flow. if sqlx::query_scalar!( diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 366cc9cdf7..f105d8c415 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -143,6 +143,7 @@ async fn update_worker_ping_full_inner( memory: memory, memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), + job_isolation: None, ping_type: PingType::MainLoop, }, ) @@ -171,6 +172,15 @@ pub async fn insert_ping( let vcpus = get_vcpus(); let memory = get_memory(); + // Determine job isolation method + let job_isolation = if crate::NSJAIL_AVAILABLE.is_some() { + Some("nsjail".to_string()) + } else if *crate::ENABLE_UNSHARE_PID && crate::UNSHARE_PATH.is_some() { + Some("unshare".to_string()) + } else { + Some("none".to_string()) + }; + match db { Connection::Sql(db) => { insert_ping_query( @@ -183,6 +193,7 @@ pub async fn insert_ping( windmill_common::utils::GIT_VERSION, vcpus, memory, + job_isolation, db, ) .await?; @@ -209,6 +220,7 @@ pub async fn insert_ping( memory: memory, memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), + job_isolation, ping_type: PingType::Initial, }, ) @@ -231,6 +243,15 @@ pub async fn update_worker_ping_from_job( let occupancy_rate_15s = occupancy.as_ref().and_then(|x| x.occupancy_rate_15s); let occupancy_rate_5m = occupancy.as_ref().and_then(|x| x.occupancy_rate_5m); let occupancy_rate_30m = occupancy.as_ref().and_then(|x| x.occupancy_rate_30m); + + let job_isolation = if crate::NSJAIL_AVAILABLE.is_some() { + Some("nsjail".to_string()) + } else if *crate::ENABLE_UNSHARE_PID && crate::UNSHARE_PATH.is_some() { + Some("unshare".to_string()) + } else { + Some("none".to_string()) + }; + match conn.clone() { Connection::Sql(ref db) => { update_worker_ping_from_job_query( @@ -243,6 +264,7 @@ pub async fn update_worker_ping_from_job( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + job_isolation, db, ) .await?; @@ -270,6 +292,7 @@ pub async fn update_worker_ping_from_job( occupancy_rate_15s: occupancy_rate_15s, occupancy_rate_5m: occupancy_rate_5m, occupancy_rate_30m: occupancy_rate_30m, + job_isolation, }, ) .await?; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d5d3bd172f..3592faccdd 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.575.4"; +export const VERSION = "v1.582.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/install_dev.sh b/cli/install_dev.sh new file mode 100755 index 0000000000..e172ac1288 --- /dev/null +++ b/cli/install_dev.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -e + +if [ -z "$1" ]; then + name="wmill" +else + name="$1" +fi + +./gen_wm_client.sh +./windmill-utils-internal/gen_wm_client.sh + +echo "Installing dev cli as $name (pass arg to override)" +deno install -f -A -g src/main.ts --name $name --unstable \ No newline at end of file diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 429fbd6ebc..0c64a0d857 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -661,29 +661,6 @@ export async function elementsToMap( const map: { [key: string]: string } = {}; const processedBasePaths = new Set(); - // First pass: collect all file paths to identify branch-specific files - const allPaths: string[] = []; - for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { - if (!entry.isDirectory && !entry.ignored) { - allPaths.push(entry.path); - } - } - - const branchSpecificExists = new Set(); - - if (specificItems) { - const currentBranch = getCurrentGitBranch(); - if (currentBranch) { - for (const path of allPaths) { - if (isCurrentBranchFile(path)) { - const basePath = fromBranchSpecificPath(path, currentBranch); - if (isSpecificItem(basePath, specificItems)) { - branchSpecificExists.add(basePath); - } - } - } - } - } for await (const entry of readDirRecursiveWithIgnore(ignore, els)) { if (entry.isDirectory || entry.ignored) continue; diff --git a/cli/src/main.ts b/cli/src/main.ts index 6dba2bce7a..157eda767a 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.575.4"; +export const VERSION = "1.582.2"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/wasm/regex/windmill_parser_wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm.d.ts index 59f34f2b18..c40f0bf2a5 100644 --- a/cli/wasm/regex/windmill_parser_wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm.d.ts @@ -1,14 +1,14 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_assets_sql(code: string): string; -export function parse_db_resource(code: string): string | undefined; -export function parse_bash(code: string): string; -export function parse_mssql(code: string): string; -export function parse_oracledb(code: string): string; export function parse_powershell(code: string): string; -export function parse_bigquery(code: string): string; -export function parse_graphql(code: string): string; +export function parse_assets_sql(code: string): string; export function parse_snowflake(code: string): string; -export function parse_duckdb(code: string): string; +export function parse_db_resource(code: string): string | undefined; +export function parse_graphql(code: string): string; +export function parse_mssql(code: string): string; export function parse_sql(code: string): string; +export function parse_oracledb(code: string): string; +export function parse_bigquery(code: string): string; export function parse_mysql(code: string): string; +export function parse_bash(code: string): string; +export function parse_duckdb(code: string): string; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index 25a60a9b8e..580d3b0334 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -68,6 +68,25 @@ function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return decodeText(ptr, len); } +/** + * @param {string} code + * @returns {string} + */ +export function parse_powershell(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_powershell(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + /** * @param {string} code * @returns {string} @@ -87,6 +106,25 @@ export function parse_assets_sql(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_snowflake(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_snowflake(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + /** * @param {string} code * @returns {string | undefined} @@ -107,13 +145,13 @@ export function parse_db_resource(code) { * @param {string} code * @returns {string} */ -export function parse_bash(code) { +export function parse_graphql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bash(ptr0, len0); + const ret = wasm.parse_graphql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -145,13 +183,13 @@ export function parse_mssql(code) { * @param {string} code * @returns {string} */ -export function parse_oracledb(code) { +export function parse_sql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_oracledb(ptr0, len0); + const ret = wasm.parse_sql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -164,13 +202,13 @@ export function parse_oracledb(code) { * @param {string} code * @returns {string} */ -export function parse_powershell(code) { +export function parse_oracledb(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_powershell(ptr0, len0); + const ret = wasm.parse_oracledb(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -202,13 +240,13 @@ export function parse_bigquery(code) { * @param {string} code * @returns {string} */ -export function parse_graphql(code) { +export function parse_mysql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_graphql(ptr0, len0); + const ret = wasm.parse_mysql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -221,13 +259,13 @@ export function parse_graphql(code) { * @param {string} code * @returns {string} */ -export function parse_snowflake(code) { +export function parse_bash(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_snowflake(ptr0, len0); + const ret = wasm.parse_bash(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -255,44 +293,6 @@ export function parse_duckdb(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_sql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_sql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_mysql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mysql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - const imports = { __wbindgen_placeholder__: { __wbindgen_init_externref_table: function() { diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm b/cli/wasm/regex/windmill_parser_wasm_bg.wasm index 45a144132d..153645bf9c 100644 Binary files a/cli/wasm/regex/windmill_parser_wasm_bg.wasm and b/cli/wasm/regex/windmill_parser_wasm_bg.wasm differ diff --git a/docker-compose.yml b/docker-compose.yml index ea67377ffc..a6d96aaec9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,10 +60,16 @@ services: memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped + # Uncomment to enable PID namespace isolation (recommended for security) + # Requires privileged mode for --mount-proc flag + # See: https://www.windmill.dev/docs/advanced/security_isolation + # privileged: true environment: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=default + # Uncomment to enable PID namespace isolation (requires privileged: true above) + # - ENABLE_UNSHARE_PID=true depends_on: db: condition: service_healthy @@ -89,12 +95,18 @@ services: memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped + # Uncomment to enable PID namespace isolation (recommended for security) + # Requires privileged mode for --mount-proc flag + # See: https://www.windmill.dev/docs/advanced/security_isolation + # privileged: true environment: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=native - NUM_WORKERS=8 - SLEEP_QUEUE=200 + # Uncomment to enable PID namespace isolation (requires privileged: true above) + # - ENABLE_UNSHARE_PID=true depends_on: db: condition: service_healthy @@ -113,10 +125,16 @@ services: # memory: 2048M # # for GB, use syntax '2Gi' # restart: unless-stopped + # # Uncomment to enable PID namespace isolation (recommended for security) + # # Requires privileged mode for --mount-proc flag + # # See: https://www.windmill.dev/docs/advanced/security_isolation + # # privileged: true # environment: # - DATABASE_URL=${DATABASE_URL} # - MODE=worker # - WORKER_GROUP=reports + # # Uncomment to enable PID namespace isolation (requires privileged: true above) + # # - ENABLE_UNSHARE_PID=true # depends_on: # db: # condition: service_healthy diff --git a/docker/RHEL8/Dockerfile b/docker/RHEL8/Dockerfile new file mode 100644 index 0000000000..c8c4fa0588 --- /dev/null +++ b/docker/RHEL8/Dockerfile @@ -0,0 +1,86 @@ +ARG DEBIAN_IMAGE=debian:bookworm-slim +ARG RUST_IMAGE=registry.access.redhat.com/ubi8/ubi:latest +ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm + +FROM ${RUST_IMAGE} AS rust_base + +RUN yum update -y && \ + yum install -y git openssl-devel npm nodejs rustfmt + +# Install rust manually +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1 + +WORKDIR /windmill + +ENV SQLX_OFFLINE=true +# ENV CARGO_INCREMENTAL=1 + +FROM node:20-alpine as frontend + +# install dependencies +WORKDIR /frontend +COPY ./frontend/package.json ./frontend/package-lock.json ./ +RUN npm ci + +# Copy all local files into the image. +COPY frontend . +RUN mkdir /backend +COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml +COPY /openflow.openapi.yaml /openflow.openapi.yaml +COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh + +RUN cd /backend/windmill-api && . ./build_openapi.sh +COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ +COPY /typescript-client/docs/ /frontend/static/tsdocs/ + +RUN npm run generate-backend-client +ENV NODE_OPTIONS "--max-old-space-size=10240" +RUN npm run build + + +FROM rust_base AS planner + +COPY ./openflow.openapi.yaml /openflow.openapi.yaml +COPY ./backend ./ + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + CARGO_NET_GIT_FETCH_WITH_CLI=true cargo chef prepare --recipe-path recipe.json + +FROM rust_base AS builder +ARG features="" + +COPY --from=planner /windmill/recipe.json recipe.json + +RUN --mount=type=secret,id=rh_username \ + --mount=type=secret,id=rh_password \ + subscription-manager register --username $(cat /run/secrets/rh_username) --password $(cat /run/secrets/rh_password) + +RUN subscription-manager repos --enable codeready-builder-for-rhel-8-$(arch)-rpms + +RUN yum update -y && \ + yum install -y perl-interpreter perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel clang llvm-devel cmake libtool-ltdl-devel + +# RUN --mount=type=cache,target=/usr/local/cargo/registry \ +# CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json + +COPY ./openflow.openapi.yaml /openflow.openapi.yaml +COPY ./backend ./ + +COPY --from=frontend /frontend /frontend +COPY --from=frontend /backend/windmill-api/openapi-deref.yaml ./windmill-api/openapi-deref.yaml +COPY .git/ .git/ + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features" + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + cd windmill-duckdb-ffi-internal && \ + CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release + +RUN mkdir -p /usr/src/app && \ + cp windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so /usr/src/app/ + +RUN subscription-manager unregister diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index 7399fd690a..098091ef17 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -61,7 +61,7 @@ RUN --mount=type=secret,id=rh_username \ RUN subscription-manager repos --enable codeready-builder-for-rhel-9-$(arch)-rpms RUN yum update -y && \ - yum install -y perl-FindBin perl-IPC-Cmd libxml2-devel xmlsec1-devel xmlsec1-openssl-devel clang llvm-devel cmake libtool-ltdl-devel + yum install -y perl-FindBin perl-IPC-Cmd perl-Time-Piece libxml2-devel xmlsec1-devel xmlsec1-openssl-devel clang llvm-devel cmake libtool-ltdl-devel # RUN --mount=type=cache,target=/usr/local/cargo/registry \ # CARGO_NET_GIT_FETCH_WITH_CLI=true RUST_BACKTRACE=1 cargo chef cook --release --features "$features" --recipe-path recipe.json @@ -76,4 +76,11 @@ COPY .git/ .git/ RUN --mount=type=cache,target=/usr/local/cargo/registry \ CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release --features "$features" +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + cd windmill-duckdb-ffi-internal && \ + CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --release + +RUN mkdir -p /usr/src/app && \ + cp windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so /usr/src/app/ + RUN subscription-manager unregister diff --git a/flake.lock b/flake.lock index 019aff4b34..57fa428e72 100644 --- a/flake.lock +++ b/flake.lock @@ -35,11 +35,11 @@ }, "nixpkgs-claude": { "locked": { - "lastModified": 1753694789, - "narHash": "sha256-cKgvtz6fKuK1Xr5LQW/zOUiAC0oSQoA9nOISB0pJZqM=", + "lastModified": 1763421233, + "narHash": "sha256-Stk9ZYRkGrnnpyJ4eqt9eQtdFWRRIvMxpNRf4sIegnw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "dc9637876d0dcc8c9e5e22986b857632effeb727", + "rev": "89c2b2330e733d6cdb5eae7b899326930c2c0648", "type": "github" }, "original": { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2c35d77034..50a0547dfa 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.575.4", + "version": "1.582.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.575.4", + "version": "1.582.2", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -55,7 +55,7 @@ "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^5.16.0", + "openai": "^6.9.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -79,7 +79,7 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.574.1", + "windmill-parser-wasm-regex": "1.576.3", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", @@ -9752,16 +9752,16 @@ } }, "node_modules/openai": { - "version": "5.23.2", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz", - "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==", + "version": "6.9.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.9.0.tgz", + "integrity": "sha512-n2sJRYmM+xfJ0l3OfH8eNnIyv3nQY7L08gZQu3dw6wSdfPtKAk92L83M2NIP5SS8Cl/bsBBG3yKzEOjkx0O+7A==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" }, "peerDependencies": { "ws": "^8.18.0", - "zod": "^3.23.8" + "zod": "^3.25 || ^4.0" }, "peerDependenciesMeta": { "ws": { @@ -13759,9 +13759,9 @@ "integrity": "sha512-s+bdIgT/fA5em3zYUwF8D14uA/dZh7iu0krZYZQqZUO7txN37hwSCVfovbMkIwm4zPbsJ50mU8DRLt7UpAPZIw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.574.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.574.1.tgz", - "integrity": "sha512-KnNBnpTGcnBPSzTQHdsjTPJbRQ81iem98eUnMGO8Zt1qgBFJ/eFZPwmUg6TZMbMITzTi0QqRCJj08eQID1FcPg==" + "version": "1.576.3", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.576.3.tgz", + "integrity": "sha512-+5or2Gzi2W+DXU3cDqUB2sn3t3t1ZJ5BGpEh94r30fbm9yGxSy9QvxsURqv59ubKCjmm3oKwqApSyn+OqXSLjg==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/frontend/package.json b/frontend/package.json index 7bd2758467..72ec938dab 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.575.4", + "version": "1.582.2", "scripts": { "dev": "vite dev", "build": "vite build", @@ -120,7 +120,7 @@ "monaco-languageclient": "10.1.0", "monaco-vim": "^0.4.1", "ol": "^7.4.0", - "openai": "^5.16.0", + "openai": "^6.9.0", "openapi-types": "^12.1.3", "p-limit": "^6.1.0", "panzoom": "^9.4.3", @@ -144,7 +144,7 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.574.1", + "windmill-parser-wasm-regex": "1.576.3", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", diff --git a/frontend/src/lib/assets/tokens/tokens.json b/frontend/src/lib/assets/tokens/tokens.json index f04c2432f3..d4749e0e2d 100644 --- a/frontend/src/lib/assets/tokens/tokens.json +++ b/frontend/src/lib/assets/tokens/tokens.json @@ -17,7 +17,7 @@ "border-light": "#e5e7eb", "border-normal": "#9ca3af", "border-accent": "#2c5beb", - "surface-accent-selected": "#bfdbfe4c", + "surface-accent-selected": "#ebefff", "surface-accent-secondary": "#293676", "surface-tertiary": "#ffffff", "text-emphasis": "#1d2430", @@ -52,7 +52,7 @@ "border-light": "#485971", "border-normal": "#718096", "border-accent": "#a0affa", - "surface-accent-selected": "#6790c34c", + "surface-accent-selected": "#33384e", "surface-accent-secondary": "#e8ebfb", "surface-tertiary": "#434c5e", "text-emphasis": "#f3f4f6", @@ -87,7 +87,7 @@ "border-light": "#374457", "border-normal": "#a9b0ba", "border-accent": "#a0affa", - "surface-accent-selected": "#6790c44c", + "surface-accent-selected": "#33384e", "surface-accent-secondary": "#e8ebfb", "surface-tertiary": "#353c4a", "text-emphasis": "#eeeff2", @@ -193,7 +193,40 @@ "purple-800": "#483c60", "purple-900": "#3a3549", "purple-950": "#31313f", - "blue-950": "#213263" + "blue-950": "#213263", + "pink-50": "#fdf2f8", + "pink-100": "#fce7f3", + "pink-200": "#fbcfe8", + "pink-300": "#f9a8d4", + "pink-400": "#f472b6", + "pink-500": "#cc4e8c", + "pink-600": "#af4677", + "pink-700": "#8e4266", + "pink-800": "#5f3e52", + "pink-900": "#473340", + "pink-950": "#372b36", + "lime-50": "#f7fee7", + "lime-100": "#ecfccb", + "lime-200": "#d9f99d", + "lime-300": "#bef264", + "lime-400": "#a3e635", + "lime-500": "#84cc16", + "lime-600": "#5d8f16", + "lime-700": "#527029", + "lime-800": "#415824", + "lime-900": "#324220", + "lime-950": "#232f16", + "yellow-50": "#fefce8", + "yellow-100": "#fef9c3", + "yellow-200": "#fef08a", + "yellow-300": "#fde047", + "yellow-400": "#facc15", + "yellow-500": "#e0ae12", + "yellow-600": "#b1882e", + "yellow-700": "#8a6e31", + "yellow-800": "#61512d", + "yellow-900": "#443d22", + "yellow-950": "#3a351a" } }, "guidelines": { "mode-1": { "blue": "#5e81ac", "demo-background": "#ffffff00" } }, diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index b63d3798e5..f912fad49f 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -28,7 +28,6 @@ import DateTimeInput from './DateTimeInput.svelte' import DateInput from './DateInput.svelte' import CurrencyInput from './apps/components/inputs/currency/CurrencyInput.svelte' - import autosize from '$lib/autosize' import PasswordArgInput from './PasswordArgInput.svelte' import Password from './Password.svelte' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' @@ -46,11 +45,7 @@ import { workspaceStore } from '$lib/stores' import { getJsonSchemaFromResource } from './schema/jsonSchemaResource.svelte' import AIProviderPicker from './AIProviderPicker.svelte' - import TextInput, { - inputBaseClass, - inputBorderClass, - inputSizeClasses - } from './text_input/TextInput.svelte' + import TextInput from './text_input/TextInput.svelte' import FileInput from './common/fileInput/FileInput.svelte' interface Props { @@ -1412,45 +1407,39 @@ {/if} {:else} {#key extra?.['minRows']} - + {error} + unifiedHeight={false} + underlyingInputEl="textarea" + /> {/key} {/if} {#if !disabled && itemPicker && extra?.['disableVariablePicker'] != true} - - + /> {/if} {@render variableInput()} diff --git a/frontend/src/lib/components/Dev.svelte b/frontend/src/lib/components/Dev.svelte index 86ded3bc0d..505002cd54 100644 --- a/frontend/src/lib/components/Dev.svelte +++ b/frontend/src/lib/components/Dev.svelte @@ -31,6 +31,8 @@ import type { FlowState } from './flows/flowState' import { initHistory } from '$lib/history.svelte' import type { FlowEditorContext, FlowInput, FlowInputEditorState } from './flows/types' + import { SelectionManager } from './graph/selectionUtils.svelte' + import { NoteEditor, setNoteEditorContext } from './graph/noteEditor.svelte' import { dfs } from './flows/dfs' import { loadSchemaFromModule } from './flows/flowInfers' import { CornerDownLeft, Play } from 'lucide-svelte' @@ -475,7 +477,7 @@ let ids = dfs(flowStore.val.value.modules ?? [], (m) => m.id) flowStateStore.val = Object.fromEntries(ids.map((k) => [k, {}])) } catch (e) {} - inferModuleArgs($selectedIdStore) + inferModuleArgs(selectedId) } } catch (e) { console.error('issue setting new flowstore', e) @@ -489,7 +491,8 @@ const moving = writable<{ id: string } | undefined>(undefined) const history = initHistory(flowStore.val) const stepsInputArgs = new StepsInputArgs() - const selectedIdStore = writable('settings-metadata') + const selectionManager = new SelectionManager() + selectionManager.selectId('settings-metadata') const triggersCount = writable(undefined) const modulesTestStates = new ModulesTestStates((moduleId) => { // console.log('FOO') @@ -508,7 +511,7 @@ let pathStore = writable('') let initialPathStore = writable('') setContext('FlowEditorContext', { - selectedId: selectedIdStore, + selectionManager, previewArgs: previewArgsStore, scriptEditorDrawer, moving, @@ -538,6 +541,13 @@ pickablePropertiesFiltered: writable(undefined) }) + // Set up NoteEditor context for note editing capabilities + const noteEditor = new NoteEditor(flowStore, () => { + // Enable notes display when a note is created + flowModuleSchemaMap?.enableNotes?.() + }) + setNoteEditorContext(noteEditor) + let lastSent: OpenFlow | undefined = undefined function updateFlow(flow: OpenFlow) { if (lockChanges) { @@ -618,7 +628,7 @@ flowStore.val && untrack(() => updateFlow(flowStore.val)) }) $effect(() => { - $selectedIdStore && untrack(() => inferModuleArgs($selectedIdStore)) + selectedId && untrack(() => inferModuleArgs(selectedId)) }) let localModuleStates: Record = $state({}) @@ -640,7 +650,7 @@ job.success && flowPreviewButtons?.getPreviewMode() === 'whole' ) { - if (flowModuleSchemaMap?.isNodeVisible('result') && $selectedIdStore !== 'Result') { + if (flowModuleSchemaMap?.isNodeVisible('Result') && selectedId !== 'Result') { outputPickerOpenFns['Result']?.() } } else { @@ -665,6 +675,8 @@ } const flowHasChanged = $derived(flowPreviewContent?.flowHasChanged()) + + const selectedId = $derived(selectionManager.getSelectedId()) @@ -846,7 +858,7 @@ on:applyArgs={(ev) => { if (ev.detail.kind === 'preprocessor') { stepsInputArgs.setStepArgs('preprocessor', ev.detail.args ?? {}) - $selectedIdStore = 'preprocessor' + selectionManager.selectId('preprocessor') } else { previewArgsStore.val = ev.detail.args ?? {} flowPreviewButtons?.openPreview() diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 2ed847bff4..6367779894 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -43,6 +43,9 @@ import FlowImportExportMenu from './flows/header/FlowImportExportMenu.svelte' import FlowPreviewButtons from './flows/header/FlowPreviewButtons.svelte' import type { FlowEditorContext, FlowInput, FlowInputEditorState } from './flows/types' + import { SelectionManager } from './graph/selectionUtils.svelte' + import { NoteEditor } from './graph/noteEditor.svelte' + import { setNoteEditorContext } from './graph/noteEditor.svelte' import { cleanFlow } from './flows/utils.svelte' import { Calendar, @@ -338,11 +341,11 @@ let savedAtNewPath = false if (newFlow) { - onSaveInitial?.({ path: $pathStore, id: getSelectedId() }) + onSaveInitial?.({ path: $pathStore, id: getSelectedId() ?? 'settings' }) } else if (savedFlow?.draft_only && $pathStore !== initialPath) { savedAtNewPath = true initialPath = $pathStore - onSaveDraftOnlyAtNewPath?.({ path: $pathStore, selectedId: getSelectedId() }) + onSaveDraftOnlyAtNewPath?.({ path: $pathStore, selectedId: getSelectedId() ?? 'settings' }) // this is so we can use the flow builder outside of sveltekit } onSaveDraft?.({ path: $pathStore, savedAtNewPath, newFlow }) @@ -561,7 +564,7 @@ encodeState({ flow: flowStore.val, path: $pathStore, - selectedId: $selectedIdStore, + selectedId: selectedIdStore, draft_triggers: triggersState.getDraftTriggersSnapshot(), selected_trigger: triggersState.getSelectedTriggerSnapshot(), loadedFromHistory: { @@ -576,10 +579,17 @@ }, 500) } - const selectedIdStore = writable(selectedId ?? 'settings-metadata') + const selectionManager = new SelectionManager() + const selectedIdStore = $derived(selectionManager.getSelectedId()) + // Initialize with selected id if provided + if (selectedId) { + selectionManager.selectId(selectedId) + } else { + selectionManager.selectId('settings-metadata') + } export function getSelectedId() { - return $selectedIdStore + return selectedIdStore } const previewArgsStore = $state({ val: initialArgs }) @@ -598,7 +608,7 @@ const stepsInputArgs = new StepsInputArgs() function select(selectedId: string) { - selectedIdStore.set(selectedId) + selectionManager.selectId(selectedId) } let insertButtonOpen = writable(false) @@ -607,7 +617,7 @@ let flowEditor: FlowEditor | undefined = $state(undefined) setContext('FlowEditorContext', { - selectedId: selectedIdStore, + selectionManager, currentEditor: writable(undefined), previewArgs: previewArgsStore, scriptEditorDrawer, @@ -629,6 +639,13 @@ outputPickerOpenFns }) + // Set up NoteEditor context for note editing capabilities + const noteEditor = new NoteEditor(flowStore, () => { + // Enable notes display when a note is created + flowEditor?.enableNotes?.() + }) + setNoteEditorContext(noteEditor) + setContext( 'FlowGraphAssetContext', initFlowGraphAssetsCtx({ getModules: () => flowStore.val.value.modules }) @@ -695,7 +712,7 @@ case 'z': if (event.ctrlKey || event.metaKey) { flowStore.val = undo(history, flowStore.val) - $selectedIdStore = 'Input' + selectionManager.selectId('Input') event.preventDefault() } break @@ -708,9 +725,9 @@ case 'ArrowDown': { if (!$insertButtonOpen && !flowPreviewButtons?.getPreviewOpen()) { let ids = generateIds() - let idx = ids.indexOf($selectedIdStore) + let idx = ids.indexOf(selectedIdStore!) if (idx > -1 && idx < ids.length - 1) { - $selectedIdStore = ids[idx + 1] + selectionManager.selectId(ids[idx + 1]) event.preventDefault() } } @@ -719,9 +736,9 @@ case 'ArrowUp': { if (!$insertButtonOpen && !flowPreviewButtons?.getPreviewOpen()) { let ids = generateIds() - let idx = ids.indexOf($selectedIdStore) + let idx = ids.indexOf(selectedIdStore!) if (idx > 0 && idx < ids.length) { - $selectedIdStore = ids[idx - 1] + selectionManager.selectId(ids[idx - 1]) event.preventDefault() } } @@ -868,7 +885,7 @@ setContext('customUi', customUi) }) $effect.pre(() => { - if (flowStore.val || $selectedIdStore) { + if (flowStore.val || selectedIdStore) { readFieldsRecursively(flowStore.val) untrack(() => saveSessionDraft()) } @@ -932,7 +949,7 @@ job.success && flowPreviewButtons?.getPreviewMode() === 'whole' ) { - if (flowEditor?.isNodeVisible('result') && $selectedIdStore !== 'Result') { + if (flowEditor?.isNodeVisible('Result') && selectedIdStore !== 'Result') { outputPickerOpenFns['Result']?.() } } else { @@ -1026,7 +1043,7 @@ } } - $selectedIdStore = 'Input' + selectionManager.selectId('Input') }} on:redo={() => { flowStore.val = redo(history) @@ -1044,7 +1061,7 @@ variant="subtle" size="xs" on:click={async () => { - select('triggers') + select('Trigger') const selected = primaryScheduleIndex ?? scheduleIndex if (selected) { triggersState.selectedTriggerIndex = selected @@ -1137,7 +1154,7 @@ {/if} { - select('triggers') + select('Trigger') handleSelectTriggerFromKind(triggersState, triggersCount, initialPath, e.detail.kind) captureOn.set(true) showCaptureHint.set(true) @@ -1190,7 +1207,7 @@ on:applyArgs={(ev) => { if (ev.detail.kind === 'preprocessor') { stepsInputArgs.setStepArgs('preprocessor', ev.detail.args ?? {}) - $selectedIdStore = 'preprocessor' + selectionManager.selectId('preprocessor') } }} on:testWithArgs={(e) => { @@ -1203,7 +1220,7 @@ {savedFlow} onDeployTrigger={handleDeployTrigger} onEditInput={(moduleId, key) => { - selectedIdStore.set(moduleId) + selectionManager.selectId(moduleId) // Use new prop-based system forceTestTab[moduleId] = true highlightArg[moduleId] = key diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 5297ed3584..c54935299d 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -45,8 +45,9 @@ modules={flow?.value?.modules} failureModule={flow?.value?.failure_module} preprocessorModule={flow?.value?.preprocessor_module} + notes={flow?.value?.notes} onSelect={(nodeId) => { - if (nodeId === 'triggers') { + if (nodeId === 'Trigger') { dispatch('triggerDetail') return } else if (nodeId === 'failure') { diff --git a/frontend/src/lib/components/FlowPreviewContent.svelte b/frontend/src/lib/components/FlowPreviewContent.svelte index 751f412a11..ec6d02f0dd 100644 --- a/frontend/src/lib/components/FlowPreviewContent.svelte +++ b/frontend/src/lib/components/FlowPreviewContent.svelte @@ -100,7 +100,7 @@ } const { - selectedId, + selectionManager, previewArgs, flowStateStore, flowStore, @@ -136,7 +136,7 @@ } else { const flow = previewFlow ?? stateSnapshot(flowStore).val const idOrders = dfs(flow.value.modules, (x) => x.id) - let upToIndex = idOrders.indexOf(upToId ?? $selectedId) + let upToIndex = idOrders.indexOf(upToId ?? selectionManager.getSelectedId() ?? '') if (upToIndex != -1) { flow.value.modules = sliceModules(flow.value.modules, upToIndex, idOrders) @@ -441,7 +441,7 @@ {#if previewMode == 'upTo'} Test up to - {$selectedId} + {selectionManager.getSelectedId()} {:else} Test flow diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 4a573bfcc5..120c7cb9a1 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -39,7 +39,6 @@ import type { FlowGraphAssetContext } from './flows/types' import { createState } from '$lib/svelte5Utils.svelte' import JobLoader from './JobLoader.svelte' - import { writable } from 'svelte/store' import { AI_TOOL_CALL_PREFIX, AI_TOOL_MESSAGE_PREFIX, @@ -48,6 +47,7 @@ } from './graph/renderers/nodes/AIToolNode.svelte' import JobAssetsViewer from './assets/JobAssetsViewer.svelte' import McpToolCallDetails from './McpToolCallDetails.svelte' + import { SelectionManager } from './graph/selectionUtils.svelte' let { flowState: flowStateStore, @@ -232,7 +232,7 @@ let expandedSubflows: Record = $state({}) - let selectedId = writable(selectedNode) + let selectionManager = new SelectionManager() function onFlowModuleId() { let modId = flowJobIds?.moduleId @@ -1730,7 +1730,7 @@ {/each} -
+
diff --git a/frontend/src/lib/components/Password.svelte b/frontend/src/lib/components/Password.svelte index f6fd20eb09..c14a654ccb 100644 --- a/frontend/src/lib/components/Password.svelte +++ b/frontend/src/lib/components/Password.svelte @@ -1,5 +1,8 @@
-
- - +
+
- {#if hideValue} - - {:else} - - {/if} + onBlur?.(e), + onkeydown: (e) => { + onKeyDown?.(e) + bubble('keydown')(e) + }, + type: hideValue ? 'password' : 'text' + }} + />
{#if red}
This field is required
diff --git a/frontend/src/lib/components/SchemaForm.svelte b/frontend/src/lib/components/SchemaForm.svelte index a0fb8b56b5..16494af9b9 100644 --- a/frontend/src/lib/components/SchemaForm.svelte +++ b/frontend/src/lib/components/SchemaForm.svelte @@ -415,7 +415,7 @@ {#snippet actions()} {@render actions_render?.({ item })} {#if linkedSecretCandidates?.includes(argName)} -
+
{ @@ -429,14 +429,12 @@ {#snippet children({ item })} { const contentHeight = Math.min(1000, editor.getContentHeight()) if (divEl) { - divEl.style.height = `${contentHeight + 2}px` + divEl.style.height = `${contentHeight}px` } try { editor.layout({ width, height: contentHeight }) diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 4457f536c6..d22e8c8539 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -1,6 +1,8 @@ + +
+ {@render children?.()} +
+ +{#if $open} +
+ {#each items as menuItem (menuItem.id)} + {#if menuItem.divider} +
+ {:else} +
handleItemClick(menuItem)} + > + {#if menuItem.icon} + + {/if} + {#if menu} + {@render menu({ item: menuItem })} + {:else} + {menuItem.label} + {/if} +
+ {/if} + {/each} +
+{/if} diff --git a/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts b/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts new file mode 100644 index 0000000000..ecbe6c6f87 --- /dev/null +++ b/frontend/src/lib/components/common/contextmenu/contextMenuStyles.ts @@ -0,0 +1,45 @@ +/** + * Shared styles for context menu components + * Ensures visual consistency across all context menu implementations + */ + +/** + * Base container styles for context menu + * @param zIndex - Optional z-index override (default: 'z-50') + */ +export function getContextMenuContainerClass(zIndex: string = 'z-50'): string { + return `${zIndex} flex flex-col gap-1 min-w-[12rem] overflow-hidden rounded-md border bg-surface p-1 shadow-md` +} + +/** + * Base styles for context menu items + */ +export const CONTEXT_MENU_ITEM_BASE_CLASS = + 'relative flex cursor-default select-none items-center rounded-md px-2 py-1.5 text-xs outline-none transition-colors' + +/** + * Hover state styles for context menu items (standard CSS hover) + */ +export const CONTEXT_MENU_ITEM_HOVER_CLASS = 'hover:bg-surface-hover' + +/** + * Hover state styles for context menu items (Melt UI data attribute) + */ +export const CONTEXT_MENU_ITEM_HOVER_MELT_CLASS = 'data-[highlighted]:bg-surface-hover' + +/** + * Disabled state styles for context menu items + */ +export const CONTEXT_MENU_ITEM_DISABLED_CLASS = 'pointer-events-none opacity-50' + +/** + * Divider styles for context menu + */ +export const CONTEXT_MENU_DIVIDER_CLASS = 'my-1 h-px bg-border-light' + +/** + * Melt UI animation classes for context menu + */ +export const CONTEXT_MENU_ANIMATION_CLASSES = + 'data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2' + diff --git a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte index 8887e6b4e0..0062c171aa 100644 --- a/frontend/src/lib/components/common/fileDownload/FileDownload.svelte +++ b/frontend/src/lib/components/common/fileDownload/FileDownload.svelte @@ -12,20 +12,22 @@ let { s3object, workspaceId = undefined, appPath = undefined }: Props = $props() - - - - {s3object.storage ? `s3://${s3object.storage}/${s3object.s3}` : `s3:///${s3object.s3}`} - - + href={`${base}/api/w/${workspaceId ?? $workspaceStore}${ + appPath ? `/apps_u/download_s3_file/${appPath}` : '/job_helpers/download_s3_file' + }?${appPath ? 's3' : 'file_key'}=${encodeURIComponent(s3object?.s3 ?? '')}${ + s3object?.storage ? `&storage=${s3object.storage}` : '' + }${appPath && s3object?.presigned ? `&${s3object?.presigned}` : ''}`} + download={s3object?.s3?.split?.('/')?.pop() ?? 'unnamed_download.file'} + > + + + {s3object?.storage ? `s3://${s3object.storage}/${s3object.s3}` : `s3:///${s3object.s3}`} + + +{/if} diff --git a/frontend/src/lib/components/copilot/IteratorGen.svelte b/frontend/src/lib/components/copilot/IteratorGen.svelte index 46eb6d1519..b9015eb2b1 100644 --- a/frontend/src/lib/components/copilot/IteratorGen.svelte +++ b/frontend/src/lib/components/copilot/IteratorGen.svelte @@ -35,7 +35,7 @@ ) let abortController = new AbortController() - const { flowStore, selectedId } = getContext('FlowEditorContext') + const { flowStore, selectionManager } = getContext('FlowEditorContext') async function generateIteratorExpr() { if (generatedContent.length > 0 || loading) { @@ -45,7 +45,7 @@ loading = true const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) const idOrders = dfs(flow.value.modules, (x) => x.id) - const upToIndex = idOrders.indexOf($selectedId) + const upToIndex = idOrders.indexOf(selectionManager.getSelectedId()) if (upToIndex === -1) { throw new Error('Could not find the selected id in the flow') } @@ -60,7 +60,7 @@ flow_input: pickableProperties?.flow_input } const user = `I'm building a workflow which is a DAG of script steps. -The current step is ${$selectedId} and represents a for-loop. You can find the details of all the steps below: +The current step is ${selectionManager.getSelectedId()} and represents a for-loop. You can find the details of all the steps below: ${flowDetails} Determine the iterator expression to pass either from the previous results or the flow inputs. Here's a summary of the available data: diff --git a/frontend/src/lib/components/copilot/PredicateGen.svelte b/frontend/src/lib/components/copilot/PredicateGen.svelte index 6bfeacd62c..c49edb70a8 100644 --- a/frontend/src/lib/components/copilot/PredicateGen.svelte +++ b/frontend/src/lib/components/copilot/PredicateGen.svelte @@ -29,7 +29,7 @@ }) let abortController = $state(new AbortController()) - const { flowStore, selectedId } = getContext('FlowEditorContext') + const { flowStore, selectionManager } = getContext('FlowEditorContext') const dispatch = createEventDispatcher() @@ -38,7 +38,7 @@ loading = true const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) const idOrders = dfs(flow.value.modules, (x) => x.id) - const upToIndex = idOrders.indexOf($selectedId) + const upToIndex = idOrders.indexOf(selectionManager.getSelectedId()) if (upToIndex === -1) { throw new Error('Could not find the selected id in the flow') } @@ -53,7 +53,7 @@ flow_input: pickableProperties?.flow_input } const user = `I'm building a workflow which is a DAG of script steps. -The current step is ${$selectedId} and is a branching step (if-else). +The current step is ${selectionManager.getSelectedId()} and is a branching step (if-else). The user wants to generate a predicate for the branching condition. Here's the user's request: ${instructions} You can find the details of all the steps below: diff --git a/frontend/src/lib/components/copilot/StepInputGen.svelte b/frontend/src/lib/components/copilot/StepInputGen.svelte index 9dec547627..d2a29cfa8c 100644 --- a/frontend/src/lib/components/copilot/StepInputGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputGen.svelte @@ -54,7 +54,7 @@ let abortController = new AbortController() let newFlowInput = $state('') - const { flowStore, selectedId } = getContext('FlowEditorContext') + const { flowStore, selectionManager } = getContext('FlowEditorContext') const { stepInputsLoading, generatedExprs } = getContext('FlowCopilotContext') || {} @@ -86,7 +86,7 @@ loading = true const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) const idOrders = dfs(flow.value.modules, (x) => x.id) - const upToIndex = idOrders.indexOf($selectedId) + const upToIndex = idOrders.indexOf(selectionManager.getSelectedId()) if (upToIndex === -1) { throw new Error('Could not find the selected id in the flow') } @@ -102,7 +102,7 @@ } const isInsideLoop = availableData.flow_input && 'iter' in availableData.flow_input const user = `I'm building a workflow which is a DAG of script steps. -The current step is ${$selectedId}, you can find the details for the step and previous ones below: +The current step is ${selectionManager.getSelectedId()}, you can find the details for the step and previous ones below: ${flowDetails} Determine for the input "${argName}", what to pass either from the previous results or the flow inputs. All possibles inputs either start with results. or flow_input. and are followed by the key of the input. diff --git a/frontend/src/lib/components/copilot/StepInputsGen.svelte b/frontend/src/lib/components/copilot/StepInputsGen.svelte index 2ce8d4b177..c2781153a6 100644 --- a/frontend/src/lib/components/copilot/StepInputsGen.svelte +++ b/frontend/src/lib/components/copilot/StepInputsGen.svelte @@ -30,7 +30,7 @@ let { pickableProperties = undefined, argNames = [], schema = undefined }: Props = $props() - const { flowStore, selectedId } = getContext('FlowEditorContext') + const { flowStore, selectionManager } = getContext('FlowEditorContext') const { exprsToSet, stepInputsLoading, generatedExprs } = getContext('FlowCopilotContext') || {} @@ -49,7 +49,7 @@ stepInputsLoading?.set(true) const flow: Flow = JSON.parse(JSON.stringify(flowStore.val)) const idOrders = dfs(flow.value.modules, (x) => x.id) - const upToIndex = idOrders.indexOf($selectedId) + const upToIndex = idOrders.indexOf(selectionManager.getSelectedId()) if (upToIndex === -1) { throw new Error('Could not find the selected id in the flow') } @@ -65,7 +65,7 @@ } const isInsideLoop = availableData.flow_input && 'iter' in availableData.flow_input const user = `I'm building a workflow which is a DAG of script steps. -The current step is ${$selectedId}, you can find the details for the step and previous ones below: +The current step is ${selectionManager.getSelectedId()}, you can find the details for the step and previous ones below: ${flowDetails} Determine for all the inputs "${argNames.join( diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 53cd319d55..5645792971 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -43,6 +43,7 @@ import type { Selection } from 'monaco-editor' import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { getAnthropicCompletion, parseAnthropicCompletion } from './anthropic' +import { getOpenAIResponsesCompletion, parseOpenAIResponsesCompletion } from './openai-responses' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, getCombinedCustomPrompt } from '$lib/aiStore' @@ -407,15 +408,42 @@ class AIChatManager { } const model = getCurrentModel() - const completionFn = model.provider === 'anthropic' ? getAnthropicCompletion : getCompletion - const parseFn = - model.provider === 'anthropic' ? parseAnthropicCompletion : parseOpenAICompletion + const isOpenAI = model.provider === 'openai' || model.provider === 'azure_openai' + const isAnthropic = model.provider === 'anthropic' - const completion = await completionFn( - [systemMessage, ...messages, ...(pendingUserMessage ? [pendingUserMessage] : [])], - abortController, - tools.map((t) => t.def) - ) + let completion: any + let parseFn: any + + const messageParams = [ + systemMessage, + ...messages, + ...(pendingUserMessage ? [pendingUserMessage] : []) + ] + const toolDefs = tools.map((t) => t.def) + + // For OpenAI/Azure, try Responses API first, fallback to Completions API + if (isOpenAI) { + try { + completion = await getOpenAIResponsesCompletion( + messageParams, + abortController, + toolDefs + ) + parseFn = parseOpenAIResponsesCompletion + } catch (err) { + console.warn('OpenAI Responses API failed, falling back to Completions API:', err) + completion = await getCompletion(messageParams, abortController, toolDefs, { + forceCompletions: true + }) + parseFn = parseOpenAICompletion + } + } else if (isAnthropic) { + completion = await getAnthropicCompletion(messageParams, abortController, toolDefs) + parseFn = parseAnthropicCompletion + } else { + completion = await getCompletion(messageParams, abortController, toolDefs) + parseFn = parseOpenAICompletion + } if (completion) { const continueCompletion = await parseFn( @@ -872,7 +900,7 @@ class AIChatManager { } listenForSelectedIdChanges = ( - selectedId: string, + selectedId: string | undefined, flowStore: ExtendedOpenFlow, flowStateStore: FlowState, currentEditor: CurrentEditor diff --git a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte index dc48253240..56c7f18f7c 100644 --- a/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte +++ b/frontend/src/lib/components/copilot/chat/flow/FlowAIChat.svelte @@ -25,8 +25,9 @@ flowModuleSchemaMap: FlowModuleSchemaMap | undefined } = $props() - const { flowStore, flowStateStore, selectedId, currentEditor } = + const { flowStore, flowStateStore, selectionManager, currentEditor } = getContext('FlowEditorContext') + const selectedId = $derived(selectionManager.getSelectedId()) const { exprsToSet } = getContext('FlowCopilotContext') ?? {} @@ -84,7 +85,7 @@ const flow = $state.snapshot(flowStore).val return { flow, - selectedId: $selectedId + selectedId: selectedId } }, // flow apply/reject @@ -382,7 +383,7 @@ value: match[2].trim() })) - if (id === $selectedId) { + if (id === selectedId) { exprsToSet?.set({}) const argsToUpdate = {} for (const { input, value } of parsedInputs) { @@ -421,7 +422,7 @@ setModuleStatus('Input', 'modified') }, selectStep: (id) => { - $selectedId = id + selectionManager.selectId(id) }, getStepCode: (id) => { const module = getModule(id) @@ -611,7 +612,7 @@ $effect(() => { const cleanup = aiChatManager.listenForSelectedIdChanges( - $selectedId, + selectedId, flowStore.val, flowStateStore.val, $currentEditor @@ -628,19 +629,18 @@ $effect(() => { if ( $currentEditor?.type === 'script' && - $selectedId && - affectedModules[$selectedId] && + selectedId && + affectedModules[selectedId] && $currentEditor.editor.getAiChatEditorHandler() ) { - const moduleLastSnapshot = getModule($selectedId, lastSnapshot) + const moduleLastSnapshot = getModule(selectedId, lastSnapshot) const content = moduleLastSnapshot?.value.type === 'rawscript' ? moduleLastSnapshot.value.content : '' if (content.length > 0) { untrack(() => $currentEditor.editor.reviewAppliedCode(content, { onFinishedReview: () => { - const id = $selectedId - flowHelpers.acceptModuleAction(id) + flowHelpers.acceptModuleAction(selectedId) $currentEditor.hideDiffMode() } }) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts new file mode 100644 index 0000000000..4229af0d16 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -0,0 +1,366 @@ +import OpenAI, { OpenAIError } from 'openai' +import type { + ChatCompletionMessageParam, + ChatCompletionMessageFunctionToolCall, + ChatCompletionCreateParams +} from 'openai/resources/index.mjs' +import type { ResponseErrorEvent } from 'openai/resources/responses/responses.mjs' +import { getProviderAndCompletionConfig, workspaceAIClients } from '../lib' +import { processToolCall, type Tool, type ToolCallbacks } from './shared' +import type { ResponseStream } from 'openai/lib/responses/ResponseStream.mjs' +import { OpenAPI } from '$lib/gen' +import type { AIProviderModel } from '$lib/gen' + +// Conversion utilities for Responses API +function convertMessagesToResponsesInput(messages: ChatCompletionMessageParam[]): { + instructions?: string + input: Array +} { + const systemMessage = messages.find((m) => m.role === 'system') + const nonSystemMessages = messages.filter((m) => m.role !== 'system') + + const input: Array = [] + + for (const m of nonSystemMessages) { + // Handle assistant messages with tool calls + if (m.role === 'assistant' && 'tool_calls' in m && m.tool_calls) { + // First, add the assistant's text content if it exists + if ('content' in m && m.content) { + input.push({ + type: 'message' as const, + role: 'assistant', + content: m.content + }) + } + + // Then add tool calls + for (const toolCall of m.tool_calls) { + if (toolCall.type === 'function') { + // Validate that required fields exist + if (!toolCall.function.name || toolCall.function.name.trim() === '') { + console.warn('Skipping tool call with empty name:', toolCall) + continue + } + + input.push({ + type: 'function_call', + call_id: toolCall.id, + name: toolCall.function.name, + arguments: toolCall.function.arguments + }) + } + } + } + // Handle tool result messages + else if (m.role === 'tool' && 'content' in m && 'tool_call_id' in m) { + input.push({ + type: 'function_call_output', + call_id: m.tool_call_id, + output: typeof m.content === 'string' ? m.content : JSON.stringify(m.content) + }) + } + // Handle regular messages + else if ('content' in m && m.content !== null && m.content !== undefined) { + input.push({ + type: 'message' as const, + role: m.role === 'developer' ? 'developer' : m.role === 'assistant' ? 'assistant' : 'user', + content: m.content + }) + } + } + + return { + instructions: + systemMessage && 'content' in systemMessage + ? typeof systemMessage.content === 'string' + ? systemMessage.content + : JSON.stringify(systemMessage.content) + : undefined, + input + } +} + +function convertCompletionConfigToResponsesConfig( + config: ChatCompletionCreateParams +): Record { + const responsesConfig: Record = { + model: config.model + } + + // Map max_tokens or max_completion_tokens to max_output_tokens + if ('max_completion_tokens' in config && config.max_completion_tokens) { + responsesConfig.max_output_tokens = config.max_completion_tokens + } else if ('max_tokens' in config && config.max_tokens) { + responsesConfig.max_output_tokens = config.max_tokens + } + + // Keep other relevant fields + if (config.temperature !== undefined) { + responsesConfig.temperature = config.temperature + } + if ('tools' in config && config.tools && config.tools.length > 0) { + responsesConfig.tools = config.tools.map((tool) => { + if (tool.type === 'function' && 'function' in tool) { + // Convert from Completions format to Responses format + return { + type: 'function', + name: tool.function.name, + description: tool.function.description, + parameters: tool.function.parameters, + strict: tool.function.strict ?? null + } + } + // Pass through other tool types unchanged + return tool + }) + } + + return responsesConfig +} + +export async function getOpenAIResponsesCompletion( + messages: ChatCompletionMessageParam[], + abortController: AbortController, + tools?: OpenAI.Chat.Completions.ChatCompletionTool[] +) { + const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { instructions, input } = convertMessagesToResponsesInput(messages) + const responsesConfig = convertCompletionConfigToResponsesConfig(config) + + const openaiClient = workspaceAIClients.getOpenaiClient() + + const runner = openaiClient.responses.stream( + { + ...responsesConfig, + input, + ...(instructions ? { instructions } : {}) + }, + { + signal: abortController.signal, + headers: { + 'X-Provider': provider + } + } + ) + + return runner +} + +// Wrapper that converts ResponseStream to ChatCompletionChunk format for lib.ts usage +export async function* getOpenAIResponsesCompletionStream( + messages: ChatCompletionMessageParam[], + abortController: AbortController, + tools?: OpenAI.Chat.Completions.ChatCompletionTool[] +): AsyncGenerator { + const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + const { instructions, input } = convertMessagesToResponsesInput(messages) + const responsesConfig = convertCompletionConfigToResponsesConfig(config) + + const openaiClient = workspaceAIClients.getOpenaiClient() + + const runner = openaiClient.responses.stream( + { + ...responsesConfig, + input, + ...(instructions ? { instructions } : {}) + }, + { + signal: abortController.signal, + headers: { + 'X-Provider': provider + } + } + ) + + // Convert ResponseStream events to ChatCompletionChunk format + for await (const event of runner) { + if (event.type === 'response.output_text.delta') { + // Yield text chunks in ChatCompletionChunk format + yield { + id: 'chatcmpl-' + Date.now(), + object: 'chat.completion.chunk', + created: Date.now(), + model: responsesConfig.model, + choices: [ + { + index: 0, + delta: { + content: event.delta || '' + }, + finish_reason: null + } + ] + } as OpenAI.Chat.Completions.ChatCompletionChunk + } + } +} + +export async function parseOpenAIResponsesCompletion( + runner: ResponseStream, + callbacks: ToolCallbacks & { + onNewToken: (token: string) => void + onMessageEnd: () => void + }, + messages: ChatCompletionMessageParam[], + addedMessages: ChatCompletionMessageParam[], + tools: Tool[], + helpers: any +): Promise { + let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] + let error: OpenAIError | ResponseErrorEvent | null = null + let textContent = '' + let toolCallsMap: Record = {} + + // Handle text streaming + runner.on('response.output_text.delta', (event) => { + callbacks.onNewToken(event.delta) + textContent += event.delta + }) + + // Handle new output items (including function calls) + runner.on('response.output_item.added', (event) => { + const item = event.item + if (item.type === 'function_call' && item.id) { + toolCallsMap[item.id] = { + name: item.name, + call_id: item.call_id + } + + // Show temporary loading state for the tool call + callbacks.onMessageEnd() + callbacks.setToolStatus(`${item.id}`, { + isLoading: true, + content: `Calling ${item.name} tool...` + }) + } + }) + + // Handle function call arguments done + runner.on('response.function_call_arguments.done', (event) => { + // Retrieve tool call metadata from map + const metadata = toolCallsMap[event.item_id] + if (!metadata) { + console.error('Missing tool call metadata for:', event.item_id) + return + } + + // Convert to OpenAI format for compatibility with existing tool processing + toolCallsToProcess.push({ + id: event.item_id, + type: 'function' as const, + function: { + name: metadata.name, + arguments: event.arguments + } + }) + }) + + // Handle errors + runner.on('error', (err: OpenAIError | ResponseErrorEvent) => { + console.error('OpenAI Responses stream error:', err) + error = err + }) + + // Wait for completion + await runner.done() + + // Add text message if we got any text + if (textContent) { + const assistantMessage = { role: 'assistant' as const, content: textContent } + messages.push(assistantMessage) + addedMessages.push(assistantMessage) + callbacks.onMessageEnd() + } + + if (error) { + throw error + } + + // Process tool calls if any + if (toolCallsToProcess.length > 0) { + const assistantWithTools = { + role: 'assistant' as const, + tool_calls: toolCallsToProcess + } + messages.push(assistantWithTools) + addedMessages.push(assistantWithTools) + + // Process each tool call + for (const toolCall of toolCallsToProcess) { + const messageToAdd = await processToolCall({ + tools, + toolCall, + helpers, + toolCallbacks: callbacks + }) + messages.push(messageToAdd) + addedMessages.push(messageToAdd) + } + return true // Continue the conversation loop + } + + return false // End the conversation +} + +export async function getNonStreamingOpenAIResponsesCompletion( + messages: ChatCompletionMessageParam[], + abortController: AbortController, + testOptions?: { + apiKey?: string + resourcePath?: string + forceModelProvider: AIProviderModel + } +): Promise { + const { provider, config } = getProviderAndCompletionConfig({ + messages, + stream: false, + forceModelProvider: testOptions?.forceModelProvider + }) + + const { instructions, input } = convertMessagesToResponsesInput(messages) + const responsesConfig = convertCompletionConfigToResponsesConfig(config) + + const fetchOptions: { + signal: AbortSignal + headers: Record + } = { + signal: abortController.signal, + headers: { + 'X-Provider': provider + } + } + + if (testOptions?.resourcePath) { + fetchOptions.headers = { + ...fetchOptions.headers, + 'X-Resource-Path': testOptions.resourcePath + } + } else if (testOptions?.apiKey) { + fetchOptions.headers = { + ...fetchOptions.headers, + 'X-API-Key': testOptions.apiKey + } + } + + const openaiClient = testOptions?.apiKey + ? new OpenAI({ + baseURL: `${location.origin}${OpenAPI.BASE}/ai/proxy`, + apiKey: 'fake-key', + defaultHeaders: { + Authorization: '' // a non empty string will be unable to access Windmill backend proxy + }, + dangerouslyAllowBrowser: true + }) + : workspaceAIClients.getOpenaiClient() + + const response = await openaiClient.responses.create( + { + ...responsesConfig, + input, + ...(instructions ? { instructions } : {}) + }, + fetchOptions + ) + + return response.output_text || '' +} diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 438fdf45ce..5c14610c45 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -204,7 +204,8 @@ export const SUPPORTED_CHAT_SCRIPT_LANGUAGES = [ 'graphql', 'powershell', 'csharp', - 'java' + 'java', + 'duckdb' ] export function getLangContext( @@ -310,6 +311,8 @@ export function getLangContext( return 'The user is coding in C#. On Windmill, it is expected the script contains a public static Main method inside a class. The class name is irrelevant. NuGet packages can be added using the format: #r "nuget: PackageName, Version" at the top of the script. The Main method signature should be: public static ReturnType Main(parameter types...)' case 'java': return 'The user is coding in Java. On Windmill, it is expected the script contains a Main public class and a public static main() method. The return type can be Object or void. Dependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script. The method signature should be: public static Object main(parameter types...)' + case 'duckdb': + return "The user is coding in DuckDB. On Windmill, arguments are defined with comments like `-- $name (text) = default` or `-- $name (text)` (one per line) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes, then perform CRUD operations. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);` and query with `SELECT * FROM db.schema.table;`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage" default: return '' } diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index e589f2d751..1fa2a0d07c 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -17,6 +17,10 @@ import { EDIT_CONFIG, FIX_CONFIG, GEN_CONFIG } from './prompts' import { formatResourceTypes } from './utils' import { z } from 'zod' import { processToolCall, type Tool, type ToolCallbacks } from './chat/shared' +import { + getNonStreamingOpenAIResponsesCompletion, + getOpenAIResponsesCompletionStream +} from './chat/openai-responses' import type { Stream } from 'openai/core/streaming.mjs' import { generateRandomString } from '$lib/utils' import { copilotInfo, getCurrentModel } from '$lib/aiStore' @@ -76,6 +80,10 @@ export const AI_PROVIDERS: Record = { label: 'Together AI', defaultModels: ['meta-llama/Llama-3.3-70B-Instruct-Turbo'] }, + aws_bedrock: { + label: 'AWS Bedrock', + defaultModels: ['global.anthropic.claude-haiku-4-5-20251001-v1:0'] + }, customai: { label: 'Custom AI', defaultModels: [] @@ -100,18 +108,102 @@ export async function fetchAvailableModels( provider: AIProvider, signal?: AbortSignal ): Promise { - const models = await fetch(`${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy/models`, { - signal, - headers: { + // Handle AWS Bedrock separately (needs both foundation-models and inference-profiles) + if (provider === 'aws_bedrock') { + const headers = { 'X-Resource-Path': resourcePath, - 'X-Provider': provider, - ...(provider === 'anthropic' ? { 'anthropic-version': '2023-06-01' } : {}) + 'X-Provider': provider } - }) + + // Fetch both foundation models and inference profiles + const [foundationModelsResp, inferenceProfilesResp] = await Promise.all([ + fetch(`${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy/foundation-models`, { + signal, + headers + }), + fetch(`${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy/inference-profiles`, { + signal, + headers + }) + ]) + + if (!foundationModelsResp.ok) { + console.error('Failed to fetch foundation models', foundationModelsResp) + throw new Error('Failed to fetch foundation models for AWS Bedrock') + } + + const foundationModelsData = (await foundationModelsResp.json()) as { + modelSummaries: Array<{ + modelId: string + modelArn: string + inputModalities: string[] + outputModalities: string[] + inferenceTypesSupported: string[] + }> + } + + // Inference profiles fetch might fail in some regions/accounts + let inferenceProfiles: Array<{ + inferenceProfileId: string + models: Array<{ modelArn: string }> + }> = [] + + if (inferenceProfilesResp.ok) { + const inferenceProfilesData = (await inferenceProfilesResp.json()) as { + inferenceProfileSummaries: Array<{ + inferenceProfileId: string + models: Array<{ modelArn: string }> + }> + } + inferenceProfiles = inferenceProfilesData.inferenceProfileSummaries || [] + } else { + console.warn('Failed to fetch inference profiles, will use direct model IDs only') + } + + // Filter to TEXT-capable models + const textModels = foundationModelsData.modelSummaries.filter( + (m) => m.inputModalities?.includes('TEXT') && m.outputModalities?.includes('TEXT') + ) + + const onDemandModels = textModels + .filter( + (model) => + model.inferenceTypesSupported?.includes('ON_DEMAND') && + !model.inferenceTypesSupported?.includes('INFERENCE_PROFILE') + ) + .map((model) => model.modelId) + const inferenceModels = inferenceProfiles.map((profile) => profile.inferenceProfileId) + const modelIds = [...onDemandModels, ...inferenceModels] + + // Sort by default models + const defaultModels = AI_PROVIDERS[provider]?.defaultModels || [] + return modelIds.sort((a, b) => { + const aInDefault = defaultModels.includes(a) + const bInDefault = defaultModels.includes(b) + if (aInDefault && !bInDefault) return -1 + if (!aInDefault && bInDefault) return 1 + return 0 + }) + } + + // Standard provider handling + const endpoint = 'models' + const models = await fetch( + `${location.origin}${OpenAPI.BASE}/w/${workspace}/ai/proxy/${endpoint}`, + { + signal, + headers: { + 'X-Resource-Path': resourcePath, + 'X-Provider': provider, + ...(provider === 'anthropic' ? { 'anthropic-version': '2023-06-01' } : {}) + } + } + ) if (!models.ok) { console.error('Failed to fetch models for provider', provider, models) throw new Error(`Failed to fetch models for provider ${provider}`) } + const data = (await models.json()) as { data: ModelResponse[] } if (data.data.length > 0) { const sortFunc = (provider: AIProvider) => (a: string, b: string) => { @@ -137,8 +229,7 @@ export async function fetchAvailableModels( .filter( (m) => (m.id.startsWith('gpt-') || m.id.startsWith('o') || m.id.startsWith('codex')) && - m.lifecycle_status !== 'deprecated' && - (m.capabilities.completion || m.capabilities.chat_completion) + m.lifecycle_status !== 'deprecated' ) .map((m) => m.id) .sort(sortFunc(provider)) @@ -271,7 +362,8 @@ export const PROVIDER_COMPLETION_CONFIG_MAP: Record @@ -696,10 +802,24 @@ export async function getFimCompletion( export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, - tools?: OpenAI.Chat.Completions.ChatCompletionTool[] + tools?: OpenAI.Chat.Completions.ChatCompletionTool[], + options?: { + forceCompletions?: boolean + } ): Promise> { const { provider, config } = getProviderAndCompletionConfig({ messages, stream: true, tools }) + // Use Responses API for OpenAI and Azure OpenAI + if ((provider === 'openai' || provider === 'azure_openai') && !options?.forceCompletions) { + try { + const stream = getOpenAIResponsesCompletionStream(messages, abortController, tools) as any + return stream + } catch (error) { + console.error('Error using Responses API:', error) + } + } + + // Use Completions API for other providers const openaiClient = workspaceAIClients.getOpenaiClient() const completion = openaiClient.chat.completions.create(config, { signal: abortController.signal, @@ -931,52 +1051,3 @@ export async function copilot( return code } - -function getStringEndDelta(prev: string, now: string) { - return now.slice(prev.length) -} - -export async function deltaCodeCompletion( - messages: ChatCompletionMessageParam[], - generatedCodeDelta: Writable, - abortController: AbortController -) { - const completion = await getCompletion(messages, abortController) - - let response = '' - let code = '' - let delta = '' - for await (const part of completion) { - response += getResponseFromEvent(part) - let match = response.match(/```[a-zA-Z]+\n([\s\S]*?)\n```/) - - if (match) { - // if we have a full code block - delta = getStringEndDelta(code, match[1]) - code = match[1] - generatedCodeDelta.set(delta) - - break - } - - // partial code block, keep going - match = response.match(/```[a-zA-Z]+\n([\s\S]*)/) - - if (!match) { - continue - } - - if (!match[1].endsWith('`')) { - // skip updating if possible that part of three ticks (end of code block)s - delta = getStringEndDelta(code, match[1]) - generatedCodeDelta.set(delta) - code = match[1] - } - } - - if (code.length === 0) { - throw new Error('No code block found') - } - - return code -} diff --git a/frontend/src/lib/components/copilot/prompts/edit.yaml b/frontend/src/lib/components/copilot/prompts/edit.yaml index e17e8139f1..22c2d54267 100644 --- a/frontend/src/lib/components/copilot/prompts/edit.yaml +++ b/frontend/src/lib/components/copilot/prompts/edit.yaml @@ -57,13 +57,13 @@ prompts: pub fn main(...) -> Result> ``` but do not call it. Favor idiomatic Rust patterns, ensuring safe handling of ownership and borrowing, robust error handling with `Result`, and concurrency if needed (`async`/`tokio` or std threading). - + Follow these guidelines: - Include necessary imports and modules only when needed. - Add comments explaining important operations and any unsafe usage (if absolutely required). - The generated code should be easily executable and testable in an integrated terminal. - My instructions: {description} + My instructions: {description} go: prompt: |- Here's my go code: @@ -213,6 +213,32 @@ prompts: No need to require autoload, it is already done. My instructions: {description} + csharp: + prompt: |- + Here's my C# code: + ```csharp + {code} + ``` + + You have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result. + NuGet packages can be added using the format: #r "nuget: PackageName, Version" at the top of the script. + The Main method signature should be: public static ReturnType Main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + My instructions: {description} + java: + prompt: |- + Here's my Java code: + ```java + {code} + ``` + + You have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result. + Dependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script. + The method signature should be: public static Object main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + My instructions: {description} frontend: prompt: |- Here's my client-side javascript code: @@ -262,3 +288,13 @@ prompts: My instructions: {description} + duckdb: + prompt: |- + Here's my DuckDB code: + ```sql + {code} + ``` + + Arguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage. + + My instructions: {description} diff --git a/frontend/src/lib/components/copilot/prompts/editPrompt.ts b/frontend/src/lib/components/copilot/prompts/editPrompt.ts index 6aec87dd35..8a5ac5ff4b 100644 --- a/frontend/src/lib/components/copilot/prompts/editPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/editPrompt.ts @@ -8,7 +8,7 @@ export const EDIT_PROMPT = { "prompt": "Here's my TypeScript code in a deno running environment:\n```typescript\n{code}\n```\n\nWe have to export a \"main\" function like this: \"export async function main(...)\" and specify the parameter types but do not call it. You should generally return the result.\nIf needed, the standard fetch method is available globally, do not import it.\nYou can take as parameters resources which are dictionaries containing credentials or configuration information. Name the resource parameters like this: \"{resource_type}Resource\". \nThe resource type name has to be exactly as specified.\n\n{resourceTypes}\n\nOnly define the type for resources that are actually needed to achieve the function purpose. If the type name conflicts with the imported object, rename the imported object NOT THE TYPE.\nNaming conventions are guidelines, but follow the user's naming choices in the existing code.\n\nMy instructions: {description}" }, "rust": { - "prompt": "Here's my Rust code:\n```rust\n{code}\n```\n\nPlease define a `main` function in Rust with this signature:\n```rust\npub fn main(...) -> Result>\n```\nbut do not call it. Favor idiomatic Rust patterns, ensuring safe handling of ownership and borrowing, robust error handling with `Result`, and concurrency if needed (`async`/`tokio` or std threading). \n\nFollow these guidelines:\n- Include necessary imports and modules only when needed.\n- Add comments explaining important operations and any unsafe usage (if absolutely required).\n- The generated code should be easily executable and testable in an integrated terminal.\n\nMy instructions: {description} " + "prompt": "Here's my Rust code:\n```rust\n{code}\n```\n\nPlease define a `main` function in Rust with this signature:\n```rust\npub fn main(...) -> Result>\n```\nbut do not call it. Favor idiomatic Rust patterns, ensuring safe handling of ownership and borrowing, robust error handling with `Result`, and concurrency if needed (`async`/`tokio` or std threading). \n\nFollow these guidelines:\n- Include necessary imports and modules only when needed.\n- Add comments explaining important operations and any unsafe usage (if absolutely required).\n- The generated code should be easily executable and testable in an integrated terminal.\n\nMy instructions: {description}" }, "go": { "prompt": "Here's my go code: \n```go\n{code}\n```\n\nWe have to export a \"main\" function. Import the packages you need. The return type of the function has to be ({return_type}, error). The file package has to be \"inner\"\n\nMy instructions: {description}" @@ -46,17 +46,20 @@ export const EDIT_PROMPT = { "php": { "prompt": "Here's my php code: \n```php\n{code}\n```\n\nYou have to write a function in php called \"main\". Specify the parameter types. Do not call the main function. You should generally return the result. The script must start with `\n{resourceTypes}\n\nYou need to define the type of the resources that are needed before the main function, but only include them if they are actually needed to achieve the function purpose.\nBefore defining each type, check if the class already exists using class_exists.\nThe resource type name has to be exactly as specified.\nIf you need to import libraries, you need to specify them as comments in the following manner before the main function:\n```\n// require:\n// mylibrary/mylibrary\n// myotherlibrary/myotherlibrary@optionalversion\n```\nNo need to require autoload, it is already done.\n\nMy instructions: {description}" }, + "csharp": { + "prompt": "Here's my C# code:\n```csharp\n{code}\n```\n\nYou have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result.\nNuGet packages can be added using the format: #r \"nuget: PackageName, Version\" at the top of the script.\nThe Main method signature should be: public static ReturnType Main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" + }, + "java": { + "prompt": "Here's my Java code:\n```java\n{code}\n```\n\nYou have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result.\nDependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script.\nThe method signature should be: public static Object main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" + }, "frontend": { "prompt": "Here's my client-side javascript code: \n```javascript\n{code}\n```\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n\nMy instructions: {description}" }, - "csharp": { - "prompt": "Here's my C# code: \n```csharp\n{code}\n```\n\nYou have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result.\nNuGet packages can be added using the format: #r \"nuget: PackageName, Version\" at the top of the script.\nThe Main method signature should be: public static ReturnType Main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" - }, - "java": { - "prompt": "Here's my Java code: \n```java\n{code}\n```\n\nYou have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result.\nDependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script.\nThe method signature should be: public static Object main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" - }, "transformer": { "prompt": "Here's my client-side javascript code: \n```javascript\n{code}\n```\n\n\nThe code should process the variable `result` according to my instructions.\nThe variable `result` is available globally.\nAt the end of the code, the processed result has to be returned.\n\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n\n\nMy instructions: {description}" + }, + "duckdb": { + "prompt": "Here's my DuckDB code:\n```sql\n{code}\n```\n\nArguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage.\n\nMy instructions: {description}" } } }; \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/prompts/fix.yaml b/frontend/src/lib/components/copilot/prompts/fix.yaml index 78921857f2..b15e30a9ca 100644 --- a/frontend/src/lib/components/copilot/prompts/fix.yaml +++ b/frontend/src/lib/components/copilot/prompts/fix.yaml @@ -61,7 +61,7 @@ prompts: pub fn main(...) -> Result> ``` but do not call it. Favor idiomatic Rust patterns, ensuring safe handling of ownership and borrowing, robust error handling with `Result`, and concurrency if needed (`async`/`tokio` or std threading). - + - Include necessary imports and modules only when needed. - Add comments explaining important operations and any unsafe usage (if absolutely required). - The generated code should be easily executable and testable in an integrated terminal. @@ -229,3 +229,42 @@ prompts: I get the following error: {error} Fix my code. + csharp: + prompt: |- + Here's my C# code: + ```csharp + {code} + ``` + + You have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result. + NuGet packages can be added using the format: #r "nuget: PackageName, Version" at the top of the script. + The Main method signature should be: public static ReturnType Main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + I get the following error: {error} + Fix my code. + java: + prompt: |- + Here's my Java code: + ```java + {code} + ``` + + You have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result. + Dependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script. + The method signature should be: public static Object main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + I get the following error: {error} + Fix my code. + duckdb: + prompt: |- + Here's my DuckDB code: + ```sql + {code} + ``` + + Arguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage. + + I get the following error: {error} + Fix my code. diff --git a/frontend/src/lib/components/copilot/prompts/fixPrompt.ts b/frontend/src/lib/components/copilot/prompts/fixPrompt.ts index 4fc6ed3227..c66f74dc31 100644 --- a/frontend/src/lib/components/copilot/prompts/fixPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/fixPrompt.ts @@ -47,10 +47,13 @@ export const FIX_PROMPT = { "prompt": "Here's my php code: \n```php\n{code}\n```\n\nYou have to write a function in php called \"main\". Specify the parameter types. Do not call the main function. You should generally return the result. The script must start with `\n{resourceTypes}\n\nYou need to define the type of the resources that are needed before the main function, but only include them if they are actually needed to achieve the function purpose.\nBefore defining each type, check if the class already exists using class_exists.\nThe resource type name has to be exactly as specified.\nIf you need to import libraries, you need to specify them as comments in the following manner before the main function:\n```\n// require:\n// mylibrary/mylibrary\n// myotherlibrary/myotherlibrary@optionalversion\n```\nNo need to require autoload, it is already done.\n\nI get the following error: {error}\nFix my code." }, "csharp": { - "prompt": "Here's my C# code: \n```csharp\n{code}\n```\n\nYou have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result.\nNuGet packages can be added using the format: #r \"nuget: PackageName, Version\" at the top of the script.\nThe Main method signature should be: public static ReturnType Main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my C# code:\n```csharp\n{code}\n```\n\nYou have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result.\nNuGet packages can be added using the format: #r \"nuget: PackageName, Version\" at the top of the script.\nThe Main method signature should be: public static ReturnType Main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nI get the following error: {error}\nFix my code." }, "java": { - "prompt": "Here's my Java code: \n```java\n{code}\n```\n\nYou have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result.\nDependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script.\nThe method signature should be: public static Object main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nI get the following error: {error}\nFix my code." + "prompt": "Here's my Java code:\n```java\n{code}\n```\n\nYou have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result.\nDependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script.\nThe method signature should be: public static Object main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nI get the following error: {error}\nFix my code." + }, + "duckdb": { + "prompt": "Here's my DuckDB code:\n```sql\n{code}\n```\n\nArguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage.\n\nI get the following error: {error}\nFix my code." } } }; \ No newline at end of file diff --git a/frontend/src/lib/components/copilot/prompts/gen.yaml b/frontend/src/lib/components/copilot/prompts/gen.yaml index e7cf420654..d22a6e1473 100644 --- a/frontend/src/lib/components/copilot/prompts/gen.yaml +++ b/frontend/src/lib/components/copilot/prompts/gen.yaml @@ -151,6 +151,24 @@ prompts: No need to require autoload, it is already done. My instructions: {description} + csharp: + prompt: |- + + You have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result. + NuGet packages can be added using the format: #r "nuget: PackageName, Version" at the top of the script. + The Main method signature should be: public static ReturnType Main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + My instructions: {description} + java: + prompt: |- + + You have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result. + Dependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script. + The method signature should be: public static Object main(parameter types...) + Arguments are used to generate the input specification and create the frontend UI for the script. + + My instructions: {description} frontend: prompt: |- Write client-side javascript code that should {description}. @@ -176,7 +194,7 @@ prompts: At the end of the code, the processed result has to be returned. - You can access the context object with the ctx global variable. + You can access the context object with the ctx global variable. The app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar' You can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean) You can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string) @@ -189,3 +207,9 @@ prompts: You can validate all fields of a form: validateAll(id: string, key: string) You can invalidate a specific field of a form: invalidate(id: string, key: string, error: string) + duckdb: + prompt: |- + + You have to write a statement for DuckDB. Arguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage. + + My instructions: {description} diff --git a/frontend/src/lib/components/copilot/prompts/genPrompt.ts b/frontend/src/lib/components/copilot/prompts/genPrompt.ts index a318c55e12..478ef99f92 100644 --- a/frontend/src/lib/components/copilot/prompts/genPrompt.ts +++ b/frontend/src/lib/components/copilot/prompts/genPrompt.ts @@ -46,17 +46,20 @@ export const GEN_PROMPT = { "php": { "prompt": "\nYou have to write a function in php called \"main\". Specify the parameter types. Do not call the main function. You should generally return the result. The script must start with `\n{resourceTypes}\n\nYou need to define the type of the resources that are needed before the main function, but only include them if they are actually needed to achieve the function purpose.\nBefore defining each type, check if the class already exists using class_exists.\nThe resource type name has to be exactly as specified.\nIf you need to import libraries, you need to specify them as comments in the following manner before the main function:\n```\n// require:\n// mylibrary/mylibrary\n// myotherlibrary/myotherlibrary@optionalversion\n```\nNo need to require autoload, it is already done.\n\nMy instructions: {description}" }, - "frontend": { - "prompt": "Write client-side javascript code that should {description}. \n\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n" - }, - "transformer": { - "prompt": "Write client-side javascript code that should process the variable `result` according to the following instructions: {description}.\nThe variable `result` is available globally.\nAt the end of the code, the processed result has to be returned.\n\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n" - }, "csharp": { "prompt": "\nYou have to write C# code with a public static Main method inside a class. The class name is irrelevant. Specify the parameter types. Do not call the main function. You should generally return the result.\nNuGet packages can be added using the format: #r \"nuget: PackageName, Version\" at the top of the script.\nThe Main method signature should be: public static ReturnType Main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" }, "java": { "prompt": "\nYou have to write Java code with a Main public class and a public static main() method. The return type can be Object or void. Do not call the main function. You should generally return the result.\nDependencies can be added using the format: //requirements://groupId:artifactId:version at the top of the script.\nThe method signature should be: public static Object main(parameter types...)\nArguments are used to generate the input specification and create the frontend UI for the script.\n\nMy instructions: {description}" + }, + "frontend": { + "prompt": "Write client-side javascript code that should {description}. \n\n\nYou can access the context object with the ctx global variable. \nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n" + }, + "transformer": { + "prompt": "Write client-side javascript code that should process the variable `result` according to the following instructions: {description}.\nThe variable `result` is available globally.\nAt the end of the code, the processed result has to be returned.\n\n\nYou can access the context object with the ctx global variable.\nThe app state is a store that can be used to store data. You can access and update the state object with the state global variable like this: state.foo = 'bar'\nYou can use the goto function to navigate to a specific URL: goto(path: string, newTab?: boolean)\nYou can use the openModal and closeModal functions to open and close a modal: openModal(id: string), closeModal(id: string)\nYou can use the setTab function to manually set the tab of a Tab component: setTab(id: string, index: string)\nYou can use the recompute function to recompute a component: recompute(id: string)\nYou can use the getAgGrid function to get the ag-grid instance of a table: getAgGrid(id: string)\nYou can use the setSelectedIndex function to select a row in a table or an AG Grid table: setSelectedIndex(id: string, index: number)\nThe setValue function is meant to set or force the value of a component: setValue(id: string, value: any)\nYou can validate a specific field of a form: validate(id: string, key: string)\nYou can validate all fields of a form: validateAll(id: string, key: string)\nYou can invalidate a specific field of a form: invalidate(id: string, key: string, error: string)\n" + }, + "duckdb": { + "prompt": "\nYou have to write a statement for DuckDB. Arguments are defined with comments like `-- $age (text) = 20` or `-- $name (text)` (one per row) and used in the statement with $age, $name, etc. To use Ducklake, attach it with `ATTACH 'ducklake' AS dl;` (for main ducklake) or `ATTACH 'ducklake://name' AS dl;` for named ducklakes. To connect to external databases, use `ATTACH '$res:path/to/resource' AS db (TYPE postgres);`. To read S3 files, use `SELECT * FROM read_csv('s3:///path/to/file.csv');` for default storage or `SELECT * FROM read_csv('s3://secondary_storage_name/path/to/file.csv');` for named storage.\n\nMy instructions: {description}" } } }; \ No newline at end of file diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index 3a9e413d7b..fa1d265dbb 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -179,11 +179,11 @@ export async function getDucklakeSchema({ args: {} } }) - const stringified = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? '[]') + const mainSchema = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? '[]') - if (!stringified) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result)) + if (!mainSchema) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result)) let schema: Omit = { - schema: { main: JSON.parse(stringified) }, + schema: { main: mainSchema }, publicOnly: true, lang: 'ducklake' } diff --git a/frontend/src/lib/components/flows/FlowEditor.svelte b/frontend/src/lib/components/flows/FlowEditor.svelte index 775bd7cadb..0d7be9c358 100644 --- a/frontend/src/lib/components/flows/FlowEditor.svelte +++ b/frontend/src/lib/components/flows/FlowEditor.svelte @@ -102,6 +102,10 @@ return flowModuleSchemaMap?.isNodeVisible(nodeId) ?? false } + export function enableNotes(): void { + flowModuleSchemaMap?.enableNotes?.() + } + setContext('PropPickerContext', { flowPropPickerConfig: writable(undefined), pickablePropertiesFiltered: writable(undefined) diff --git a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte index 3e6d4e22af..f0288e6533 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorPanel.svelte @@ -15,6 +15,7 @@ import { computeMissingInputWarnings } from '../missingInputWarnings' import FlowResult from './FlowResult.svelte' import type { StateStore } from '$lib/utils' + import FlowSelectionPanel from './FlowSelectionPanel.svelte' interface Props { noEditor?: boolean @@ -55,7 +56,7 @@ }: Props = $props() const { - selectedId, + selectionManager, flowStore, flowStateStore, flowInputsStore, @@ -66,6 +67,8 @@ flowInputEditorState } = getContext('FlowEditorContext') + const selectedId = $derived(selectionManager.getSelectedId()) + const { showCaptureHint, triggersState, triggersCount } = getContext('TriggerContext') function checkDup(modules: FlowModule[]): string | undefined { @@ -84,14 +87,16 @@ }) -{#if $selectedId?.startsWith('settings')} +{#if selectionManager && selectionManager.selectedIds.length > 1} + +{:else if selectedId?.startsWith('settings')} -{:else if $selectedId === 'Input'} +{:else if selectedId === 'Input'} { - $selectedId = 'triggers' + selectionManager.selectId('Trigger') handleSelectTriggerFromKind(triggersState, triggersCount, savedFlow?.path, ev.detail.kind) showCaptureHint.set(true) }} @@ -99,22 +104,22 @@ {onTestFlow} {previewOpen} /> -{:else if $selectedId === 'Result'} +{:else if selectedId === 'Result'} -{:else if $selectedId === 'constants'} +{:else if selectedId === 'constants'} -{:else if $selectedId === 'failure'} +{:else if selectedId === 'failure'} -{:else if $selectedId === 'preprocessor'} +{:else if selectedId === 'preprocessor'} -{:else if $selectedId === 'triggers'} +{:else if selectedId === 'Trigger'} { await insertNewPreprocessorModule(flowStore, flowStateStore, { language: 'bun' }) - $selectedId = 'preprocessor' + selectionManager.selectId('preprocessor') }} on:updateSchema={(e) => { const { payloadData, redirect } = e.detail @@ -122,7 +127,7 @@ previewArgs.val = JSON.parse(JSON.stringify(payloadData)) } if (redirect) { - $selectedId = 'Input' + selectionManager.selectId('Input') $flowInputEditorState.selectedTab = 'captures' $flowInputEditorState.payloadData = payloadData } @@ -141,7 +146,7 @@ schema={flowStore.val.schema} {onDeployTrigger} /> -{:else if $selectedId.startsWith('subflow:')} +{:else if selectedId?.startsWith('subflow:')}
Selected step is witin an expanded subflow and is not directly editable in the flow editor
@@ -150,7 +155,7 @@ {#if dup}
There are duplicate modules in the flow at id: {dup}
{:else} - {#key $selectedId} + {#key selectedId} {#each flowStore.val.value.modules as flowModule, index (flowModule.id ?? index)} m.value.type === 'aiagent') - if (!hasAiAgent) { + + // Find all AI agent modules + const aiAgentModules = flowStore.val.value.modules.filter((m) => m.value.type === 'aiagent') + + if (aiAgentModules.length === 0) { + // No AI agent exists, create one with context memory set to 10 const aiAgentId = nextId(flowStateStore.val, flowStore.val) flowStore.val.value.modules = [ ...flowStore.val.value.modules, @@ -442,6 +446,8 @@ input_transforms: Object.keys(AI_AGENT_SCHEMA.properties ?? {}).reduce((accu, key) => { if (key === 'user_message') { accu[key] = { type: 'javascript', expr: 'flow_input.user_message' } + } else if (key === 'messages_context_length') { + accu[key] = { type: 'static', value: 10 } } else { accu[key] = { type: 'static', @@ -453,7 +459,34 @@ } } ] + sendUserToast( + 'Chat mode enabled. AI agent created with user message input and context memory set to 10.', + false + ) + } else if (aiAgentModules.length === 1) { + // Exactly one AI agent exists, configure it + const aiAgent = aiAgentModules[0] + const value = aiAgent.value as AiAgent + + // Set user_message to flow_input.user_message + value.input_transforms['user_message'] = { + type: 'javascript', + expr: 'flow_input.user_message' + } + + // Set messages_context_length to 10 + value.input_transforms['messages_context_length'] = { + type: 'static', + value: 10 + } + + sendUserToast( + 'Chat mode enabled. AI agent configured with user message input and context memory set to 10.', + false + ) } + // If there are multiple AI agents, don't auto-configure (ambiguous which one to configure) + showChatModeWarning = false } diff --git a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte index d7b1266868..c40f6465a3 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleComponent.svelte @@ -59,7 +59,7 @@ import { DynamicInput } from '$lib/utils' const { - selectedId, + selectionManager, currentEditor, previewArgs, flowStateStore, @@ -70,6 +70,8 @@ executionCount } = getContext('FlowEditorContext') + const selectedId = $derived(selectionManager.getSelectedId()) + interface Props { flowModule: FlowModule failureModule?: boolean @@ -215,7 +217,7 @@ let stepHistoryLoader = getStepHistoryLoaderContext() function onSelectedIdChange() { - if (!flowStateStore?.val?.[$selectedId]?.schema && flowModule) { + if (!flowStateStore?.val?.[selectedId]?.schema && flowModule) { reload(flowModule) } } @@ -252,7 +254,7 @@ ) $effect.pre(() => { - $selectedId && untrack(() => onSelectedIdChange()) + selectedId && untrack(() => onSelectedIdChange()) }) let parentLoop = $derived( flowStore.val && flowModule ? checkIfParentLoop(flowStore.val, flowModule.id) : undefined @@ -404,7 +406,7 @@ on:createScriptFromInlineScript={async () => { const [module, state] = await createScriptFromInlineScript( flowModule, - $selectedId, + selectedId, flowStateStore.val[flowModule.id].schema, $pathStore ) @@ -468,7 +470,7 @@ automaticLayout={true} cmdEnterAction={async () => { selected = 'test' - if ($selectedId == flowModule.id) { + if (selectedId == flowModule.id) { if (flowModule.value.type === 'rawscript' && editor) { flowModule.value.content = editor.getCode() } @@ -578,7 +580,7 @@ class="px-2 xl:px-4" bind:this={inputTransformSchemaForm} pickableProperties={stepPropPicker.pickableProperties} - schema={flowStateStore.val[$selectedId]?.schema ?? {}} + schema={flowStateStore.val[selectedId]?.schema ?? {}} previousModuleId={previousModule?.id} bind:args={ () => { @@ -609,7 +611,7 @@ bind:this={modulePreview} mod={flowModule} {noEditor} - schema={flowStateStore.val[$selectedId]?.schema ?? {}} + schema={flowStateStore.val[selectedId]?.schema ?? {}} bind:testJob bind:testIsLoading bind:scriptProgress @@ -623,7 +625,7 @@ active={flowModule.retry !== undefined} label="Retries" /> - {#if !$selectedId.includes('failure')} + {#if !selectedId.includes('failure')} { - $selectedId = 'settings-same-worker' + selectionManager.selectId('settings-same-worker') }} > Set shared directory in the flow settings diff --git a/frontend/src/lib/components/flows/content/FlowModuleSleep.svelte b/frontend/src/lib/components/flows/content/FlowModuleSleep.svelte index e9e15120fc..4a393893bf 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleSleep.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleSleep.svelte @@ -20,7 +20,7 @@ let { flowModule = $bindable(), previousModuleId }: Props = $props() - const { selectedId, flowStore, flowStateStore, previewArgs } = + const { selectionManager, flowStore, flowStateStore, previewArgs } = getContext('FlowEditorContext') let schema = $state(emptySchema()) schema.properties['sleep'] = { @@ -41,7 +41,7 @@ ) ) - const result = flowStateStore.val[$selectedId]?.previewResult ?? {} + const result = flowStateStore.val[selectionManager.getSelectedId()]?.previewResult ?? {} let isSleepEnabled = $derived(Boolean(flowModule.sleep)) diff --git a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte index fffc702a22..d1d946c5cd 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte @@ -18,8 +18,8 @@ import EditableSchemaDrawer from '$lib/components/schema/EditableSchemaDrawer.svelte' import AddProperty from '$lib/components/schema/AddProperty.svelte' - const { selectedId, flowStateStore } = getContext('FlowEditorContext') - const result = flowStateStore.val[$selectedId]?.previewResult ?? {} + const { selectionManager, flowStateStore } = getContext('FlowEditorContext') + const result = flowStateStore.val[selectionManager.getSelectedId()]?.previewResult ?? {} let editor: SimpleEditor | undefined = $state(undefined) interface Props { diff --git a/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte b/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte index 21b689206c..d724f241e8 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleWorkerTagSelect.svelte @@ -17,7 +17,7 @@ noLabel?: boolean } = $props() - const { flowStore, selectedId } = getContext('FlowEditorContext') + const { flowStore, selectionManager } = getContext('FlowEditorContext') const dispatch = createEventDispatcher() loadWorkerGroups() @@ -44,7 +44,7 @@ diff --git a/frontend/src/lib/components/flows/content/FlowModuleWrapper.svelte b/frontend/src/lib/components/flows/content/FlowModuleWrapper.svelte index 8af071e7f5..cb50e8a327 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleWrapper.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleWrapper.svelte @@ -23,7 +23,8 @@ import { formatCron } from '$lib/utils' import AgentToolWrapper from './AgentToolWrapper.svelte' - const { selectedId, flowStateStore } = getContext('FlowEditorContext') + const { selectionManager, flowStateStore } = getContext('FlowEditorContext') + const selectedId = $derived(selectionManager.getSelectedId()) const { triggersState, triggersCount } = getContext('TriggerContext') @@ -113,7 +114,7 @@ } -{#if flowModule.id === $selectedId} +{#if flowModule.id === selectedId} {#if flowModule.value.type === 'forloopflow'} {:else if flowModule.value.type === 'whileloopflow'} @@ -123,13 +124,13 @@ {:else if flowModule.value.type === 'branchall'} {:else if flowModule.value.type === 'identity'} - {#if $selectedId == 'failure'} + {#if selectedId == 'failure'}
If defined, the error handler will take the error as input.
- {:else if $selectedId == 'preprocessor'} + {:else if selectedId == 'preprocessor'}
{ const { path, summary, kind, hash } = detail createModuleFromScript(path, summary, kind, hash) @@ -187,8 +188,8 @@ flowModule = module flowStateStore.val[module.id] = state }} - failureModule={$selectedId === 'failure'} - preprocessorModule={$selectedId === 'preprocessor'} + failureModule={selectedId === 'failure'} + preprocessorModule={selectedId === 'preprocessor'} /> {/if} {:else if flowModule.value.type === 'rawscript' || flowModule.value.type === 'script' || flowModule.value.type === 'flow' || flowModule.value.type === 'aiagent'} @@ -197,8 +198,8 @@ bind:flowModule {parentModule} {previousModule} - failureModule={$selectedId === 'failure'} - preprocessorModule={$selectedId === 'preprocessor'} + failureModule={selectedId === 'failure'} + preprocessorModule={selectedId === 'preprocessor'} {scriptKind} {scriptTemplate} {enableAi} @@ -225,7 +226,7 @@ /> {/each} {:else if flowModule.value.type === 'branchone'} - {#if $selectedId === `${flowModule?.id}-branch-default`} + {#if selectedId === `${flowModule?.id}-branch-default`}

Default branch

Nothing to configure, this is the default branch if none of the predicates are met. @@ -247,7 +248,7 @@ {/each} {/if} {#each flowModule.value.branches as branch, branchIndex (branchIndex)} - {#if $selectedId === `${flowModule?.id}-branch-${branchIndex}`} + {#if selectedId === `${flowModule?.id}-branch-${branchIndex}`} {:else} {#each branch.modules as _, index} @@ -295,7 +296,7 @@ {/each} {:else if flowModule.value.type === 'aiagent'} {#each flowModule.value.tools as tool, toolIndex (toolIndex)} - {#if $selectedId === tool.id} + {#if selectedId === tool.id} + import FlowCard from '../common/FlowCard.svelte' + import type { SelectionManager } from '$lib/components/graph/selectionUtils.svelte' + import { Button } from '$lib/components/common' + import { getNoteEditorContext } from '$lib/components/graph/noteEditor.svelte' + import { StickyNote } from 'lucide-svelte' + + interface Props { + selectionManager: SelectionManager + noEditor: boolean + } + let { selectionManager, noEditor }: Props = $props() + + const noteEditorContext = getNoteEditorContext() + + function addGroupNote() { + if (selectionManager.selectedIds.length > 0 && noteEditorContext?.noteEditor) { + // Create the group note + noteEditorContext.noteEditor.createGroupNote(selectionManager.selectedIds) + } + } + + + + {#snippet action()} + + {/snippet} +
+

{selectionManager.selectedIds.length} nodes selected

+
+ {#each selectionManager.selectedIds as nodeId} +
+ {nodeId} +
+ {/each} +
+
+ diff --git a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte index d349b05500..64c4416d45 100644 --- a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte +++ b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte @@ -25,7 +25,7 @@ localModuleStates = $bindable({}) }: Props = $props() - const { selectedId } = getContext('FlowEditorContext') + const { selectionManager } = getContext('FlowEditorContext') let flowPreviewContent: FlowPreviewContent | undefined = $state(undefined) let preventEscape = $state(false) @@ -70,7 +70,7 @@ $state('timeline') let upToDisabled = $derived.by(() => { - const upToSelected = upToId ?? $selectedId + const upToSelected = upToId ?? selectionManager.getSelectedId() return ( upToSelected == undefined || [ @@ -92,7 +92,7 @@ 'constants', 'Result', 'Input', - 'triggers' + 'Trigger' ].includes(upToSelected) || upToSelected?.includes('branch') || aiChatManager.flowAiChatHelpers?.getModuleAction(upToSelected) === 'removed' @@ -144,8 +144,8 @@ dropdownItems={!upToDisabled ? [ { - label: 'Test up to ' + $selectedId, - onClick: () => testUpTo($selectedId, true) + label: 'Test up to ' + selectionManager.getSelectedId(), + onClick: () => testUpTo(selectionManager.getSelectedId(), true) } ] : undefined} diff --git a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte index 2a80a34f37..c9d681da50 100644 --- a/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowErrorHandlerItem.svelte @@ -29,7 +29,7 @@ generateStep: { moduleId: string; instructions: string; lang: ScriptLang } }>() - const { selectedId, flowStateStore, flowStore } = + const { selectionManager, flowStateStore, flowStore } = getContext('FlowEditorContext') async function insertFailureModule( @@ -50,7 +50,7 @@ }) } - $selectedId = 'failure' + selectionManager.selectId('failure') refreshStateStore(flowStore) } @@ -70,10 +70,10 @@ aiModuleActionToTextColor(action) )} id="flow-editor-error-handler" - selected={$selectedId?.includes('failure')} + selected={selectionManager.getSelectedId()?.includes('failure')} onClick={() => { if (flowStore.val?.value?.failure_module) { - $selectedId = 'failure' + selectionManager.selectId('failure') } }} > @@ -95,7 +95,7 @@ class="ml-1" onclick={() => { flowStore.val.value.failure_module = undefined - $selectedId = 'settings-metadata' + selectionManager.selectId('settings-metadata') }} > diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 494715ad29..75a12d1e11 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -281,7 +281,7 @@ style="width: 275px; height: 34px;" onmouseenter={() => (hover = true)} onmouseleave={() => (hover = false)} - onpointerdown={stopPropagation(preventDefault(() => dispatch('pointerdown')))} + onpointerdown={stopPropagation(preventDefault((e) => dispatch('pointerdown', e)))} > {#if deletable} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 0b0bcdb778..eab09789b6 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -42,6 +42,7 @@ import { ModulesTestStates } from '$lib/components/modulesTest.svelte' import type { StateStore } from '$lib/utils' import { type AgentTool, flowModuleToAgentTool, createMcpTool } from '../agentToolUtils' + import { getNoteEditorContext } from '$lib/components/graph/noteEditor.svelte' interface Props { sidebarSize?: number | undefined @@ -105,12 +106,15 @@ let flowTutorials: FlowTutorials | undefined = $state(undefined) - const { customUi, selectedId, moving, history, flowStateStore, flowStore, pathStore } = + const { customUi, selectionManager, moving, history, flowStateStore, flowStore, pathStore } = getContext('FlowEditorContext') const { triggersCount, triggersState } = getContext('TriggerContext') const { flowPropPickerConfig } = getContext('PropPickerContext') + // Get NoteEditor context for note position updates + const noteEditorContext = getNoteEditorContext() + export async function insertNewModuleAtIndex( modules: FlowModule[] | AgentTool[], index: number, @@ -238,9 +242,9 @@ let allIds = dfs(flowStore.val.value.modules, (mod) => mod.id) if (allIds.length > 1) { const idx = allIds.indexOf(id) - $selectedId = idx == 0 ? allIds[0] : allIds[idx - 1] + selectionManager.selectId(idx == 0 ? allIds[0] : allIds[idx - 1]) } else { - $selectedId = 'settings-metadata' + selectionManager.selectId('settings-metadata') } } } @@ -290,10 +294,19 @@ let dependents: Record = $state({}) let graph: FlowGraphV2 | undefined = $state(undefined) + let noteMode = $state(false) export function isNodeVisible(nodeId: string): boolean { return graph?.isNodeVisible(nodeId) ?? false } + export function enableNotes(): void { + graph?.enableNotes?.() + } + + function toggleNoteMode() { + noteMode = !noteMode + } + function shouldRunTutorial(tutorialName: string, name: string, index: number) { return ( $tutorialsToDo.includes(index) && @@ -400,6 +413,8 @@ on:generateStep {aiChatOpen} {toggleAiChat} + {noteMode} + {toggleNoteMode} />
@@ -418,8 +433,10 @@ moving={$moving?.id} maxHeight={minHeight} modules={flowStore.val.value.modules} + {noteMode} + notes={flowStore.val.value.notes} preprocessorModule={flowStore.val.value?.preprocessor_module} - {selectedId} + {selectionManager} {workspace} editMode {onTestUpTo} @@ -438,7 +455,7 @@ const cb = () => { push(history, flowStore.val) if (id === 'preprocessor') { - $selectedId = 'Input' + selectionManager.selectId('Input') flowStore.val.value.preprocessor_module = undefined } else { selectNextId(id) @@ -467,7 +484,7 @@ let targetModules if ( detail.sourceId == 'Input' || - detail.targetId == 'result' || + detail.targetId == 'Result' || detail.kind == 'trigger' ) { targetModules = flowStore.val.value.modules @@ -497,7 +514,7 @@ let [removedModule] = originalModules.splice(indexToRemove, 1) targetModules.splice(detail.index, 0, removedModule) - $selectedId = removedModule.id + selectionManager.selectId(removedModule.id) $moving = undefined } else { if (detail.isPreprocessor) { @@ -507,7 +524,7 @@ detail.inlineScript, detail.script ) - $selectedId = 'preprocessor' + selectionManager.selectId('preprocessor') if (detail.inlineScript?.instructions) { dispatch('generateStep', { @@ -534,7 +551,7 @@ toolKind ) const id = targetModules[index].id - $selectedId = id + selectionManager.selectId(id) if (detail.inlineScript?.instructions) { dispatch('generateStep', { @@ -619,13 +636,13 @@ flowStateStore.val[newId] = flowStateStore.val[id] delete flowStateStore.val[id] refreshStateStore(flowStore) - $selectedId = newId + selectionManager.selectId(newId) }} onDeleteBranch={async ({ id, index }) => { if (id) { await removeBranch(id, index) refreshStateStore(flowStore) - $selectedId = id + selectionManager.selectId(id) } }} onMove={(id) => { @@ -645,6 +662,14 @@ {onCancelTestFlow} {onOpenPreview} {onHideJobStatus} + exitNoteMode={() => (noteMode = false)} + onNotePositionUpdate={(noteId, position) => { + // Update note position via NoteEditor context in edit mode + if (noteEditorContext?.noteEditor) { + noteEditorContext.noteEditor.updatePosition(noteId, position) + } + }} + multiSelectEnabled />
diff --git a/frontend/src/lib/components/flows/map/FlowStickyNode.svelte b/frontend/src/lib/components/flows/map/FlowStickyNode.svelte index 48e7445593..b1821f98e3 100644 --- a/frontend/src/lib/components/flows/map/FlowStickyNode.svelte +++ b/frontend/src/lib/components/flows/map/FlowStickyNode.svelte @@ -2,7 +2,7 @@ import type { FlowEditorContext } from '../types' import { getContext } from 'svelte' import { Badge } from '$lib/components/common' - import { DollarSign, Settings } from 'lucide-svelte' + import { DollarSign, Settings, StickyNote } from 'lucide-svelte' import FlowErrorHandlerItem from './FlowErrorHandlerItem.svelte' import FlowAIButton from '$lib/components/copilot/chat/flow/FlowAIButton.svelte' import Popover from '$lib/components/Popover.svelte' @@ -15,6 +15,8 @@ aiChatOpen?: boolean showFlowAiButton?: boolean toggleAiChat?: () => void + noteMode?: boolean + toggleNoteMode?: () => void disableAi?: boolean } @@ -25,10 +27,13 @@ aiChatOpen, showFlowAiButton, toggleAiChat, + noteMode, + toggleNoteMode, disableAi }: Props = $props() - const { selectedId, flowStore } = getContext('FlowEditorContext') + const { selectionManager, flowStore } = getContext('FlowEditorContext') + const selectedId = $derived(selectionManager.getSelectedId())
@@ -37,10 +42,10 @@ unifiedSize="sm" wrapperClasses="min-w-36" startIcon={{ icon: Settings }} - selected={$selectedId?.startsWith('settings')} + selected={selectedId?.startsWith('settings')} variant="default" title="Settings" - onClick={() => ($selectedId = 'settings')} + onClick={() => selectionManager.selectId('settings')} > Settings {#if flowStore.val.value.same_worker} @@ -60,10 +65,10 @@ wrapperClasses="h-full" unifiedSize="sm" startIcon={{ icon: DollarSign }} - selected={$selectedId === 'constants'} + selected={selectedId === 'constants'} variant="default" iconOnly - onClick={() => ($selectedId = 'constants')} + onClick={() => selectionManager.selectId('constants')} /> {#snippet text()} Environment Variables @@ -83,4 +88,17 @@ {/snippet} {/if} + + + {#snippet text()} + {noteMode ? 'Exit note mode' : 'Add sticky notes'} + {/snippet} +
diff --git a/frontend/src/lib/components/flows/map/MapItem.svelte b/frontend/src/lib/components/flows/map/MapItem.svelte index 2021b10e91..08d88f26aa 100644 --- a/frontend/src/lib/components/flows/map/MapItem.svelte +++ b/frontend/src/lib/components/flows/map/MapItem.svelte @@ -2,7 +2,6 @@ import { Button } from '$lib/components/common' import type { FlowModule, Job } from '$lib/gen' import { createEventDispatcher, getContext } from 'svelte' - import type { Writable } from 'svelte/store' import FlowModuleSchemaItem from './FlowModuleSchemaItem.svelte' import FlowModuleIcon from '../FlowModuleIcon.svelte' import { prettyLanguage } from '$lib/common' @@ -17,6 +16,7 @@ import { twMerge } from 'tailwind-merge' import type { FlowNodeState } from '$lib/components/graph' import type { AIModuleAction } from '$lib/components/copilot/chat/flow/core' + import { getGraphContext } from '$lib/components/graph/graphContext' interface Props { moduleId: string @@ -74,9 +74,7 @@ maximizeSubflow }: Props = $props() - const { selectedId } = getContext<{ - selectedId: Writable - }>('FlowGraphContext') + const { selectionManager } = getGraphContext() const { flowStore } = getContext('FlowEditorContext') || {} @@ -88,7 +86,7 @@ }>() let itemProps = $derived({ - selected: $selectedId === mod.id, + selected: selectionManager && selectionManager.isNodeSelected(mod.id), retry: mod.retry?.constant != undefined || mod.retry?.exponential != undefined, earlyStop: mod.stop_after_if != undefined || mod.stop_after_all_iters_if != undefined, skip: Boolean(mod.skip_if), @@ -102,6 +100,13 @@ let parentLoop = $derived( flowStore?.val && mod ? checkIfParentLoop(flowStore.val, mod.id) : undefined ) + + function handlePointerDown(e: CustomEvent) { + // Only handle left clicks (button 0) + if (e.detail.button === 0) { + onSelect(mod.id) + } + } {#if mod} @@ -164,7 +169,7 @@ on:changeId on:move on:delete - on:pointerdown={() => onSelect(mod.id)} + on:pointerdown={handlePointerDown} onUpdateMock={(mock) => { mod.mock = mock onUpdateMock?.({ id: mod.id, mock }) @@ -193,7 +198,7 @@ on:changeId on:delete on:move - on:pointerdown={() => onSelect(mod.id)} + on:pointerdown={handlePointerDown} {...itemProps} id={mod.id} label={mod.summary || 'Run one branch'} @@ -213,7 +218,7 @@ on:changeId on:delete on:move - on:pointerdown={() => onSelect(mod.id)} + on:pointerdown={handlePointerDown} id={mod.id} {...itemProps} label={mod.summary || `Run all branches${mod.value.parallel ? ' (parallel)' : ''}`} @@ -231,7 +236,7 @@ {moduleAction} {onShowModuleDiff} on:changeId - on:pointerdown={() => onSelect(mod.id)} + on:pointerdown={handlePointerDown} on:delete on:move onUpdateMock={(mock) => { diff --git a/frontend/src/lib/components/flows/types.ts b/frontend/src/lib/components/flows/types.ts index 733cd304aa..6ed09a1436 100644 --- a/frontend/src/lib/components/flows/types.ts +++ b/frontend/src/lib/components/flows/types.ts @@ -15,6 +15,8 @@ import type ResourceEditorDrawer from '../ResourceEditorDrawer.svelte' import type { ModulesTestStates } from '../modulesTest.svelte' import type { ButtonProp } from '$lib/components/DiffEditor.svelte' +import type { SelectionManager } from '../graph/selectionUtils.svelte' + export type FlowInput = Record< string, { @@ -28,6 +30,7 @@ export type FlowInput = Record< } > +// Extended OpenFlow with additional properties not in the core spec export type ExtendedOpenFlow = OpenFlow & { tag?: string ws_error_handler_muted?: boolean @@ -68,7 +71,7 @@ export type CurrentEditor = | undefined export type FlowEditorContext = { - selectedId: Writable + selectionManager: SelectionManager currentEditor: Writable moving: Writable<{ id: string } | undefined> previewArgs: StateStore> diff --git a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts index c23ccf590e..33c2f2fa5c 100644 --- a/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts +++ b/frontend/src/lib/components/git_sync/GitSyncContext.svelte.ts @@ -595,7 +595,7 @@ export function createGitSyncContext(workspace: string) { return result } - function getLegacyPromotionRepositories(): { repo: GitSyncRepository, idx: number }[] { + function getSecondaryPromotionRepositories(): { repo: GitSyncRepository, idx: number }[] { const result: { repo: GitSyncRepository, idx: number }[] = [] let foundFirst = false repositories.forEach((repo, idx) => { @@ -736,7 +736,7 @@ export function createGitSyncContext(workspace: string) { getPrimarySyncRepository, getPrimaryPromotionRepository, getSecondarySyncRepositories, - getLegacyPromotionRepositories, + getSecondaryPromotionRepositories, // Helper methods getTargetBranch, diff --git a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte index 85c19362d7..c5700622d4 100644 --- a/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncRepositoryCard.svelte @@ -94,10 +94,19 @@ : isLegacy ? 'Legacy promotion repository' : isSecondary - ? 'Secondary sync repository' + ? repo?.use_individual_branch + ? 'Secondary promotion repository' + : 'Secondary sync repository' : `Repository #${(idx ?? 0) + 1}` ) + // Determine the actual mode based on repository configuration + const repoMode = $derived<'sync' | 'promotion'>( + variant === 'primary-promotion' || variant === 'legacy' || repo?.use_individual_branch + ? 'promotion' + : 'sync' + ) + // Determine display description based on variant and mode const displayDescription = $derived( variant === 'primary-sync' || variant === 'primary-promotion' @@ -353,7 +362,7 @@ {#if repo.isUnsavedConnection && !emptyString(repo.git_repo_resource_path) && idx !== null} {:else}
diff --git a/frontend/src/lib/components/git_sync/GitSyncSection.svelte b/frontend/src/lib/components/git_sync/GitSyncSection.svelte index 41269a82bd..52b4b4326d 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSection.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSection.svelte @@ -30,14 +30,15 @@ const primarySync = $derived(gitSyncContext?.getPrimarySyncRepository() || null) const primaryPromotion = $derived(gitSyncContext?.getPrimaryPromotionRepository() || null) const secondarySync = $derived(gitSyncContext?.getSecondarySyncRepositories() || []) - const legacyPromotion = $derived(gitSyncContext?.getLegacyPromotionRepositories() || []) + const secondaryPromotion = $derived(gitSyncContext?.getSecondaryPromotionRepositories() || []) // State for collapsible sections let secondarySyncExpanded = $state(false) - let legacyPromotionExpanded = $state(false) + let secondaryPromotionExpanded = $state(false) // Check if any secondary repositories are unsaved const hasUnsavedSecondary = $derived(secondarySync.some((s) => s.repo.isUnsavedConnection)) + const hasUnsavedSecondaryPromotion = $derived(secondaryPromotion.some((s) => s.repo.isUnsavedConnection)) {#if !gitSyncContext} @@ -169,38 +170,70 @@ isCollapsible={false} showEmptyState={primaryPromotion?.repo === null} /> -
- - {#if legacyPromotion.length > 0} - - Multiple promotion repositories are no longer supported. Please reduce to a single - promotion repository. Only deletion is allowed for the additional repositories below. - -
- + + {#if primaryPromotion && !primaryPromotion.repo?.isUnsavedConnection} + {#if secondaryPromotion.length > 0 || secondaryPromotionExpanded} +
+ - {#if legacyPromotionExpanded} -
- {#each legacyPromotion as { repo, idx } (repo.git_repo_resource_path)} -
- + {#if secondaryPromotionExpanded} +
+ {#if secondaryPromotion.length === 0} +
+ No secondary promotion repositories configured +
+ {:else} + {#each secondaryPromotion as { repo, idx } (repo.git_repo_resource_path)} +
+ +
+ {/each} + {/if} + + {#if !hasUnsavedSecondaryPromotion} +
+ +
+ {/if}
- {/each} + {/if}
+ {:else} + + {#if !hasUnsavedSecondaryPromotion} +
+ +
+ {/if} {/if} -
- {/if} + {/if} +
diff --git a/frontend/src/lib/components/graph/FlowGraphV2.svelte b/frontend/src/lib/components/graph/FlowGraphV2.svelte index 3b41cf2035..9be2534b9f 100644 --- a/frontend/src/lib/components/graph/FlowGraphV2.svelte +++ b/frontend/src/lib/components/graph/FlowGraphV2.svelte @@ -1,7 +1,7 @@ {#if insertable} @@ -589,8 +819,9 @@ {/if}
{#if graph?.error}
@@ -615,14 +846,29 @@ bind:this={viewportSynchronizer} /> {/if} + { + onpaneclick={() => { document.dispatchEvent(new Event('focus')) + selectionManager.clearSelection() + }} + onpanecontextmenu={({ event }) => { + paneContextMenu?.onPaneContextMenu(event) + }} + onnodedragstop={(event) => { + const node = event.targetNode + if (node && node.type === 'note') { + const positionWithOffset = { + x: node.position.x, + y: node.position.y - yOffset + } + onNotePositionUpdate?.(node.id, positionWithOffset) + } }} onmove={(event, viewport) => { viewportSynchronizer?.handleLocalViewportChange(event, viewport) }} - {nodes} + nodes={nodesWithOffset} {edges} {edgeTypes} {nodeTypes} @@ -633,26 +879,85 @@ connectionLineType={ConnectionLineType.SmoothStep} defaultEdgeOptions={{ type: 'smoothstep' }} preventScrolling={scroll} + selectionOnDrag={selectionManager.mode === 'rect-select'} + elementsSelectable={true} + selectionMode={SelectionMode.Partial} + selectionKey={selectionManager.mode === 'rect-select' || !editMode ? null : modifierKey} + panActivationKey={selectionManager.mode === 'rect-select' ? modifierKey : null} + panOnDrag={selectionManager.mode === 'rect-select' ? [1] : true} zoomOnDoubleClick={false} - elementsSelectable={false} + elevateNodesOnSelect={false} {proOptions} + multiSelectionKey={'Shift'} nodesDraggable={false} --background-color={false} >
+ + {#if noteMode} + + {/if} + + {#if multiSelectEnabled} + + {/if} + + + + {#if leftHeader}
{@render leftHeader()}
{:else} + {#if multiSelectEnabled} +
+ + { + selectionManager.mode = + selectionManager.mode === 'normal' ? 'rect-select' : 'normal' + }} + > + {#if selectionManager.mode === 'rect-select'} + + {:else} + + {/if} + + {#snippet text()} +
+
+ + Grab: Click and drag to pan. Hold + {getModifierKey()} to box select. +
+
+ + Select Click and drag to box + select. Hold + {getModifierKey()} to pan. +
+
+ {/snippet} + +
+ {/if} {#if download} { try { localStorage.setItem( 'svelvet', - encodeState({ modules, failureModule, preprocessorModule }) + encodeState({ modules, failureModule, preprocessorModule, notes }) ) } catch (e) { console.error('error interacting with local storage', e) @@ -678,6 +983,9 @@ {#if !hideAssetsToggle} {/if} + {#if !hideNotesToggle} + + {/if} {#if showDataflow} {/if} @@ -703,4 +1011,9 @@ :global(.svelte-flow__edgelabel-renderer) { @apply z-50; } + + :global(.svelte-flow__selection) { + display: none; + pointer-events: none; + } diff --git a/frontend/src/lib/components/graph/NodeContextMenu.svelte b/frontend/src/lib/components/graph/NodeContextMenu.svelte new file mode 100644 index 0000000000..034fc2d7da --- /dev/null +++ b/frontend/src/lib/components/graph/NodeContextMenu.svelte @@ -0,0 +1,47 @@ + + +{#if noteEditorContext?.noteEditor && selectedNodeIds.length > 1} + + {@render children()} + +{/if} diff --git a/frontend/src/lib/components/graph/NoteColorPicker.svelte b/frontend/src/lib/components/graph/NoteColorPicker.svelte new file mode 100644 index 0000000000..d17adb9317 --- /dev/null +++ b/frontend/src/lib/components/graph/NoteColorPicker.svelte @@ -0,0 +1,50 @@ + + + + {#snippet trigger()} + + {/each} +
+ {/snippet} + diff --git a/frontend/src/lib/components/graph/NoteTool.svelte b/frontend/src/lib/components/graph/NoteTool.svelte new file mode 100644 index 0000000000..ad41bd974a --- /dev/null +++ b/frontend/src/lib/components/graph/NoteTool.svelte @@ -0,0 +1,216 @@ + + + +
{ + // Capture the position when context menu is triggered + const flowPosition = screenToFlowPosition({ + x: e.clientX, + y: e.clientY + }) + contextMenuPosition = { + x: flowPosition.x, + y: flowPosition.y - yOffset + } + }} + role="button" + tabindex="0" + aria-label="Click and drag to create a note, or right-click to add a sticky note" + onkeydown={(e) => { + if (e.key === 'Escape') { + if (isDrawing) { + // Cancel current drawing + isDrawing = false + startPosition = null + } else { + // Exit note mode + exitNoteMode?.() + } + } + }} + > + + {#if previewNote} +
+
+ {/if} +
+ + + diff --git a/frontend/src/lib/components/graph/PaneContextMenu.svelte b/frontend/src/lib/components/graph/PaneContextMenu.svelte new file mode 100644 index 0000000000..b6b6b49ac9 --- /dev/null +++ b/frontend/src/lib/components/graph/PaneContextMenu.svelte @@ -0,0 +1,115 @@ + + +{#if contextMenuVisible} + + + + + +{/if} diff --git a/frontend/src/lib/components/graph/SelectionBoundingBox.svelte b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte new file mode 100644 index 0000000000..5faff92d28 --- /dev/null +++ b/frontend/src/lib/components/graph/SelectionBoundingBox.svelte @@ -0,0 +1,82 @@ + + +{#if bounds() && selectedNodes.length > 1} + {@const currentBounds = bounds()!} + +
+ + {#if noteEditorContext?.noteEditor} +
+ +
+ {/if} +
+ +{/if} diff --git a/frontend/src/lib/components/graph/SelectionTool.svelte b/frontend/src/lib/components/graph/SelectionTool.svelte new file mode 100644 index 0000000000..e7af55bc24 --- /dev/null +++ b/frontend/src/lib/components/graph/SelectionTool.svelte @@ -0,0 +1,43 @@ + + + +{#if store.selectionRect} + {@const bounds = store.selectionRect!} +
+
+{/if} diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index a6f0ebdd6a..6dded81b9c 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -88,6 +88,7 @@ export type NodeLayout = { data: { offset?: number } + selectable?: boolean } & FlowNode export type FlowNode = @@ -447,7 +448,8 @@ export function graphBuilder( moduleAction: extra.moduleActions?.[module.id], onShowModuleDiff: extra.onShowModuleDiff }, - type: 'module' + type: 'module', + selectable: true }) return module.id @@ -540,7 +542,8 @@ export function graphBuilder( ...extra, insertable: extra.insertable && !options?.disableInsert && prefix == undefined, shouldOffsetInsertBtnDueToAssetNode: nodeIdsWithOutputAssets.has(sourceId) - } + }, + selectable: false }) } @@ -596,7 +599,7 @@ export function graphBuilder( } const resultNode: NodeLayout = { - id: 'result', + id: 'Result', data: { eventHandlers: eventHandlers, success: success, @@ -1085,14 +1088,14 @@ export function graphBuilder( let pid = x[0] if (input?.startsWith('flow_input.iter')) { - const parent = dfsByModule(selectedId!, modules ?? [])?.pop() + const parent = dfsByModule(selectedId, modules ?? [])?.pop() if (parent?.id) { pid = parent.id } } - addEdge(pid, selectedId!, undefined, undefined, { + addEdge(pid, selectedId, undefined, undefined, { customId: `dep-${pid}-${selectedId}-${input}-${index}`, type: 'dataflowedge' }) @@ -1102,7 +1105,7 @@ export function graphBuilder( Object.entries(deps.dependents).forEach((x, i) => { let pid = x[0] - addEdge(selectedId!, pid, undefined, undefined, { + addEdge(selectedId, pid, undefined, undefined, { customId: `dep-${selectedId}-${pid}-${i}`, type: 'dataflowedge' }) diff --git a/frontend/src/lib/components/graph/graphContext.ts b/frontend/src/lib/components/graph/graphContext.ts new file mode 100644 index 0000000000..176fc5d3bf --- /dev/null +++ b/frontend/src/lib/components/graph/graphContext.ts @@ -0,0 +1,19 @@ +import { getContext, setContext } from 'svelte' +import type { SelectionManager } from './selectionUtils.svelte' +import type { NoteManager } from './noteManager.svelte' +import type { Writable } from 'svelte/store' + +export type GraphContext = { + selectionManager: SelectionManager + useDataflow: Writable + showAssets: Writable + noteManager?: NoteManager + clearFlowSelection?: () => void + yOffset?: number +} + +const graphContextKey = 'FlowGraphContext' + +//TODO: use https://svelte.dev/docs/svelte/context#Type-safe-context after migrating svelte 5 to latest version +export const getGraphContext = () => getContext(graphContextKey) +export const setGraphContext = (context: GraphContext) => setContext(graphContextKey, context) diff --git a/frontend/src/lib/components/graph/groupDetectionUtils.ts b/frontend/src/lib/components/graph/groupDetectionUtils.ts new file mode 100644 index 0000000000..e2c81c4d98 --- /dev/null +++ b/frontend/src/lib/components/graph/groupDetectionUtils.ts @@ -0,0 +1,86 @@ +type FlowNode = { id: string; parentIds?: string[] } + +/** + * Use a simple algorithm to complete a group and split it into connected components + */ +export function completeAndSplitGroup(groupNodes: string[], flowNodes: FlowNode[]): string[][] { + if (groupNodes.length <= 1) { + return groupNodes.length === 1 ? [groupNodes] : [] + } + + // Build parent map for upward traversal only + const parents = new Map() + for (const node of flowNodes) { + parents.set(node.id, node.parentIds || []) + } + + const groupSet = new Set(groupNodes) + const assignedComponent = new Map() + const components: Array> = [] + + const mergeComponents = (fromIdx: number, toIdx: number): void => { + if (fromIdx === toIdx) return + const target = components[toIdx] + const source = components[fromIdx] + + source.forEach((node) => target.add(node)) + source.clear() + + for (const [nodeId, idx] of assignedComponent.entries()) { + if (idx === fromIdx) { + assignedComponent.set(nodeId, toIdx) + } + } + } + + for (const startNode of groupNodes) { + if (assignedComponent.has(startNode)) continue + + const componentIdx = components.length + components.push(new Set([startNode])) + assignedComponent.set(startNode, componentIdx) + + const stack: { nodeId: string; path: string[]; seen: Set }[] = [ + { nodeId: startNode, path: [startNode], seen: new Set([startNode]) } + ] + + while (stack.length > 0) { + const { nodeId, path, seen } = stack.pop()! + const parentIds = parents.get(nodeId) || [] + + for (const parentId of parentIds) { + if (seen.has(parentId)) continue + + const newPath = [...path, parentId] + const newSeen = new Set(seen) + newSeen.add(parentId) + + if (groupSet.has(parentId)) { + const existingIdx = assignedComponent.get(parentId) + if (existingIdx === undefined) { + assignedComponent.set(parentId, componentIdx) + components[componentIdx].add(parentId) + stack.push({ nodeId: parentId, path: [parentId], seen: new Set([parentId]) }) + } else if (existingIdx !== componentIdx) { + mergeComponents(existingIdx, componentIdx) + } + + for (const node of newPath) { + components[componentIdx].add(node) + } + } else { + stack.push({ nodeId: parentId, path: newPath, seen: newSeen }) + } + } + } + } + + return components + .filter((component) => component.size > 0) + .map((component) => + Array.from(component) + .filter((nodeId) => !nodeId.startsWith('subflow:')) + .sort() + ) + .filter((component) => component.length > 0) +} diff --git a/frontend/src/lib/components/graph/noteColors.ts b/frontend/src/lib/components/graph/noteColors.ts new file mode 100644 index 0000000000..f9024adf7e --- /dev/null +++ b/frontend/src/lib/components/graph/noteColors.ts @@ -0,0 +1,135 @@ +// Note color definitions with Tailwind classes for light and dark mode +export enum NoteColor { + YELLOW = 'yellow', + BLUE = 'blue', + GREEN = 'green', + PURPLE = 'purple', + PINK = 'pink', + ORANGE = 'orange', + RED = 'red', + CYAN = 'cyan', + LIME = 'lime', + GRAY = 'gray' +} + +export interface NoteColorConfig { + background: string + outline: string + outlineHover: string + text: string + hover: string +} + +// Color configurations for each note color with dark mode support +export const NOTE_COLORS: Record = { + [NoteColor.YELLOW]: { + background: 'bg-yellow-200 dark:bg-yellow-900', + outline: 'outline-yellow-300 dark:outline-yellow-600', + outlineHover: 'outline-yellow-300/60 dark:outline-yellow-600/60', + text: 'text-yellow-900 dark:text-yellow-100', + hover: 'hover:bg-yellow-200 dark:hover:bg-yellow-800' + }, + [NoteColor.BLUE]: { + background: 'bg-blue-100 dark:bg-blue-950', + outline: 'outline-blue-300 dark:outline-blue-600', + outlineHover: 'outline-blue-300/60 dark:outline-blue-600/60', + text: 'text-blue-900 dark:text-blue-100', + hover: 'hover:bg-blue-200 dark:hover:bg-blue-800' + }, + [NoteColor.GREEN]: { + background: 'bg-green-200 dark:bg-green-900', + outline: 'outline-green-300 dark:outline-green-600', + outlineHover: 'outline-green-300/60 dark:outline-green-600/60', + text: 'text-green-900 dark:text-green-100', + hover: 'hover:bg-green-200 dark:hover:bg-green-800' + }, + [NoteColor.PURPLE]: { + background: 'bg-purple-200 dark:bg-purple-900', + outline: 'outline-purple-300 dark:outline-purple-600', + outlineHover: 'outline-purple-300/60 dark:outline-purple-600/60', + text: 'text-purple-900 dark:text-purple-100', + hover: 'hover:bg-purple-200 dark:hover:bg-purple-800' + }, + [NoteColor.PINK]: { + background: 'bg-pink-200 dark:bg-pink-900', + outline: 'outline-pink-300 dark:outline-pink-600', + outlineHover: 'outline-pink-300/60 dark:outline-pink-600/60', + text: 'text-pink-900 dark:text-pink-100', + hover: 'hover:bg-pink-200 dark:hover:bg-pink-800' + }, + [NoteColor.ORANGE]: { + background: 'bg-orange-200 dark:bg-orange-900', + outline: 'outline-orange-300 dark:outline-orange-600', + outlineHover: 'outline-orange-300/60 dark:outline-orange-600/60', + text: 'text-orange-900 dark:text-orange-100', + hover: 'hover:bg-orange-200 dark:hover:bg-orange-800' + }, + [NoteColor.RED]: { + background: 'bg-red-200 dark:bg-red-900', + outline: 'outline-red-300 dark:outline-red-600', + outlineHover: 'outline-red-300/60 dark:outline-red-600/60', + text: 'text-red-900 dark:text-red-100', + hover: 'hover:bg-red-200 dark:hover:bg-red-800' + }, + [NoteColor.CYAN]: { + background: 'bg-cyan-200 dark:bg-cyan-900', + outline: 'outline-cyan-300 dark:outline-cyan-600', + outlineHover: 'outline-cyan-300/60 dark:outline-cyan-600/60', + text: 'text-cyan-900 dark:text-cyan-100', + hover: 'hover:bg-cyan-200 dark:hover:bg-cyan-800' + }, + [NoteColor.LIME]: { + background: 'bg-lime-200 dark:bg-lime-900', + outline: 'outline-lime-300 dark:outline-lime-600', + outlineHover: 'outline-lime-300/60 dark:outline-lime-600/60', + text: 'text-lime-900 dark:text-lime-100', + hover: 'hover:bg-lime-200 dark:hover:bg-lime-800' + }, + [NoteColor.GRAY]: { + background: 'bg-gray-200 dark:bg-gray-800', + outline: 'outline-gray-300 dark:outline-gray-600', + outlineHover: 'outline-gray-300/60 dark:outline-gray-600/60', + text: 'text-gray-900 dark:text-gray-100', + hover: 'hover:bg-gray-200 dark:hover:bg-gray-700' + } +} + +// Color swatch colors for the picker (solid colors for the palette dots) +export const NOTE_COLOR_SWATCHES: Record = { + [NoteColor.YELLOW]: 'bg-yellow-400', + [NoteColor.BLUE]: 'bg-blue-400', + [NoteColor.GREEN]: 'bg-green-400', + [NoteColor.PURPLE]: 'bg-purple-400', + [NoteColor.PINK]: 'bg-pink-400', + [NoteColor.ORANGE]: 'bg-orange-400', + [NoteColor.RED]: 'bg-red-400', + [NoteColor.CYAN]: 'bg-cyan-400', + [NoteColor.LIME]: 'bg-lime-400', + [NoteColor.GRAY]: 'bg-gray-400' +} + +// Default note color +export const DEFAULT_NOTE_COLOR = NoteColor.GREEN +export const DEFAULT_GROUP_NOTE_COLOR = NoteColor.BLUE + +/** + * Get the next available color that's not in the used colors set + * Cycles through all available colors in order + */ +export function getNextAvailableColor(usedColors: Set): NoteColor { + const allColors = Object.values(NoteColor) + + // Find first unused color + for (const color of allColors) { + if (!usedColors.has(color)) { + return color + } + } + + // If all colors are used, return the default + return DEFAULT_GROUP_NOTE_COLOR +} + +// Minimum note size constraints +export const MIN_NOTE_WIDTH = 275 +export const MIN_NOTE_HEIGHT = 60 diff --git a/frontend/src/lib/components/graph/noteEditor.svelte.ts b/frontend/src/lib/components/graph/noteEditor.svelte.ts new file mode 100644 index 0000000000..4d59cfa480 --- /dev/null +++ b/frontend/src/lib/components/graph/noteEditor.svelte.ts @@ -0,0 +1,322 @@ +import type { FlowNote } from '$lib/gen' +import type { StateStore } from '$lib/utils' +import type { ExtendedOpenFlow } from '../flows/types' +import type { NoteColor } from './noteColors' +import { DEFAULT_GROUP_NOTE_COLOR, getNextAvailableColor } from './noteColors' +import { generateId } from './util' +import { getContext, setContext } from 'svelte' +import { completeAndSplitGroup } from './groupDetectionUtils' + +/** + * Utility class for editing flow notes via direct flowStore mutations + * This class is designed to be used in editor contexts via Svelte context + */ +export class NoteEditor { + private flowStore: StateStore + private onNoteAdded?: () => void + + constructor(flowStore: StateStore, onNoteAdded?: () => void) { + this.flowStore = flowStore + this.onNoteAdded = onNoteAdded + } + + /** + * Get the current notes array from the flow store + */ + private getNotes(): FlowNote[] { + return this.flowStore.val.value?.notes || [] + } + + /** + * Set the notes array in the flow store + */ + private setNotes(notes: FlowNote[]): void { + if (this.flowStore.val.value) { + this.flowStore.val.value.notes = notes + } + } + + /** + * Add a new note to the flow + */ + addNote(note: Omit): string { + const notes = this.getNotes() + const newNote: FlowNote = { + id: generateId(), + ...note + } + this.setNotes([...notes, newNote]) + + // Call callback to enable notes display when a note is created + this.onNoteAdded?.() + + return newNote.id + } + + /** + * Update the text content of a note + */ + updateText(noteId: string, text: string): void { + const notes = this.getNotes() + const updatedNotes = notes.map((note) => (note.id === noteId ? { ...note, text } : note)) + this.setNotes(updatedNotes) + } + + /** + * Update the color of a note + */ + updateColor(noteId: string, color: NoteColor): void { + const notes = this.getNotes() + const updatedNotes = notes.map((note) => (note.id === noteId ? { ...note, color } : note)) + this.setNotes(updatedNotes) + } + + /** + * Update the position of a note + */ + updatePosition(noteId: string, position: { x: number; y: number }): void { + const notes = this.getNotes() + const updatedNotes = notes.map((note) => (note.id === noteId ? { ...note, position } : note)) + this.setNotes(updatedNotes) + } + + /** + * Update the size of a note + */ + updateSize(noteId: string, size: { width: number; height: number }): void { + const notes = this.getNotes() + const updatedNotes = notes.map((note) => (note.id === noteId ? { ...note, size } : note)) + this.setNotes(updatedNotes) + } + + /** + * Toggle the locked state of a note + */ + updateLock(noteId: string, locked: boolean): void { + const notes = this.getNotes() + const updatedNotes = notes.map((note) => (note.id === noteId ? { ...note, locked } : note)) + this.setNotes(updatedNotes) + } + + /** + * Delete a note from the flow + */ + deleteNote(noteId: string): void { + const notes = this.getNotes() + const updatedNotes = notes.filter((note) => note.id !== noteId) + this.setNotes(updatedNotes) + } + + /** + * Find which nodes from the given list are already in existing group notes + */ + private findNodesInExistingGroups(nodeIds: string[]): { + overlappingGroups: FlowNote[] + nodesInGroups: Set + } { + const notes = this.getNotes() + const groupNotes = notes.filter((note) => note.type === 'group') + const overlappingGroups: FlowNote[] = [] + const nodesInGroups = new Set() + + for (const groupNote of groupNotes) { + const containedNodeIds = groupNote.contained_node_ids || [] + const hasOverlap = nodeIds.some((nodeId) => containedNodeIds.includes(nodeId)) + + if (hasOverlap) { + overlappingGroups.push(groupNote) + containedNodeIds.forEach((nodeId) => nodesInGroups.add(nodeId)) + } + } + + return { overlappingGroups, nodesInGroups } + } + + /** + * Get smart color for group note based on existing groups + */ + private getSmartGroupNoteColor(nodeIds: string[]): NoteColor { + const { overlappingGroups } = this.findNodesInExistingGroups(nodeIds) + + // If no overlapping groups, use default color + if (overlappingGroups.length === 0) { + return DEFAULT_GROUP_NOTE_COLOR + } + + // Get colors used by overlapping groups + const usedColors = new Set() + overlappingGroups.forEach((group) => { + if (group.color) { + usedColors.add(group.color as NoteColor) + } + }) + + // Return next available color + return getNextAvailableColor(usedColors) + } + + /** + * Create a group note containing the specified node IDs + */ + createGroupNote( + nodeIds: string[], + text: string = '### Group note\nDouble click to edit me' + ): string { + // Filter ids in case they contain subflow nodes + let filteredNodeIds: string[] = nodeIds + let subflowIds: string[] = [] + for (const id of nodeIds) { + if (id.startsWith('subflow:')) { + const match = id.match(/^subflow:([^:]+)/) + if (match) { + subflowIds.push(match[1]) + } + } + } + if (subflowIds.length > 0) { + filteredNodeIds = filteredNodeIds.filter((id) => !subflowIds.includes(id)) + filteredNodeIds = [...filteredNodeIds, ...subflowIds] + } + + // Position and size will be calculated dynamically by layout + const smartColor = this.getSmartGroupNoteColor(filteredNodeIds) + + const groupNote: Omit = { + text, + color: smartColor, + type: 'group', + contained_node_ids: filteredNodeIds, + locked: false + } + + return this.addNote(groupNote) + } + + /** + * Check if a node is the only member of an existing group note + */ + isNodeOnlyMemberOfGroupNote(nodeId: string): boolean { + const notes = this.getNotes() + const groupNotes = notes.filter((note) => note.type === 'group') + + for (const groupNote of groupNotes) { + const containedNodeIds = groupNote.contained_node_ids || [] + if (containedNodeIds.length === 1 && containedNodeIds.includes(nodeId)) { + return true + } + } + + return false + } + + /** + * Check if editing is available (flowStore is properly initialized) + */ + isAvailable(): boolean { + return !!this.flowStore.val.value + } + + /** + * Clean up group notes using DAG path completion + */ + cleanupGroupNotes(flowNodes: { id: string; parentIds?: string[]; offset?: number }[]): void { + if (!this.isAvailable()) { + return + } + + const allNotes = this.getNotes() + const groupNotes = allNotes.filter((note) => note.type === 'group') + if (groupNotes.length === 0) return + + let hasChanges = false + const nodeSet = new Set(flowNodes.map((n) => n.id)) + + // Step 1: Clean invalid nodes from existing group notes + for (const note of groupNotes) { + const originalIds = note.contained_node_ids || [] + const validIds = originalIds.filter((id) => nodeSet.has(id)) + + if (validIds.length !== originalIds.length) { + note.contained_node_ids = validIds + hasChanges = true + } + } + + // Step 2: Complete paths for each group using the DAG algorithm + const splitGroups: FlowNote[] = [] + + for (const note of groupNotes) { + const originalNodes = note.contained_node_ids || [] + if (originalNodes.length === 0) continue + + // Use the DAG path completion and splitting algorithm + const completedGroups = completeAndSplitGroup(originalNodes, flowNodes) + + if (completedGroups.length <= 1) { + // Single group or no change needed + const completeNodes = completedGroups.length > 0 ? completedGroups[0] : [] + const sortedComplete = completeNodes.sort() + const sortedOriginal = originalNodes.sort() + + if ( + sortedComplete.length !== sortedOriginal.length || + !sortedComplete.every((id, i) => id === sortedOriginal[i]) + ) { + note.contained_node_ids = completeNodes + hasChanges = true + } + } else { + // Multiple groups - split into separate notes + hasChanges = true + // Mark original note for removal + note.contained_node_ids = [] + + // Create new notes for each completed group + for (const completedGroup of completedGroups) { + splitGroups.push({ + ...note, + id: generateId(), + contained_node_ids: completedGroup + }) + } + } + } + + // Remove empty group notes and add split component notes + const nonEmptyGroupNotes = groupNotes.filter( + (note) => (note.contained_node_ids?.length || 0) > 0 + ) + + if (hasChanges || splitGroups.length > 0) { + const updatedNotes = [ + ...allNotes.filter((note) => note.type !== 'group'), + ...nonEmptyGroupNotes, + ...splitGroups + ] + this.setNotes(updatedNotes) + } + } +} + +/** + * Context type for NoteEditor + */ +export type NoteEditorContext = { + noteEditor: NoteEditor +} + +const CONTEXT_KEY = 'NoteEditorContext' + +/** + * Set the NoteEditor context (used in FlowBuilder) + */ +export function setNoteEditorContext(noteEditor: NoteEditor): void { + setContext(CONTEXT_KEY, { noteEditor }) +} + +/** + * Get the NoteEditor context (used in components that need editing capabilities) + */ +export function getNoteEditorContext(): NoteEditorContext | undefined { + return getContext(CONTEXT_KEY) +} diff --git a/frontend/src/lib/components/graph/noteManager.svelte.ts b/frontend/src/lib/components/graph/noteManager.svelte.ts new file mode 100644 index 0000000000..6e605327a8 --- /dev/null +++ b/frontend/src/lib/components/graph/noteManager.svelte.ts @@ -0,0 +1,150 @@ +import type { FlowNote } from '$lib/gen' +import type { Node } from '@xyflow/svelte' +import { getLayoutSignature, getPropertySignature } from './noteUtils.svelte' +import { deepEqual } from 'fast-equals' +import { untrack } from 'svelte' + +/** + * Utility class for managing flow note text height caching, selection, and fine-grained reactivity + * Handles both fast visual updates and structural changes + */ +export class NoteManager { + renderCount = $state(0) + + // Track notes for layout change detection + #notes: () => FlowNote[] + #previousLayoutSignature: ReturnType = $state({ + notesCount: 0, + noteIds: [], + groupMemberships: [] + }) + #previousPropertySignature: ReturnType = $state([]) + + // Function to update nodes array with reactivity + #setNodes: (nodes: Node[]) => void + #getNodes: () => Node[] + + // Selection state + #selectedNoteId = $state(undefined) + + constructor(notes: () => FlowNote[], setNodes: (nodes: Node[]) => void, getNodes: () => Node[]) { + this.#notes = notes + this.#setNodes = setNodes + this.#getNodes = getNodes + + // Effect to monitor note changes with dual signature tracking + $effect(() => { + const currentNotes = this.#notes() + const currentLayoutSignature = getLayoutSignature(currentNotes) + const currentPropertySignature = getPropertySignature(currentNotes) + + untrack(() => { + const hasLayoutChanges = !deepEqual(currentLayoutSignature, this.#previousLayoutSignature) + const hasPropertyChanges = !deepEqual( + currentPropertySignature, + this.#previousPropertySignature + ) + + if (hasLayoutChanges) { + // Structural changes require full re-render + this.#previousLayoutSignature = currentLayoutSignature + this.#previousPropertySignature = currentPropertySignature + this.render() + } else if (hasPropertyChanges) { + // Property changes can be handled with fast updates + this.#updateNodesProperties(currentNotes) + this.#previousPropertySignature = currentPropertySignature + } + }) + }) + } + + /** + * Triggers a re-render of the graph by incrementing the render count + */ + render(): void { + this.renderCount++ + } + + /** + * Update node properties using setter function for proper reactivity + * Only updates visual properties that don't affect layout + */ + #updateNodesProperties(currentNotes: FlowNote[]): void { + const currentNodes = this.#getNodes() + if (currentNodes.length === 0) return + + // Create a new array with updated nodes to trigger reactivity + const updatedNodes = currentNodes.map((node) => { + const note = currentNotes.find((n) => n.id === node.id) + if (!note || node.type !== 'note') return node + + // Clone the node to avoid mutation + const updatedNode = { ...node, data: { ...node.data } } + + // Update properties that don't affect layout + if (updatedNode.data) { + updatedNode.data.text = note.text + updatedNode.data.color = note.color + updatedNode.data.locked = note.locked || false + } + + // Update draggable property based on lock state + const isGroupNote = note.type === 'group' + updatedNode.draggable = isGroupNote ? false : !note.locked + + // Update free note size and position (group notes are calculated differently) + if (!isGroupNote && note.size && note.position) { + updatedNode.width = note.size.width + updatedNode.height = note.size.height + updatedNode.position = { ...note.position } + } + + return updatedNode + }) + + // Use setter function to trigger reactivity + this.#setNodes(updatedNodes) + } + + /** + * Select a note by ID (single selection only) + */ + selectNote(noteId: string): void { + if (this.#selectedNoteId === noteId) { + return + } + this.#selectedNoteId = noteId + } + + /** + * Clear note selection + */ + clearNoteSelection(): void { + this.#selectedNoteId = undefined + } + + /** + * Deselect a note by ID (single selection only) + */ + deselectNote(noteId?: string): void { + if (this.#selectedNoteId === noteId) { + this.#selectedNoteId = undefined + } + } + + /** + * Check if a note is currently selected + */ + isNoteSelected(noteId: string): boolean { + return this.#selectedNoteId === noteId + } + + // Handle keyboard shortcuts + handleKeyDown(event: KeyboardEvent) { + if (event.key === 'Escape') { + // Escape key clears selection regardless of mode + this.clearNoteSelection() + } + } +} diff --git a/frontend/src/lib/components/graph/noteUtils.svelte.ts b/frontend/src/lib/components/graph/noteUtils.svelte.ts new file mode 100644 index 0000000000..d84817a693 --- /dev/null +++ b/frontend/src/lib/components/graph/noteUtils.svelte.ts @@ -0,0 +1,420 @@ +import type { FlowNote } from '$lib/gen' +import type { Node } from '@xyflow/svelte' +import { deepEqual } from 'fast-equals' +import { calculateNodesBoundsWithOffset } from './util' +import { MIN_NOTE_WIDTH, MIN_NOTE_HEIGHT } from './noteColors' +import type { NodeLayout } from './graphBuilder.svelte' +import { topologicalSort } from './graphBuilder.svelte' +import type { AssetWithAltAccessType } from '../assets/lib' +import type { NoteEditorContext } from './noteEditor.svelte' +import { StickyNote } from 'lucide-svelte' + +export type NodeDep = { + id: string + position: { x: number; y: number } + data?: { assets?: AssetWithAltAccessType[] } + parentIds?: string[] + offset?: number + type?: string +} + +export type NoteComputeResult = { + noteNodes: (Node & NodeLayout)[] + newNodePositions: Record +} + +export type AIToolSpacingInfo = { + toolNodes: (Node & NodeLayout)[] + toolEdges: any[] + newNodePositions: Record +} + +export interface GroupNoteBounds { + x: number + y: number + width: number + height: number +} + +let computeNoteNodesCache: + | [NodeDep[], FlowNote[], Record, NoteComputeResult] + | undefined + +/** + * Extracts layout-affecting signature for change detection + * Only includes properties that affect graph layout (structure, grouping) + */ +export function getLayoutSignature(notes: FlowNote[]) { + return { + notesCount: notes.length, + noteIds: notes.map((n) => n.id).sort(), + // Group memberships affect layout spacing + groupMemberships: notes + .filter((note) => note.type === 'group') + .map((note) => ({ + id: note.id, + containedIds: note.contained_node_ids?.slice().sort() || [] + })) + .sort((a, b) => a.id.localeCompare(b.id)) + } +} + +/** + * Extracts property-only signature for change detection + * Only includes visual/content properties that don't affect layout + */ +export function getPropertySignature(notes: FlowNote[]) { + return notes + .map((note) => ({ + id: note.id, + text: note.text, + color: note.color, + locked: note.locked || false, + position: { ...note.position }, + size: { ...note.size } + })) + .sort((a, b) => a.id.localeCompare(b.id)) +} + +/** + * Calculates z-index values for all notes + * Group notes are ordered by their topmost node's hierarchy position + * Free notes get undefined z-index to use SvelteFlow's native behavior + */ +export function calculateAllNoteZIndexes( + notes: FlowNote[], + nodes: NodeDep[] +): Record { + const zIndexMap: Record = {} + + // Use topological sort to get proper hierarchy order based on parentIds relationships + const sortedNodes = topologicalSort(nodes).reverse() + + // Create a mapping from node ID to its hierarchy position (topological order) + const nodeHierarchyMap: Record = {} + sortedNodes.forEach((node, index) => { + nodeHierarchyMap[node.id] = index + }) + + // Process each note + for (const note of notes) { + if (note.type === 'free') { + // Free notes use SvelteFlow's native z-index behavior (last selected on top) + zIndexMap[note.id] = undefined + } else if (note.type === 'group') { + // Group notes get z-index based on topmost contained node's hierarchy + // Since sortedNodes is in topological order, the first matching node is the topmost + const topmostNode = sortedNodes.find((node) => note.contained_node_ids?.includes(node.id)) + + if (topmostNode) { + const hierarchyPosition = nodeHierarchyMap[topmostNode.id] ?? 0 + // Higher hierarchy position = lower z-index (appears behind) + // Use negative values starting from -2000 to stay below other elements + zIndexMap[note.id] = hierarchyPosition - 2000 + } else { + // Fallback for group notes without valid contained nodes + zIndexMap[note.id] = -2000 + } + } + } + + return zIndexMap +} + +/** + * Calculate extra spacing needed for asset nodes of the topmost node + */ +function calculateExtraAssetSpacing(topmostNodeId: string, nodes: NodeDep[]): number { + // Find the topmost node position + const topmostNode = nodes.find((n) => n.id === topmostNodeId) + if (!topmostNode) { + return 0 + } + + // Find actual asset nodes for the topmost node: {topmostNodeId}-asset-in, type 'asset' + const assetNodes = nodes.filter((n) => n.id.startsWith(`${topmostNodeId}-asset-in-`)) + + if (assetNodes.length === 0) { + return 0 + } + + // Calculate the spacing based on actual asset node positions + const assetSpacing = Math.max( + ...assetNodes.map((assetNode) => { + // Calculate how much space the asset node takes above the main node + return Math.max(0, -assetNode.position.y) + }) + ) + + return assetSpacing +} + +/** + * Calculate extra spacing needed for AI tool nodes of the topmost node + */ +function calculateExtraAIToolSpacing(topmostNodeId: string, nodes: NodeDep[]): number { + // Find the topmost node position + const topmostNode = nodes.find((n) => n.id === topmostNodeId) + if (!topmostNode) { + return 0 + } + + // Find actual AI tool nodes for the topmost node: {topmostNodeId}-tool-, type 'aiTool' + const toolNodes = nodes.filter((n) => n.id.startsWith(`${topmostNodeId}-tool-`)) + + if (toolNodes.length === 0) { + return 0 + } + + // Calculate the spacing based on actual AI tool node positions + const toolSpacing = Math.max( + ...toolNodes.map((toolNode) => { + // Calculate how much space the tool node takes above/below the main node + return Math.max(0, -toolNode.position.y) + }) + ) + + return toolSpacing +} + +/** + * Calculate position and size for group notes based on contained nodes + */ +function calculateGroupNoteLayout( + note: FlowNote, + nodes: NodeDep[], + textHeight: number = 60, + topMostNodeId: string +): { position: { x: number; y: number }; size: { width: number; height: number } } { + if (note.type !== 'group' || !note.contained_node_ids?.length) { + return { + position: note.position ?? { x: 0, y: 0 }, + size: note.size ?? { width: MIN_NOTE_WIDTH, height: MIN_NOTE_HEIGHT } + } + } + + const containedNodes = nodes.filter((node) => note.contained_node_ids?.includes(node.id)) + + if (containedNodes.length === 0) { + return { + position: note.position ?? { x: 0, y: 0 }, + size: note.size ?? { width: MIN_NOTE_WIDTH, height: MIN_NOTE_HEIGHT } + } + } + + const bounds = calculateNodesBoundsWithOffset( + note.contained_node_ids || [], + nodes.map((n) => ({ + id: n.id, + position: n.position, + data: { offset: n.offset ?? 0 }, + type: n.type ?? '' + })) + ) + + const padding = 16 + + // Calculate extra spacing for asset nodes and AI tool nodes of the topmost node + const extraAssetSpacing = topMostNodeId ? calculateExtraAssetSpacing(topMostNodeId, nodes) : 0 + + const extraAIToolSpacing = topMostNodeId ? calculateExtraAIToolSpacing(topMostNodeId, nodes) : 0 + + const totalTextHeight = textHeight + extraAssetSpacing + extraAIToolSpacing + + return { + position: { + x: bounds.minX - padding, + y: bounds.minY - totalTextHeight - padding + }, + size: { + width: bounds.maxX - bounds.minX + 2 * padding, + height: bounds.maxY - bounds.minY + totalTextHeight + 2 * padding + } + } +} + +/** + * Create common data object for note nodes + */ +function createNoteData( + note: FlowNote, + onTextHeightChange: (noteId: string, height: number) => void, + isGroupNote: boolean, + editMode: boolean +) { + return { + noteId: note.id, + text: note.text, + color: note.color, + locked: note.locked || false, + isGroupNote, + editMode, + ...(isGroupNote && { containedNodeIds: note.contained_node_ids || [] }), + onTextHeightChange: (textHeight: number) => { + onTextHeightChange(note.id, textHeight) + } + } +} + +/** + * Main function to compute note nodes and adjust nodes position based on group notes + */ +export function computeNoteNodes( + nodes: NodeDep[], + notes: FlowNote[], + noteTextHeights: Record, + onTextHeightChange: (noteId: string, height: number) => void, + editMode: boolean = false, + noteEditorContext: NoteEditorContext | undefined +): NoteComputeResult { + // Check cache first + if ( + computeNoteNodesCache && + deepEqual(nodes, computeNoteNodesCache[0]) && + deepEqual(notes, computeNoteNodesCache[1]) && + deepEqual(noteTextHeights, computeNoteNodesCache[2]) + ) { + return computeNoteNodesCache[3] + } + + if (editMode) { + if (noteEditorContext?.noteEditor?.isAvailable()) { + noteEditorContext.noteEditor.cleanupGroupNotes(nodes) + } + } + + const allNoteNodes: (Node & NodeLayout)[] = [] + + // Build a map of Y positions that need extra spacing for group notes + const yPosMap: Record = {} // Y position -> spacing needed + + // Group notes that need spacing + const groupNotes = notes.filter((n) => n.type === 'group') + + const topMostNodesMap: Record = {} + + const sortedNodes = topologicalSort(nodes).reverse() + + for (const groupNote of groupNotes) { + if (groupNote.contained_node_ids?.length) { + const topmostNodeId = sortedNodes.find((node) => + groupNote.contained_node_ids?.includes(node.id) + )?.id + const topmostNode = nodes.find((node) => node.id === topmostNodeId) + if (topmostNode) { + const textHeight = noteTextHeights[groupNote.id] || 60 + const spacing = textHeight + 16 // padding + // Mark this Y position as needing spacing + yPosMap[topmostNode.position.y] = Math.max(yPosMap[topmostNode.position.y] || 0, spacing) + topMostNodesMap[groupNote.id] = topmostNode.id + } + } + } + + // Calculate new positions for nodes (offset by group notes) + const sortedNewNodes = nodes + .map((n) => ({ position: { ...n.position }, id: n.id })) + .sort((a, b) => a.position.y - b.position.y) + + let currentYOffset = 0 + let prevYPos = NaN + + for (const node of sortedNewNodes) { + if (node.position.y !== prevYPos) { + // Add spacing for group notes at this Y level + if (yPosMap[node.position.y]) { + currentYOffset += yPosMap[node.position.y] + } + prevYPos = node.position.y + } + node.position.y += currentYOffset + } + + // Create note nodes AFTER calculating adjusted node positions + // For group notes, we need to use the adjusted node positions + const adjustedNodes = sortedNewNodes.map((n) => { + const origNode = nodes.find((orig) => orig.id === n.id) + return { + ...n, + data: origNode?.data, + offset: origNode?.offset, + type: origNode?.type + } + }) + + // Calculate all z-indexes at once using hierarchy information + const noteZIndexes = calculateAllNoteZIndexes(notes, nodes) + + for (const note of notes) { + const isGroupNote = note.type === 'group' + const zIndex = noteZIndexes[note.id] + + // Calculate position and size using adjusted node positions for group notes + const { position, size } = isGroupNote + ? calculateGroupNoteLayout( + note, + adjustedNodes, + noteTextHeights[note.id] || 60, + topMostNodesMap[note.id] + ) + : { + position: note.position ?? { x: 0, y: 0 }, + size: note.size ?? { width: MIN_NOTE_WIDTH, height: MIN_NOTE_HEIGHT } + } + + // Create the note node + const noteNode: Node & NodeLayout = { + id: note.id, + type: 'note' as any, // Note nodes are handled specially + position, + width: size.width, + height: size.height, + zIndex, + draggable: isGroupNote ? false : editMode && !note.locked, + selectable: false, + data: createNoteData(note, onTextHeightChange, isGroupNote, editMode) as any + } + + allNoteNodes.push(noteNode) + } + + const newNodePositions: Record = Object.fromEntries( + sortedNewNodes.map((n) => [n.id, n.position]) + ) + + const result: NoteComputeResult = { + noteNodes: allNoteNodes, + newNodePositions + } + + // Cache the result + computeNoteNodesCache = [ + structuredClone($state.snapshot(nodes)), + structuredClone($state.snapshot(notes)), + structuredClone($state.snapshot(noteTextHeights)), + result + ] + + return result +} + +export function addGroupNoteContextMenuItem( + nodeId: string, + noteEditorContext: NoteEditorContext | undefined +) { + const isDisabled = + !noteEditorContext?.noteEditor || + (noteEditorContext?.noteEditor?.isNodeOnlyMemberOfGroupNote(nodeId) ?? false) + + return { + id: 'add-group-note', + label: 'Add note', + icon: StickyNote, + disabled: isDisabled, + onClick: () => { + if (noteEditorContext?.noteEditor && !isDisabled) { + noteEditorContext.noteEditor.createGroupNote([nodeId]) + } + } + } +} diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index 568f16c0a0..3010a94325 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -2,8 +2,6 @@ import InsertModulePopover from '$lib/components/flows/map/InsertModulePopover.svelte' import { getBezierPath, BaseEdge, type EdgeProps, EdgeLabel } from '@xyflow/svelte' import { ClipboardCopy, Hourglass } from 'lucide-svelte' - import { getContext } from 'svelte' - import type { Writable } from 'svelte/store' import type { GraphEventHandlers } from '../../graphBuilder.svelte' import { getStraightLinePath } from '../utils' import { twMerge } from 'tailwind-merge' @@ -13,11 +11,9 @@ import type { Job } from '$lib/gen' import type { GraphModuleState } from '../../model' import InsertModuleButton from '$lib/components/flows/map/InsertModuleButton.svelte' + import { getGraphContext } from '../../graphContext' - const { useDataflow, showAssets } = getContext<{ - useDataflow: Writable - showAssets?: Writable - }>('FlowGraphContext') + const { useDataflow, showAssets } = getGraphContext() let { // id, diff --git a/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte index 47a099b7c8..0f622bc65c 100644 --- a/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/EmptyEdge.svelte @@ -1,8 +1,7 @@ @@ -322,7 +322,7 @@
+{/snippet} + +
{ + dragging = false + }} + ondragstart={() => { + dragging = true + }} + ondragend={() => { + dragging = false + }} + onmouseenter={handleMouseEnter} + onmouseleave={handleMouseLeave} + role="button" + tabindex={editMode ? -1 : 0} + ondblclick={handleDoubleClick} + use:clickOutside={{ + onClickOutside: () => { + noteManager?.deselectNote(data.noteId) + } + }} +> + + {#if hovering || selected} + {#if !editMode && isEditModeAvailable} +
+ {locked + ? 'Note is locked' + : isEditModeAvailable + ? 'Double click to edit' + : 'View only mode'} +
+ {:else if !locked && isEditModeAvailable} +
GH Markdown
+ {/if} + {/if} + + +
+ {#if editMode} + + + {:else} + +
containerHeight, + (v) => { + if (v > 0 && v !== containerHeight) { + data.onTextHeightChange?.(v) + } + containerHeight = v + } + } + > + {#if textForDisplay} +
+ +
+ {:else} +
+ Double click to edit me +
+ {/if} +
+ {/if} +
+ + + {#if !locked && isEditModeAvailable} + { + // Update note size when resizing ends + if (params.width !== undefined && params.height !== undefined) { + const size = { width: params.width, height: params.height } + if (isEditModeAvailable && noteEditorContext?.noteEditor) { + // Use NoteEditor context in edit mode + noteEditorContext.noteEditor.updateSize(data.noteId, size) + } + } + }} + /> + {/if} + + + {#if isEditModeAvailable} + {#if data.isGroupNote && currentNode?.position} + + +
+ {@render actionButtons()} +
+ + {:else} + +
+ {@render actionButtons()} +
+ {/if} + {/if} +
+ + diff --git a/frontend/src/lib/components/graph/renderers/nodes/ResultNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/ResultNode.svelte index 3917206edb..13ede87991 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/ResultNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/ResultNode.svelte @@ -1,19 +1,17 @@ @@ -22,7 +20,7 @@ id={'Result'} label={'Result'} selectable={true} - selected={$selectedId === 'Result'} + selected={selectionManager && selectionManager.isNodeSelected(id)} hideId={true} on:select={(e) => { setTimeout(() => data?.eventHandlers?.select(e.detail)) diff --git a/frontend/src/lib/components/graph/renderers/nodes/SubflowBound.svelte b/frontend/src/lib/components/graph/renderers/nodes/SubflowBound.svelte index bb23d4566a..7e5438dda0 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/SubflowBound.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/SubflowBound.svelte @@ -5,12 +5,16 @@ import NodeWrapper from './NodeWrapper.svelte' import { Minimize2 } from 'lucide-svelte' import type { SubflowBoundN } from '../../graphBuilder.svelte' + import { getGraphContext } from '../../graphContext' interface Props { data: SubflowBoundN['data'] + id: string } - let { data }: Props = $props() + let { data, id }: Props = $props() + + const { selectionManager } = getGraphContext() @@ -19,7 +23,7 @@ label={data.label} preLabel={data.preLabel} selectable - selected={data.selected} + selected={selectionManager && selectionManager.isNodeSelected(id)} on:select={() => { setTimeout(() => data.eventHandlers?.select(data.id)) }} diff --git a/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte index 7283739b5f..58f95d4b33 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte @@ -1,9 +1,10 @@ @@ -76,26 +82,26 @@ const primarySchedule = triggersState.triggers.findIndex((t) => t.isPrimary && !t.isDraft) triggersState.selectedTriggerIndex = primarySchedule }} - on:select={() => data?.eventHandlers?.select('triggers')} + on:select={() => data?.eventHandlers?.select('Trigger')} onSelect={async (triggerIndex: number) => { - data?.eventHandlers?.select('triggers') + data?.eventHandlers?.select('Trigger') await tick() triggersState.selectedTriggerIndex = triggerIndex }} onAddDraftTrigger={async (type: TriggerType) => { const newTrigger = triggersState.addDraftTrigger(triggersCount, type) - data?.eventHandlers?.select('triggers') + data?.eventHandlers?.select('Trigger') await tick() triggersState.selectedTriggerIndex = newTrigger }} - selected={$selectedId == 'triggers'} + selected={selectionManager?.getSelectedId() === 'Trigger'} newItem={data.newFlow} /> {:else} { data?.eventHandlers?.select(e.detail) }} @@ -116,7 +122,7 @@ {:else} + +
+
+ {/snippet} + +
+
+ {@render sectionHeader('Scripts', selectAllScripts, clearAllScripts)} + {#if allScripts.length > 0} + + {:else} +

No scripts available

+ {/if} +
+ +
+ {@render sectionHeader('Flows', selectAllFlows, clearAllFlows)} + {#if allFlows.length > 0} + + {:else} +

No flows available

+ {/if} +
+ +
+ {@render sectionHeader('API Endpoints', selectAllEndpoints, clearAllEndpoints)} + e.name))} + placeholder="Select endpoints" + bind:value={selectedEndpoints} + /> +
+ +
+ Selected: {selectedScripts.length} scripts, {selectedFlows.length} flows, {selectedEndpoints.length} + endpoints +
+
+ {/if} + {:else if mcpCreationMode && (newMcpScope !== 'folder' || selectedFolder.length > 0)} {#if loadingRunnables}
createToken(mcpCreationMode)} disabled={mcpCreationMode && - (newTokenWorkspace == undefined || (newMcpScope === 'folder' && !selectedFolder))} + (newTokenWorkspace == undefined || + (newMcpScope === 'folder' && !selectedFolder) || + (newMcpScope === 'custom' && + selectedScripts.length === 0 && + selectedFlows.length === 0 && + selectedEndpoints.length === 0))} variant="accent" > New token diff --git a/frontend/src/lib/components/text_input/TextInput.svelte b/frontend/src/lib/components/text_input/TextInput.svelte index 59fc94be18..9bd226e394 100644 --- a/frontend/src/lib/components/text_input/TextInput.svelte +++ b/frontend/src/lib/components/text_input/TextInput.svelte @@ -20,9 +20,15 @@ export const inputBaseClass = 'rounded-md focus:ring-0 no-default-style text-xs text-primary font-normal !bg-surface-input disabled:!bg-surface-disabled disabled:!border-transparent disabled:!text-disabled disabled:cursor-not-allowed shadow-none !placeholder-hint' + import autosize from '$lib/autosize' import { ButtonType } from '$lib/components/common/button/model' export const inputSizeClasses = { + xs: twMerge( + ButtonType.UnifiedSizingClasses.xs, + ButtonType.UnifiedMinHeightClasses.xs, + 'px-1 !py-0.5' + ), sm: twMerge( ButtonType.UnifiedSizingClasses.sm, ButtonType.UnifiedMinHeightClasses.sm, @@ -33,49 +39,69 @@ } - - { - e.stopImmediatePropagation() - }} - bind:this={inputEl} - bind:value -/> +{#if underlyingInputEl === 'textarea'} + + +{:else if underlyingInputEl === 'input'} + e.stopImmediatePropagation()} + bind:this={inputEl} + bind:value + /> +{/if} diff --git a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte index 291a3e08ac..4e8a37e9e0 100644 --- a/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/email/EmailTriggerEditorInner.svelte @@ -65,6 +65,7 @@ let error_handler_path: string | undefined = $state() let error_handler_args: Record = $state({}) let retry: Retry | undefined = $state() + let enabled = $state(false) // Component references let drawer = $state(undefined) let initialConfig: NewEmailTrigger | undefined = undefined @@ -72,7 +73,7 @@ let optionTabSelected: 'error_handler' | 'retries' = $state('error_handler') let errorHandlerSelected: ErrorHandler = $state('slack') const isAdmin = $derived($userStore?.is_admin || $userStore?.is_super_admin) - const routeConfig = $derived.by(getEmailTriggerConfig) + const emailConfig = $derived.by(getEmailTriggerConfig) const captureConfig = $derived.by(isEditor ? getCaptureConfig : () => ({})) const saveDisabled = $derived( drawerLoading || !can_write || pathError != '' || !isValid || emptyString(script_path) @@ -141,6 +142,7 @@ error_handler_args = defaultValues?.error_handler_args ?? {} retry = defaultValues?.retry ?? undefined errorHandlerSelected = getHandlerType(error_handler_path ?? '') + enabled = defaultValues?.enabled ?? false } finally { clearTimeout(loader) drawerLoading = false @@ -161,6 +163,7 @@ error_handler_args = cfg?.error_handler_args ?? {} retry = cfg?.retry errorHandlerSelected = getHandlerType(error_handler_path ?? '') + enabled = cfg?.enabled ?? false } async function loadTrigger(defaultConfig?: Partial): Promise { @@ -179,11 +182,11 @@ async function triggerScript(): Promise { if (customSaveBehavior) { - customSaveBehavior(routeConfig) + customSaveBehavior(emailConfig) drawer?.closeDrawer() } else { deploymentLoading = true - const saveCfg = routeConfig + const saveCfg = emailConfig const isSaved = await saveEmailTriggerFromCfg( initialPath, saveCfg, @@ -210,17 +213,30 @@ extra_perms: extraPerms, error_handler_path, error_handler_args, - retry + retry, + enabled } return nCfg } + async function handleToggleEnabled(newEnabled: boolean) { + enabled = newEnabled + if (!trigger?.draftConfig) { + await EmailTriggerService.setEmailTriggerEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: newEnabled } + }) + sendUserToast(`${newEnabled ? 'enabled' : 'disabled'} email trigger ${initialPath}`) + } + } + // Update config for captures function getCaptureConfig() { const newCaptureConfig = { - local_part: routeConfig.local_part, - path: routeConfig.path + local_part: emailConfig.local_part, + path: emailConfig.path } // return newCaptureConfig @@ -233,7 +249,7 @@ $effect(() => { if (!drawerLoading) { - handleConfigChange(routeConfig, initialConfig, saveDisabled, edit, onConfigChange) + handleConfigChange(emailConfig, initialConfig, saveDisabled, edit, onConfigChange) } }) @@ -338,8 +354,9 @@ {trigger} permissions={drawerLoading || !can_write ? 'none' : can_write && isAdmin ? 'create' : 'write'} {saveDisabled} - enabled={undefined} + {enabled} {allowDraft} + onToggleEnabled={handleToggleEnabled} {edit} isLoading={deploymentLoading} onUpdate={triggerScript} diff --git a/frontend/src/lib/components/triggers/email/utils.ts b/frontend/src/lib/components/triggers/email/utils.ts index 82de2a0a0d..49a94093eb 100644 --- a/frontend/src/lib/components/triggers/email/utils.ts +++ b/frontend/src/lib/components/triggers/email/utils.ts @@ -15,21 +15,22 @@ export function getEmailAddress( export async function saveEmailTriggerFromCfg( initialPath: string, - routeCfg: Record, + emailCfg: Record, edit: boolean, workspace: string, isAdmin: boolean, usedTriggerKinds: Writable ): Promise { const requestBody: NewEmailTrigger = { - path: routeCfg.path, - script_path: routeCfg.script_path, - local_part: routeCfg.local_part, - is_flow: routeCfg.is_flow, - workspaced_local_part: routeCfg.workspaced_local_part, - error_handler_path: routeCfg.error_handler_path, - error_handler_args: routeCfg.error_handler_path ? routeCfg.error_handler_args : undefined, - retry: routeCfg.retry + path: emailCfg.path, + script_path: emailCfg.script_path, + local_part: emailCfg.local_part, + is_flow: emailCfg.is_flow, + workspaced_local_part: emailCfg.workspaced_local_part, + error_handler_path: emailCfg.error_handler_path, + error_handler_args: emailCfg.error_handler_path ? emailCfg.error_handler_args : undefined, + retry: emailCfg.retry, + enabled: emailCfg.enabled } try { if (edit) { @@ -38,16 +39,16 @@ export async function saveEmailTriggerFromCfg( path: initialPath, requestBody: { ...requestBody, - local_part: isAdmin || !edit ? routeCfg.local_part : undefined + local_part: isAdmin || !edit ? emailCfg.local_part : undefined } }) - sendUserToast(`Route ${routeCfg.path} updated`) + sendUserToast(`Email trigger ${emailCfg.path} updated`) } else { await EmailTriggerService.createEmailTrigger({ workspace: workspace, - requestBody: requestBody + requestBody: { ...requestBody, enabled: true } }) - sendUserToast(`Route ${routeCfg.path} created`) + sendUserToast(`Email trigger ${emailCfg.path} created`) } if (!get(usedTriggerKinds).includes('email')) { usedTriggerKinds.update((t) => [...t, 'email']) diff --git a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte index 2fb5bfeb26..085360df6a 100644 --- a/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte +++ b/frontend/src/lib/components/triggers/http/RouteEditorInner.svelte @@ -83,6 +83,7 @@ let static_asset_config = $state<{ s3: string; storage?: string; filename?: string } | undefined>( undefined ) + let enabled: boolean = $state(false) let is_static_website = $state(false) let s3FileUploadRawMode = $state(false) let workspaced_route = $state(false) @@ -235,6 +236,7 @@ s3FileUploadRawMode = defaultValues?.s3FileUploadRawMode ?? false path = defaultValues?.path ?? '' initialPath = '' + enabled = defaultValues?.enabled ?? false dirtyPath = false is_static_website = defaultValues?.is_static_website ?? false workspaced_route = defaultValues?.workspaced_route ?? false @@ -268,6 +270,7 @@ wrap_body = cfg?.wrap_body ?? false raw_string = cfg?.raw_string ?? false summary = cfg?.summary ?? '' + enabled = cfg?.enabled ?? false routeDescription = cfg?.description ?? '' authentication_resource_path = cfg?.authentication_resource_path ?? '' if (cfg?.authentication_method === 'custom_script') { @@ -344,6 +347,7 @@ http_method, request_type, workspaced_route, + enabled, wrap_body, raw_string, authentication_resource_path, @@ -361,6 +365,19 @@ return nCfg } + async function handleToggleEnabled(newEnabled: boolean) { + enabled = newEnabled + if (!trigger?.draftConfig) { + await HttpTriggerService.setHttpTriggerEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: newEnabled } + }) + sendUserToast(`${newEnabled ? 'enabled' : 'disabled'} HTTP trigger ${initialPath}`) + } + } + + // Update config for captures function getCaptureConfig() { const newCaptureConfig = { @@ -829,7 +846,8 @@ {trigger} permissions={drawerLoading || !can_write ? 'none' : can_write && isAdmin ? 'create' : 'write'} {saveDisabled} - enabled={undefined} + {enabled} + onToggleEnabled={handleToggleEnabled} {allowDraft} {edit} isLoading={deploymentLoading} diff --git a/frontend/src/lib/components/triggers/http/utils.ts b/frontend/src/lib/components/triggers/http/utils.ts index 31c70dc5dd..5cb2cf2b4f 100644 --- a/frontend/src/lib/components/triggers/http/utils.ts +++ b/frontend/src/lib/components/triggers/http/utils.ts @@ -60,7 +60,8 @@ export async function saveHttpRouteFromCfg( summary: routeCfg.summary, error_handler_path: routeCfg.error_handler_path, error_handler_args: routeCfg.error_handler_path ? routeCfg.error_handler_args : undefined, - retry: routeCfg.retry + retry: routeCfg.retry, + enabled: routeCfg.enabled } try { if (edit) { @@ -76,7 +77,7 @@ export async function saveHttpRouteFromCfg( } else { await HttpTriggerService.createHttpTrigger({ workspace: workspace, - requestBody: requestBody + requestBody: { ...requestBody, enabled: true } }) sendUserToast(`Route ${routeCfg.path} created`) } diff --git a/frontend/src/lib/components/tutorials/FlowBuilderTutorialForLoop.svelte b/frontend/src/lib/components/tutorials/FlowBuilderTutorialForLoop.svelte index a5b6d42e58..39d548343d 100644 --- a/frontend/src/lib/components/tutorials/FlowBuilderTutorialForLoop.svelte +++ b/frontend/src/lib/components/tutorials/FlowBuilderTutorialForLoop.svelte @@ -8,7 +8,7 @@ import { nextId } from '../flows/flowModuleNextId' const dispatch = createEventDispatcher() - const { flowStore, selectedId, flowStateStore } = + const { flowStore, selectionManager, flowStateStore } = getContext('FlowEditorContext') let tutorial: Tutorial | undefined = undefined @@ -158,7 +158,7 @@ title: 'Step of the loop', description: 'We added an action to the loop. Let’s configure it', onNextClick: () => { - $selectedId = tempId + selectionManager.selectId(tempId) dispatch('reload') setTimeout(() => { diff --git a/frontend/src/lib/components/vscode.ts b/frontend/src/lib/components/vscode.ts index 14438d1ad1..22e955dc5c 100644 --- a/frontend/src/lib/components/vscode.ts +++ b/frontend/src/lib/components/vscode.ts @@ -291,4 +291,4 @@ export function keepModelAroundToAvoidDisposalOfWorkers() { } } -export let MONACO_Y_PADDING = 7 +export let MONACO_Y_PADDING = 6.5 diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 86d69636f8..5b131cf45d 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -291,7 +291,7 @@
+ +
+

Context Menus

+

Right-click triggered menus with contextual actions.

+
+ + +
+

Context menu tests

+ +
+
+
+

Basic Context Menu

+

Right-click the area below

+
+ +
+ Right-click me for context menu +
+ +
+ +
+
+

Text Context Menu

+

Right-click the text below

+
+ +
+

+ Right-click this text to open the context menu. You can test various interactions + here. +

+
+ +
+ +
+
+

Button Context Menu

+

Right-click the button below

+
+ + + +
+
+
+

Input Components

diff --git a/frontend/src/routes/view_graph/+page.svelte b/frontend/src/routes/view_graph/+page.svelte index 18b49846a8..86bbba17cf 100644 --- a/frontend/src/routes/view_graph/+page.svelte +++ b/frontend/src/routes/view_graph/+page.svelte @@ -4,7 +4,7 @@ import { decodeState } from '$lib/utils' let content = localStorage.getItem('svelvet') - const { modules, failureModule, preprocessorModule } = content + const { modules, failureModule, preprocessorModule, notes } = content ? decodeState(content) : { modules: [], failureModule: undefined, preprocessorModule: undefined } @@ -15,6 +15,7 @@ {modules} {failureModule} {preprocessorModule} + {notes} />